blender-archive/source/blender/blenlib/intern/string_utf8.c

/*
 * $Id$
 *
 * ***** BEGIN GPL LICENSE BLOCK *****
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 *
 * The Original Code is Copyright (C) 2011 Blender Foundation.
 * All rights reserved.
 *
 * Contributor(s): Campbell Barton.
 *
 * ***** END GPL LICENSE BLOCK *****
 *
 */

 /** \file blender/blenlib/intern/string_utf8.c
 *  \ingroup bli
 */

#include <string.h>
#include <wchar.h>
#include <wctype.h>

#include "BLI_string.h"

/* from libswish3, originally called u8_isvalid(),
 * modified to return the index of the bad character (byte index not utf).
 * http://svn.swish-e.org/libswish3/trunk/src/libswish3/utf8.c r3044 - campbell */

/* based on the valid_utf8 routine from the PCRE library by Philip Hazel

   length is in bytes, since without knowing whether the string is valid
   it's hard to know how many characters there are! */

static const char trailingBytesForUTF8[256] = {
	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
};

int BLI_utf8_invalid_byte(const char *str, int length)
{
	const unsigned char *p, *pend = (unsigned char*)str + length;
	unsigned char c;
	int ab;

	for (p = (unsigned char*)str; p < pend; p++) {
		c = *p;
		if (c < 128)
			continue;
		if ((c & 0xc0) != 0xc0)
			goto utf8_error;
		ab = trailingBytesForUTF8[c];
		if (length < ab)
			goto utf8_error;
		length -= ab;

		p++;
		/* Check top bits in the second byte */
		if ((*p & 0xc0) != 0x80)
			goto utf8_error;

		/* Check for overlong sequences for each different length */
		switch (ab) {
			/* Check for xx00 000x */
		case 1:
			if ((c & 0x3e) == 0) goto utf8_error;
			continue;   /* We know there aren't any more bytes to check */

			/* Check for 1110 0000, xx0x xxxx */
		case 2:
			if (c == 0xe0 && (*p & 0x20) == 0) goto utf8_error;
			break;

			/* Check for 1111 0000, xx00 xxxx */
		case 3:
			if (c == 0xf0 && (*p & 0x30) == 0) goto utf8_error;
			break;

			/* Check for 1111 1000, xx00 0xxx */
		case 4:
			if (c == 0xf8 && (*p & 0x38) == 0) goto utf8_error;
			break;

			/* Check for leading 0xfe or 0xff,
			   and then for 1111 1100, xx00 00xx */
		case 5:
			if (c == 0xfe || c == 0xff ||
				(c == 0xfc && (*p & 0x3c) == 0)) goto utf8_error;
			break;
		}

		/* Check for valid bytes after the 2nd, if any; all must start 10 */
		while (--ab > 0) {
			if ((*(p+1) & 0xc0) != 0x80) goto utf8_error;
			p++; /* do this after so we get usable offset - campbell */
		}
	}

	return -1;

utf8_error:

	return (int)((char *)p - (char *)str) - 1;
}

int BLI_utf8_invalid_strip(char *str, int length)
{
	int bad_char, tot= 0;

	while((bad_char= BLI_utf8_invalid_byte(str, length)) != -1) {
		str += bad_char;
		length -= bad_char;

		if(length == 0) {
			/* last character bad, strip it */
			*str= '\0';
			tot++;
			break;
		}
		else {
			/* strip, keep looking */
			memmove(str, str + 1, length);
			tot++;
		}
	}

	return tot;
}


/* compatible with BLI_strncpy, but esnure no partial utf8 chars */

/* array copied from glib's gutf8.c,
 * note: this looks to be at odd's with 'trailingBytesForUTF8',
 * need to find out what gives here! - campbell */
static const size_t utf8_skip_data[256] = {
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
    3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1
};

char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy)
{
	char *dst_r= dst;
	size_t utf8_size;

	/* note: currently we dont attempt to deal with invalid utf8 chars */

	while(*src != '\0' && (utf8_size= utf8_skip_data[*src]) < maxncpy) {
		maxncpy -= utf8_size;
		switch(utf8_size) {
			case 6: *dst ++ = *src ++;
			case 5: *dst ++ = *src ++;
			case 4: *dst ++ = *src ++;
			case 3: *dst ++ = *src ++;
			case 2: *dst ++ = *src ++;
			case 1: *dst ++ = *src ++;
		}
	}
	*dst= '\0';
	return dst_r;
}


/* --------------------------------------------------------------------------*/
/* wchar_t / utf8 functions  */

/* UTF-8 <-> wchar transformations */
static size_t chtoutf8(const unsigned long c, char o[4])
{
	// Variables and initialization
/*	memset(o, 0, 4);	*/

	// Create the utf-8 string
	if (c < 0x80) {
		o[0] = (char) c;
		return 1;
	}
	else if (c < 0x800) {
		o[0] = (0xC0 | (c>>6));
		o[1] = (0x80 | (c & 0x3f));
		return 2;
	}
	else if (c < 0x10000) {
		o[0] = (0xe0 | (c >> 12));
		o[1] = (0x80 | (c >>6 & 0x3f));
		o[2] = (0x80 | (c & 0x3f));
		return 3;
	}
	else if (c < 0x200000) {
		o[0] = (0xf0 | (c>>18));
		o[1] = (0x80 | (c >>12 & 0x3f));
		o[2] = (0x80 | (c >> 6 & 0x3f));
		o[3] = (0x80 | (c & 0x3f));
		return 4;
	}

	/* should we assert here? */
	return 0;
}

size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy)
{
	size_t len = 0;
	while(*src && len < maxcpy) { /* XXX can still run over the buffer because utf8 size isnt known :| */
		len += chtoutf8(*src++, dst+len);
	}

	dst[len]= '\0';

	return len;
}

/* wchar len in utf8 */
size_t BLI_wstrlen_utf8(const wchar_t *src)
{
	char ch_dummy[4];
	size_t len = 0;

	while(*src) {
		len += chtoutf8(*src++, ch_dummy);
	}

	return len;
}

// utf8slen
size_t BLI_strlen_utf8(const char *strc)
{
	int len=0;

	while(*strc) {
		if ((*strc & 0xe0) == 0xc0) {
			if((strc[1] & 0x80) && (strc[1] & 0x40) == 0x00)
				strc++;
		} else if ((*strc & 0xf0) == 0xe0) {
			if((strc[1] & strc[2] & 0x80) && ((strc[1] | strc[2]) & 0x40) == 0x00)
				strc += 2;
		} else if ((*strc & 0xf8) == 0xf0) {
			if((strc[1] & strc[2] & strc[3] & 0x80) && ((strc[1] | strc[2] | strc[3]) & 0x40) == 0x00)
				strc += 3;
		}

		strc++;
		len++;
	}

	return len;
}


/* Converts Unicode to wchar

According to RFC 3629 "UTF-8, a transformation format of ISO 10646"
(http://tools.ietf.org/html/rfc3629), the valid UTF-8 encoding are:

  Char. number range  |        UTF-8 octet sequence
	  (hexadecimal)    |              (binary)
   --------------------+---------------------------------------------
   0000 0000-0000 007F | 0xxxxxxx
   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

If the encoding incidated by the first character is incorrect (because the
1 to 3 following characters do not match 10xxxxxx), the output is a '?' and
only a single input character is consumed.

*/

size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst_w, const char *src_c, const size_t maxcpy)
{
	int len=0;

	if(dst_w==NULL || src_c==NULL) return(0);

	while(*src_c && len < maxcpy) {
		if ((*src_c & 0xe0) == 0xc0) {
			if((src_c[1] & 0x80) && (src_c[1] & 0x40) == 0x00) {
				*dst_w=((src_c[0] &0x1f)<<6) | (src_c[1]&0x3f);
				src_c++;
			} else {
				*dst_w = '?';
			}
		} else if ((*src_c & 0xf0) == 0xe0) {
			if((src_c[1] & src_c[2] & 0x80) && ((src_c[1] | src_c[2]) & 0x40) == 0x00) {
				*dst_w=((src_c[0] & 0x0f)<<12) | ((src_c[1]&0x3f)<<6) | (src_c[2]&0x3f);
				src_c += 2;
			} else {
				*dst_w = '?';
			}
		} else if ((*src_c & 0xf8) == 0xf0) {
			if((src_c[1] & src_c[2] & src_c[3] & 0x80) && ((src_c[1] | src_c[2] | src_c[3]) & 0x40) == 0x00) {
				*dst_w=((src_c[0] & 0x07)<<18) | ((src_c[1]&0x1f)<<12) | ((src_c[2]&0x3f)<<6) | (src_c[3]&0x3f);
				src_c += 3;
			} else {
				*dst_w = '?';
			}
		} else {
			*dst_w=(src_c[0] & 0x7f);
		}

		src_c++;
		dst_w++;
		len++;
	}
	return len;
}

/* end wchar_t / utf8 functions  */
/* --------------------------------------------------------------------------*/





/* copied from glib */
/**
 * g_utf8_find_prev_char:
 * @str: pointer to the beginning of a UTF-8 encoded string
 * @p: pointer to some position within @str
 *
 * Given a position @p with a UTF-8 encoded string @str, find the start
 * of the previous UTF-8 character starting before @p. Returns %NULL if no
 * UTF-8 characters are present in @str before @p.
 *
 * @p does not have to be at the beginning of a UTF-8 character. No check
 * is made to see if the character found is actually valid other than
 * it starts with an appropriate byte.
 *
 * Return value: a pointer to the found character or %NULL.
 **/
char * BLI_str_find_prev_char_utf8(const char *str, const char *p)
{
	for (--p; p >= str; --p) {
		if ((*p & 0xc0) != 0x80) {
			return (char *)p;
		}
	}
	return NULL;
}

/**
 * g_utf8_find_next_char:
 * @p: a pointer to a position within a UTF-8 encoded string
 * @end: a pointer to the byte following the end of the string,
 * or %NULL to indicate that the string is nul-terminated.
 *
 * Finds the start of the next UTF-8 character in the string after @p.
 *
 * @p does not have to be at the beginning of a UTF-8 character. No check
 * is made to see if the character found is actually valid other than
 * it starts with an appropriate byte.
 *
 * Return value: a pointer to the found character or %NULL
 **/
char *BLI_str_find_next_char_utf8(const char *p, const char *end)
{
	if (*p) {
		if (end) {
			for (++p; p < end && (*p & 0xc0) == 0x80; ++p) {
				/* do nothing */
			}
		}
		else {
			for (++p; (*p & 0xc0) == 0x80; ++p) {
				/* do nothing */
			}
		}
	}
	return (p == end) ? NULL : (char *)p;
}

/**
 * g_utf8_prev_char:
 * @p: a pointer to a position within a UTF-8 encoded string
 *
 * Finds the previous UTF-8 character in the string before @p.
 *
 * @p does not have to be at the beginning of a UTF-8 character. No check
 * is made to see if the character found is actually valid other than
 * it starts with an appropriate byte. If @p might be the first
 * character of the string, you must use g_utf8_find_prev_char() instead.
 *
 * Return value: a pointer to the found character.
 **/
char *BLI_str_prev_char_utf8(const char *p)
{
	while (1) {
		p--;
		if ((*p & 0xc0) != 0x80) {
			return (char *)p;
		}
	}
}
/* end glib copy */