archive/blender-archive
Archived
1
1
Fork 0
Code Pull Requests Packages Activity

- add BLI_string_utf8.h for unicode functions.

Browse Source 
- move font.c unicode functions into string_utf8.c and rename to fit with other BLI_string funcs.
This commit is contained in:
Campbell Barton
2011-10-20 09:47:05 +00:00
parent e02dfe4a79
commit 8d6a554d75
21 changed files with 234 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
source/blender/blenkernel/BKE_font.h
View File

@@ -85,11 +85,6 @@ struct chartrans *BKE_text_to_curve(struct Main *bmain, struct Scene *scene, str
int BKE_font_getselection(struct Object *ob, int *start, int *end);
size_t chtoutf8(const unsigned long c, char o[4]);
void wcs2utf8s(char *dst, const wchar_t *src);
size_t wcsleninu8(wchar_t *src);
size_t utf8towchar(wchar_t *w, const char *c);
#ifdef __cplusplus
}
#endif

140
source/blender/blenkernel/intern/font.c
View File

@@ -64,142 +64,6 @@
static ListBase ttfdata= {NULL, NULL};
/* UTF-8 <-> wchar transformations */
size_t chtoutf8(const unsigned long c, char o[4])
{
// Variables and initialization
/* memset(o, 0, 4); */
// Create the utf-8 string
if (c < 0x80) {
o[0] = (char) c;
return 1;
}
else if (c < 0x800) {
o[0] = (0xC0 | (c>>6));
o[1] = (0x80 | (c & 0x3f));
return 2;
}
else if (c < 0x10000) {
o[0] = (0xe0 | (c >> 12));
o[1] = (0x80 | (c >>6 & 0x3f));
o[2] = (0x80 | (c & 0x3f));
return 3;
}
else if (c < 0x200000) {
o[0] = (0xf0 | (c>>18));
o[1] = (0x80 | (c >>12 & 0x3f));
o[2] = (0x80 | (c >> 6 & 0x3f));
o[3] = (0x80 | (c & 0x3f));
return 4;
}
/* should we assert here? */
return 0;
}
void wcs2utf8s(char *dst, const wchar_t *src)
{
while(*src) {
dst += chtoutf8(*src++, dst);
}
*dst= '\0';
}
size_t wcsleninu8(wchar_t *src)
{
char ch_dummy[4];
size_t len = 0;
while(*src) {
len += chtoutf8(*src++, ch_dummy);
}
return len;
}
static size_t utf8slen(const char *strc)
{
int len=0;
while(*strc) {
if ((*strc & 0xe0) == 0xc0) {
if((strc[1] & 0x80) && (strc[1] & 0x40) == 0x00)
strc++;
} else if ((*strc & 0xf0) == 0xe0) {
if((strc[1] & strc[2] & 0x80) && ((strc[1] | strc[2]) & 0x40) == 0x00)
strc += 2;
} else if ((*strc & 0xf8) == 0xf0) {
if((strc[1] & strc[2] & strc[3] & 0x80) && ((strc[1] | strc[2] | strc[3]) & 0x40) == 0x00)
strc += 3;
}
strc++;
len++;
}
return len;
}
/* Converts Unicode to wchar
According to RFC 3629 "UTF-8, a transformation format of ISO 10646"
(http://tools.ietf.org/html/rfc3629), the valid UTF-8 encoding are:
Char. number range | UTF-8 octet sequence
(hexadecimal) | (binary)
--------------------+---------------------------------------------
0000 0000-0000 007F | 0xxxxxxx
0000 0080-0000 07FF | 110xxxxx 10xxxxxx
0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
If the encoding incidated by the first character is incorrect (because the
1 to 3 following characters do not match 10xxxxxx), the output is a '?' and
only a single input character is consumed.
*/
size_t utf8towchar(wchar_t *w, const char *c)
{
int len=0;
if(w==NULL || c==NULL) return(0);
while(*c) {
if ((*c & 0xe0) == 0xc0) {
if((c[1] & 0x80) && (c[1] & 0x40) == 0x00) {
*w=((c[0] &0x1f)<<6) | (c[1]&0x3f);
c++;
} else {
*w = '?';
}
} else if ((*c & 0xf0) == 0xe0) {
if((c[1] & c[2] & 0x80) && ((c[1] | c[2]) & 0x40) == 0x00) {
*w=((c[0] & 0x0f)<<12) | ((c[1]&0x3f)<<6) | (c[2]&0x3f);
c += 2;
} else {
*w = '?';
}
} else if ((*c & 0xf8) == 0xf0) {
if((c[1] & c[2] & c[3] & 0x80) && ((c[1] | c[2] | c[3]) & 0x40) == 0x00) {
*w=((c[0] & 0x07)<<18) | ((c[1]&0x1f)<<12) | ((c[2]&0x3f)<<6) | (c[3]&0x3f);
c += 3;
} else {
*w = '?';
}
} else
*w=(c[0] & 0x7f);
c++;
w++;
len++;
}
return len;
}
/* The vfont code */
void free_vfont(struct VFont *vf)
{
@@ -691,10 +555,10 @@ struct chartrans *BKE_text_to_curve(Main *bmain, Scene *scene, Object *ob, int m
if(vfont == NULL) return NULL;
// Create unicode string
utf8len = utf8slen(cu->str);
utf8len = BLI_strlen_utf8(cu->str);
mem = MEM_callocN(((utf8len + 1) * sizeof(wchar_t)), "convertedmem");
utf8towchar(mem, cu->str);
BLI_strncpy_wchar_from_utf8(mem, cu->str, utf8len + 1);
// Count the wchar_t string length
slen = wcslen(mem);

1
source/blender/blenkernel/intern/particle_system.c
View File

@@ -70,7 +70,6 @@
#include "BLI_listbase.h"
#include "BLI_threads.h"
#include "BLI_storage.h" /* For _LARGEFILE64_SOURCE; zlib needs this on some systems */
#include "BLI_string.h"
#include "BLI_utildefines.h"
#include "BKE_main.h"

3
source/blender/blenkernel/intern/smoke.c
View File

@@ -812,13 +812,12 @@ void smokeModifier_copy(struct SmokeModifierData *smd, struct SmokeModifierData
}
}
#ifdef WITH_SMOKE
// forward decleration
static void smoke_calc_transparency(float *result, float *input, float *p0, float *p1, int res[3], float dx, float *light, bresenham_callback cb, float correct);
static float calc_voxel_transp(float *result, float *input, int res[3], int *pixel, float *tRay, float correct);
#ifdef WITH_SMOKE
static int get_lamp(Scene *scene, float *light)
{
Base *base_tmp = NULL;

2
source/blender/blenlib/BLI_blenlib.h
View File

@@ -74,6 +74,8 @@ extern "C" {
#include "BLI_string.h"
#include "BLI_string_utf8.h"
#include "BLI_path_util.h"
#include "BLI_storage.h"

12
source/blender/blenlib/BLI_string.h
View File

@@ -25,8 +25,6 @@
* Contributor(s): none yet.
*
* ***** END GPL LICENSE BLOCK *****
*
* $Id$
*/
#ifndef BLI_STRING_H
@@ -142,16 +140,6 @@ void BLI_timestr(double _time, char *str); /* time var is global */
void BLI_ascii_strtolower(char *str, int len);
void BLI_ascii_strtoupper(char *str, int len);
/* string_utf8.c - may move these into their own header some day - campbell */
char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy);
int BLI_utf8_invalid_byte(const char *str, int length);
int BLI_utf8_invalid_strip(char *str, int length);
/* copied from glib */
char *BLI_str_find_prev_char_utf8(const char *str, const char *p);
char *BLI_str_find_next_char_utf8(const char *p, const char *end);
char *BLI_str_prev_char_utf8(const char *p);
#ifdef __cplusplus
}
#endif

55
source/blender/blenlib/BLI_string_utf8.h Normal file
View File

@@ -0,0 +1,55 @@
/*
* $Id:
*
* ***** BEGIN GPL LICENSE BLOCK *****
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
* Contributor(s): none yet.
*
* ***** END GPL LICENSE BLOCK *****
*/
#ifndef BLI_STRING_UTF8_H
#define BLI_STRING_UTF8_H
/** \file BLI_string.h
* \ingroup bli
*/
#ifdef __cplusplus
extern "C" {
#endif
char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy);
int BLI_utf8_invalid_byte(const char *str, int length);
int BLI_utf8_invalid_strip(char *str, int length);
/* copied from glib */
char *BLI_str_find_prev_char_utf8(const char *str, const char *p);
char *BLI_str_find_next_char_utf8(const char *p, const char *end);
char *BLI_str_prev_char_utf8(const char *p);
/* wchar_t functions, copied from blenders own font.c originally */
size_t BLI_wstrlen_utf8(const wchar_t *src);
size_t BLI_strlen_utf8(const char *strc);
size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy);
size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst, const char *src, const size_t maxcpy);
#ifdef __cplusplus
}
#endif
#endif

1
source/blender/blenlib/CMakeLists.txt
View File

@@ -128,6 +128,7 @@ set(SRC
BLI_storage.h
BLI_storage_types.h
BLI_string.h
BLI_string_utf8.h
BLI_threads.h
BLI_utildefines.h
BLI_uvproject.h

1
source/blender/blenlib/intern/BLI_args.c
View File

@@ -39,7 +39,6 @@
#include "MEM_guardedalloc.h"
#include "BLI_listbase.h"
#include "BLI_string.h"
#include "BLI_utildefines.h"
#include "BLI_args.h"
#include "BLI_ghash.h"

155
source/blender/blenlib/intern/string_utf8.c
View File

@@ -31,6 +31,8 @@
*/
#include <string.h>
#include <wchar.h>
#include <wctype.h>
#include "BLI_string.h"
@@ -183,6 +185,159 @@ char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy)
return dst_r;
}
/* --------------------------------------------------------------------------*/
/* wchar_t / utf8 functions */
/* UTF-8 <-> wchar transformations */
static size_t chtoutf8(const unsigned long c, char o[4])
{
// Variables and initialization
/* memset(o, 0, 4); */
// Create the utf-8 string
if (c < 0x80) {
o[0] = (char) c;
return 1;
}
else if (c < 0x800) {
o[0] = (0xC0 | (c>>6));
o[1] = (0x80 | (c & 0x3f));
return 2;
}
else if (c < 0x10000) {
o[0] = (0xe0 | (c >> 12));
o[1] = (0x80 | (c >>6 & 0x3f));
o[2] = (0x80 | (c & 0x3f));
return 3;
}
else if (c < 0x200000) {
o[0] = (0xf0 | (c>>18));
o[1] = (0x80 | (c >>12 & 0x3f));
o[2] = (0x80 | (c >> 6 & 0x3f));
o[3] = (0x80 | (c & 0x3f));
return 4;
}
/* should we assert here? */
return 0;
}
size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy)
{
size_t len = 0;
while(*src && len < maxcpy) { /* XXX can still run over the buffer because utf8 size isnt known :| */
len += chtoutf8(*src++, dst+len);
}
dst[len]= '\0';
return len;
}
/* wchar len in utf8 */
size_t BLI_wstrlen_utf8(const wchar_t *src)
{
char ch_dummy[4];
size_t len = 0;
while(*src) {
len += chtoutf8(*src++, ch_dummy);
}
return len;
}
// utf8slen
size_t BLI_strlen_utf8(const char *strc)
{
int len=0;
while(*strc) {
if ((*strc & 0xe0) == 0xc0) {
if((strc[1] & 0x80) && (strc[1] & 0x40) == 0x00)
strc++;
} else if ((*strc & 0xf0) == 0xe0) {
if((strc[1] & strc[2] & 0x80) && ((strc[1] | strc[2]) & 0x40) == 0x00)
strc += 2;
} else if ((*strc & 0xf8) == 0xf0) {
if((strc[1] & strc[2] & strc[3] & 0x80) && ((strc[1] | strc[2] | strc[3]) & 0x40) == 0x00)
strc += 3;
}
strc++;
len++;
}
return len;
}
/* Converts Unicode to wchar
According to RFC 3629 "UTF-8, a transformation format of ISO 10646"
(http://tools.ietf.org/html/rfc3629), the valid UTF-8 encoding are:
Char. number range | UTF-8 octet sequence
(hexadecimal) | (binary)
--------------------+---------------------------------------------
0000 0000-0000 007F | 0xxxxxxx
0000 0080-0000 07FF | 110xxxxx 10xxxxxx
0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
If the encoding incidated by the first character is incorrect (because the
1 to 3 following characters do not match 10xxxxxx), the output is a '?' and
only a single input character is consumed.
*/
size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst_w, const char *src_c, const size_t maxcpy)
{
int len=0;
if(dst_w==NULL || src_c==NULL) return(0);
while(*src_c && len < maxcpy) {
if ((*src_c & 0xe0) == 0xc0) {
if((src_c[1] & 0x80) && (src_c[1] & 0x40) == 0x00) {
*dst_w=((src_c[0] &0x1f)<<6) | (src_c[1]&0x3f);
src_c++;
} else {
*dst_w = '?';
}
} else if ((*src_c & 0xf0) == 0xe0) {
if((src_c[1] & src_c[2] & 0x80) && ((src_c[1] | src_c[2]) & 0x40) == 0x00) {
*dst_w=((src_c[0] & 0x0f)<<12) | ((src_c[1]&0x3f)<<6) | (src_c[2]&0x3f);
src_c += 2;
} else {
*dst_w = '?';
}
} else if ((*src_c & 0xf8) == 0xf0) {
if((src_c[1] & src_c[2] & src_c[3] & 0x80) && ((src_c[1] | src_c[2] | src_c[3]) & 0x40) == 0x00) {
*dst_w=((src_c[0] & 0x07)<<18) | ((src_c[1]&0x1f)<<12) | ((src_c[2]&0x3f)<<6) | (src_c[3]&0x3f);
src_c += 3;
} else {
*dst_w = '?';
}
} else {
*dst_w=(src_c[0] & 0x7f);
}
src_c++;
dst_w++;
len++;
}
return len;
}
/* end wchar_t / utf8 functions */
/* --------------------------------------------------------------------------*/
/* copied from glib */
/**
* g_utf8_find_prev_char:

14
source/blender/editors/curve/editfont.c
View File

@@ -221,13 +221,13 @@ static void update_string(Curve *cu)
MEM_freeN(cu->str);
// Calculate the actual string length in UTF-8 variable characters
len = wcsleninu8(ef->textbuf);
len = BLI_wstrlen_utf8(ef->textbuf);
// Alloc memory for UTF-8 variable char length string
cu->str = MEM_callocN(len + sizeof(wchar_t), "str");
// Copy the wchar to UTF-8
wcs2utf8s(cu->str, ef->textbuf);
BLI_strncpy_wchar_as_utf8(cu->str, ef->textbuf, len + 1);
}
static int insert_into_textbuf(Object *obedit, uintptr_t c)
@@ -373,7 +373,7 @@ static int paste_file(bContext *C, ReportList *reports, const char *filename)
if(cu->len+filelen<MAXTEXT) {
int tmplen;
wchar_t *mem = MEM_callocN((sizeof(wchar_t)*filelen)+(4*sizeof(wchar_t)), "temporary");
tmplen = utf8towchar(mem, strp);
tmplen = BLI_strncpy_wchar_from_utf8(mem, strp, filelen + 1);
wcscat(ef->textbuf, mem);
MEM_freeN(mem);
cu->len += tmplen;
@@ -1241,10 +1241,10 @@ static int insert_text_exec(bContext *C, wmOperator *op)
return OPERATOR_CANCELLED;
inserted_utf8= RNA_string_get_alloc(op->ptr, "text", NULL, 0);
len= strlen(inserted_utf8);
len= BLI_strlen_utf8(inserted_utf8);
inserted_text= MEM_callocN(sizeof(wchar_t)*(len+1), "FONT_insert_text");
utf8towchar(inserted_text, inserted_utf8);
BLI_strncpy_wchar_from_utf8(inserted_text, inserted_utf8, len+1);
for(a=0; a<len; a++)
insert_into_textbuf(obedit, inserted_text[a]);
@@ -1348,7 +1348,7 @@ static int insert_text_invoke(bContext *C, wmOperator *op, wmEvent *evt)
/* store as utf8 in RNA string */
char inserted_utf8[8] = {0};
wcs2utf8s(inserted_utf8, inserted_text);
BLI_strncpy_wchar_as_utf8(inserted_utf8, inserted_text, sizeof(inserted_utf8));
RNA_string_set(op->ptr, "text", inserted_utf8);
}
@@ -1478,7 +1478,7 @@ void make_editText(Object *obedit)
}
// Convert the original text to wchar_t
utf8towchar(ef->textbuf, cu->str);
BLI_strncpy_wchar_from_utf8(ef->textbuf, cu->str, MAXTEXT+4); /* length is bogus */
wcscpy(ef->oldstr, ef->textbuf);
cu->len= wcslen(ef->textbuf);

1
source/blender/editors/interface/interface_anim.c
View File

@@ -38,6 +38,7 @@
#include "BLI_listbase.h"
#include "BLI_string.h"
#include "BLI_string_utf8.h"
#include "BLI_utildefines.h"
#include "BKE_context.h"

4
source/blender/editors/interface/interface_draw.c
View File

@@ -600,13 +600,13 @@ static void ui_draw_but_CHARTAB(uiBut *but)
wstr[0] = cs;
if(strcmp(G.selfont->name, FO_BUILTIN_NAME))
{
wcs2utf8s((char *)ustr, (wchar_t *)wstr);
BLI_strncpy_wchar_as_utf8((char *)ustr, (wchar_t *)wstr, sizeof(ustr));
}
else
{
if(G.ui_international == TRUE)
{
wcs2utf8s((char *)ustr, (wchar_t *)wstr);
BLI_strncpy_wchar_as_utf8((char *)ustr, (wchar_t *)wstr, sizeof(ustr));
}
else
{

1
source/blender/editors/interface/interface_templates.c
View File

@@ -38,7 +38,6 @@
#include "DNA_scene_types.h"
#include "DNA_userdef_types.h"
#include "BLI_string.h"
#include "BLI_utildefines.h"
#include "BLI_ghash.h"

1
source/blender/editors/space_file/fsmenu.c
View File

@@ -44,7 +44,6 @@
#include "BLI_blenlib.h"
#include "BLI_linklist.h"
#include "BLI_dynstr.h"
#include "BLI_string.h"
#ifdef WIN32
#include <windows.h> /* need to include windows.h so _WIN32_IE is defined */

3
source/blender/imbuf/intern/indexer.c
View File

@@ -29,10 +29,11 @@
#include "AVI_avi.h"
#include "imbuf.h"
#include "MEM_guardedalloc.h"
#include "BLI_utildefines.h"
#include "BLI_blenlib.h"
#include "BLI_math_base.h"
#include "BLI_string.h"
#include "MEM_guardedalloc.h"
#include "DNA_userdef_types.h"
#include "BKE_global.h"

2
source/blender/makesrna/intern/rna_curve.c
View File

@@ -483,7 +483,7 @@ void rna_Curve_body_set(PointerRNA *ptr, const char *value)
cu->str = MEM_callocN(len + sizeof(wchar_t), "str");
cu->strinfo = MEM_callocN( (len+4) *sizeof(CharInfo), "strinfo"); /* don't know why this is +4, just duplicating load_editText() */
//wcs2utf8s(cu->str, value); // value is not wchar_t
//BLI_strncpy_wchar_as_utf8(cu->str, value, len+1); // value is not wchar_t
BLI_strncpy(cu->str, value, len+1);
}

1
source/blender/makesrna/intern/rna_define.c
View File

@@ -39,7 +39,6 @@
#include "DNA_genfile.h"
#include "DNA_sdna_types.h"
#include "BLI_string.h"
#include "BLI_utildefines.h"
#include "BLI_ghash.h"

4
source/blender/python/generic/py_capi_utils.c
View File

@@ -30,7 +30,7 @@
#include "py_capi_utils.h"
#include "BKE_font.h" /* only for utf8towchar, should replace with py funcs but too late in release now */
#include "BLI_string_utf8.h" /* only for BLI_strncpy_wchar_from_utf8, should replace with py funcs but too late in release now */
#ifdef _WIN32 /* BLI_setenv */
#include "BLI_path_util.h"
@@ -478,7 +478,7 @@ void PyC_SetHomePath(const char *py_path_bundle)
/* cant use this, on linux gives bug: #23018, TODO: try LANG="en_US.UTF-8" /usr/bin/blender, suggested 22008 */
/* mbstowcs(py_path_bundle_wchar, py_path_bundle, FILE_MAXDIR); */
utf8towchar(py_path_bundle_wchar, py_path_bundle);
BLI_strncpy_wchar_from_utf8(py_path_bundle_wchar, py_path_bundle, sizeof(py_path_bundle_wchar) / sizeof(wchar_t));
Py_SetPythonHome(py_path_bundle_wchar);
// printf("found python (wchar_t) '%ls'\n", py_path_bundle_wchar);

4
source/blender/python/intern/bpy_interface.c
View File

@@ -52,12 +52,12 @@
#include "BLI_path_util.h"
#include "BLI_math_base.h"
#include "BLI_string.h"
#include "BLI_string_utf8.h"
#include "BLI_utildefines.h"
#include "BKE_context.h"
#include "BKE_text.h"
#include "BKE_font.h" /* only for utf8towchar */
#include "BKE_main.h"
#include "BKE_global.h" /* only for script checking */
@@ -194,7 +194,7 @@ void BPY_python_start(int argc, const char **argv)
/* not essential but nice to set our name */
static wchar_t bprogname_wchar[FILE_MAXDIR+FILE_MAXFILE]; /* python holds a reference */
utf8towchar(bprogname_wchar, bprogname);
BLI_strncpy_wchar_from_utf8(bprogname_wchar, bprogname, sizeof(bprogname_wchar) / sizeof(wchar_t));
Py_SetProgramName(bprogname_wchar);
/* must run before python initializes */

1
source/blender/windowmanager/intern/wm_operators.c
View File

@@ -56,7 +56,6 @@
#include "BLI_blenlib.h"
#include "BLI_dynstr.h" /*for WM_operator_pystring */
#include "BLI_math.h"
#include "BLI_string.h"
#include "BLI_utildefines.h"
#include "BLI_ghash.h"