devs/foxbox
1
0
Fork 0
forked from len0rd/rockbox
Code Pull requests Activity

unicode: Support characters beyond the first unicode plane

Browse source 
We used 16-bit variables to store the 'character code' everywhere but
this won't let us represent anything beyond U+FFFF.

This patch changes those variables to a custom type that can be 32 or 16
bits depending on the build, and adjusts numerous internal APIs and
datastructures to match.  This includes:

 * utf8decode() and friends
 * font manipulation, caching, rendering, and generation
 * on-screen keyboard
 * FAT filesystem (parsing and generating utf16 LFNs)
 * WIN32 simulator platform code

Note that this patch doesn't _enable_ >16bit unicode support; a followup
patch will turn that on for appropriate targets.

Appears to work on:

  * hosted linux, native, linux simulator in both 16/32-bit modes.

Needs testing on:

  * windows and macos simulator (16bit+32bit)

Change-Id: Iba111b27d2433019b6bff937cf1ebd2c4353a0e8
This commit is contained in:
Solomon Peachy 2024-12-17 08:55:21 -05:00
parent 2a88253426
commit a2c10f6189
44 changed files with 476 additions and 330 deletions
Download patch file Download diff file Expand all files Collapse all files

6
apps/plugins/announce_status.c
View file

@ -70,7 +70,7 @@ static const char keybd_layout[] =
* - \n does not create a key, but it also consumes one element
* - the final null terminator is equivalent to \n
* - since sizeof includes the null terminator we don't need +1 for that. */
static unsigned short kbd_buf[sizeof(keybd_layout)];
static ucschar_t kbd_buf[sizeof(keybd_layout)];
/****************** prototypes ******************/
void print_scroll(char* string); /* implements a scrolling screen */
@ -164,7 +164,7 @@ static void config_set_defaults(void)
gAnnounce.announce_on = 0;
gAnnounce.grouping = 0;
gAnnounce.wps_fmt[0] = '\0';
gAnnounce.show_prompt = true;
gAnnounce.show_prompt = true;
}
static void config_reset_voice(void)
@ -250,7 +250,7 @@ static int announce_menu_cb(int action,
struct gui_synclist *this_list)
{
(void)this_item;
unsigned short* kbd_p;
ucschar_t *kbd_p;
int selection = rb->gui_synclist_get_sel_pos(this_list);

10
apps/plugins/frotz/frotz.c
View file

@ -128,7 +128,7 @@ void wait_for_key()
case PLA_EXIT:
hot_key_quit();
break;
case PLA_SELECT:
return;
}
@ -159,7 +159,7 @@ zchar do_input(int timeout, bool show_cursor)
{
case PLA_EXIT:
return ZC_HKEY_QUIT;
case PLA_CANCEL:
menu_ret = menu();
if (menu_ret != ZC_BAD)
@ -174,7 +174,7 @@ zchar do_input(int timeout, bool show_cursor)
return ZC_BAD;
default:
if (timeout != TIMEOUT_BLOCK &&
if (timeout != TIMEOUT_BLOCK &&
!TIME_BEFORE(*rb->current_tick, timeout_at))
return ZC_TIME_OUT;
}
@ -185,7 +185,7 @@ zchar os_read_key(int timeout, bool show_cursor)
{
int r;
char inputbuf[5];
short key;
ucschar_t key;
zchar zkey;
for(;;)
@ -214,7 +214,7 @@ zchar os_read_line(int max, zchar *buf, int timeout, int width, int continued)
char inputbuf[256];
const char *in;
char *out;
short key;
ucschar_t key;
zchar zkey;
for(;;)

8
apps/plugins/keyremap.c
View file

@ -206,8 +206,8 @@ static int prompt_filename(char *buf, size_t bufsz)
{
#define KBD_LAYOUT "abcdefghijklmnop\nqrstuvwxyz |()[]\n1234567890 /._-+\n\n" \
"\nABCDEFGHIJKLMNOP\nQRSTUVWXYZ |()[]\n1234567890 /._-+"
unsigned short kbd[sizeof(KBD_LAYOUT) + 10];
unsigned short *kbd_p = kbd;
ucschar_t kbd[sizeof(KBD_LAYOUT) + 10];
ucschar_t *kbd_p = kbd;
if (!kbd_create_layout(KBD_LAYOUT, kbd, sizeof(kbd)))
kbd_p = NULL;
@ -1002,7 +1002,7 @@ next_line:
{
bufleft = bufsz - (pctx - filenamebuf);
ctx = -1;
int ctx_x_flag_count = (LAST_CONTEXT_PLACEHOLDER
int ctx_x_flag_count = (LAST_CONTEXT_PLACEHOLDER
* ARRAYLEN(context_flags));
for (int i=0;i < ctx_x_flag_count ;i++)
@ -2058,7 +2058,7 @@ static void synclist_set(int id, int selected_item, int items, int sel_size)
}
else if (menu_id == MENU_ID(M_SETKEYS))
{
keyset.view_columns = printcell_set_columns(&lists, NULL,
keyset.view_columns = printcell_set_columns(&lists, NULL,
ACTVIEW_HEADER, Icon_Rockbox);
printcell_enable(true);
int curcol = printcell_get_column_selected();

24
apps/plugins/lib/grey_draw.c
View file

@ -199,8 +199,8 @@ void grey_hline(int x1, int x2, int y)
/* nothing to draw? */
if (y < _grey_info.clip_t || y >= _grey_info.clip_b ||
x1 >= _grey_info.clip_r || x2 < _grey_info.clip_l)
return;
return;
/* drawmode and optimisation */
if (vp->drawmode & DRMODE_INVERSEVID)
{
@ -251,7 +251,7 @@ void grey_vline(int x, int y1, int y2)
unsigned char *dst, *dst_end;
void (*pfunc)(unsigned char *address);
int dwidth;
/* direction flip */
if (y2 < y1)
{
@ -264,7 +264,7 @@ void grey_vline(int x, int y1, int y2)
if (x < _grey_info.clip_l || x >= _grey_info.clip_r ||
y1 >= _grey_info.clip_b || y2 < _grey_info.clip_t)
return;
/* clipping */
if (y1 < _grey_info.clip_t)
y1 = _grey_info.clip_t;
@ -425,7 +425,7 @@ void grey_fillrect(int x, int y, int width, int height)
if (height <= 0)
return;
dwidth = _grey_info.cb_width;
dst = &_grey_info.curbuffer[
_GREY_MULUQ(dwidth, _grey_info.vp->y - _grey_info.cb_y + y) +
@ -653,8 +653,8 @@ void grey_gray_bitmap(const unsigned char *src, int x, int y, int width,
/* Put a string at a given pixel position, skipping first ofs pixel columns */
void grey_putsxyofs(int x, int y, int ofs, const unsigned char *str)
{
int ch;
unsigned short *ucs;
ucschar_t ch;
ucschar_t *ucs;
struct font* pf;
if (_grey_info.clip_b <= _grey_info.clip_t)
@ -680,7 +680,7 @@ void grey_putsxyofs(int x, int y, int ofs, const unsigned char *str)
bits = rb->font_get_bits(pf, ch);
grey_mono_bitmap_part(bits, ofs, 0, width, x, y, width - ofs, pf->height);
x += width - ofs;
ofs = 0;
}
@ -709,7 +709,7 @@ void grey_ub_clear_display(void)
#endif
}
/* Assembler optimised helper function for copying a single line to the
/* Assembler optimised helper function for copying a single line to the
* greyvalue buffer. */
void _grey_line1(int width, unsigned char *dst, const unsigned char *src,
const unsigned char *lut);
@ -725,7 +725,7 @@ void grey_ub_gray_bitmap_part(const unsigned char *src, int src_x, int src_y,
if ((width <= 0) || (height <= 0) || (x >= _grey_info.width)
|| (y >= _grey_info.height) || (x + width <= 0) || (y + height <= 0))
return;
/* clipping */
if (x < 0)
{
@ -744,7 +744,7 @@ void grey_ub_gray_bitmap_part(const unsigned char *src, int src_x, int src_y,
if (y + height > _grey_info.height)
height = _grey_info.height - y;
src += _GREY_MULUQ(stride, src_y) + src_x; /* move starting point */
src += _GREY_MULUQ(stride, src_y) + src_x; /* move starting point */
yc = y;
ye = y + height;
dst = _grey_info.values + (x << _GREY_BSHIFT);
@ -773,7 +773,7 @@ void grey_ub_gray_bitmap_part(const unsigned char *src, int src_x, int src_y,
}
while (src_row < src_end);
#endif
src += stride;
}
while (++yc < ye);

12
apps/plugins/lib/kbd_helper.c
View file

@ -22,8 +22,8 @@
#include "kbd_helper.h"
/* USAGE:
unsigned short kbd[64];
unsigned short *kbd_p = kbd;
ucschar_t kbd[64];
ucschar_t *kbd_p = kbd;
if (!kbd_create_layout("ABCD1234\n", kbd, sizeof(kbd)))
kbd_p = NULL;
@ -34,14 +34,14 @@
* success returns size of buffer used
* failure returns 0
*/
int kbd_create_layout(const char *layout, unsigned short *buf, int bufsz)
int kbd_create_layout(const char *layout, ucschar_t *buf, int bufsz)
{
unsigned short *pbuf;
ucschar_t *pbuf;
const unsigned char *p = layout;
int len = 0;
int total_len = 0;
pbuf = buf;
while (*p && (pbuf - buf + (ptrdiff_t) sizeof(unsigned short)) < bufsz)
while (*p && (pbuf - buf + (ptrdiff_t) sizeof(ucschar_t)) < bufsz)
{
p = rb->utf8decode(p, &pbuf[len+1]);
if (pbuf[len+1] == '\n')
@ -60,7 +60,7 @@ int kbd_create_layout(const char *layout, unsigned short *buf, int bufsz)
*pbuf = len;
pbuf[len+1] = 0xFEFF; /* mark end of characters */
total_len += len + 1;
return total_len * sizeof(unsigned short);
return total_len * sizeof(ucschar_t);
}
return 0;

14
apps/plugins/lib/kbd_helper.h
View file

@ -1,10 +1,10 @@
/***************************************************************************
* __________ __ ___.
* Open \______ \ ____ ____ | | _\_ |__ _______ ___
* Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
* Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
* Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
* \/ \/ \/ \/ \/
* __________ __ ___.
* Open \______ \ ____ ____ | | _\_ |__ _______ ___
* Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
* Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
* Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
* \/ \/ \/ \/ \/
* $Id$
*
* Copyright (C) 2020 William Wilgus
@ -22,6 +22,6 @@
#define KBD_HELPER_H
/* create a custom keyboard layout for kbd_input */
int kbd_create_layout(const char *layout, unsigned short *buf, int bufsz);
int kbd_create_layout(const char *layout, ucschar_t *buf, int bufsz);
#endif /* KBD_HELPER_H */

2
apps/plugins/lib/simple_viewer.c
View file

@ -62,7 +62,7 @@ static const char* get_next_line(const char *text, struct view_info *info)
total = 0;
while(*ptr)
{
unsigned short ch;
ucschar_t ch;
n = ((intptr_t)rb->utf8decode(ptr, &ch) - (intptr_t)ptr);
if (rb->is_diacritic(ch, NULL))
w = 0;

2
apps/plugins/lrcplayer.c
View file

@ -422,7 +422,7 @@ static struct lrc_brpos *calc_brpos(struct lrc_line *lrc_line, int i)
int nlrcbrpos = 0, max_lrcbrpos;
uifont = rb->screens[0]->getuifont();
struct font* pf = rb->font_get(uifont);
unsigned short ch;
ucschar_t ch;
struct snap {
int count, width;
int nword;

3
apps/plugins/lua/rocklib.c
View file

@ -141,6 +141,7 @@ RB_WRAP(touchscreen_mode)
#endif
// XXX this may be broken with 32-bit ucschar_t
RB_WRAP(kbd_input)
{
/*kbd_input(text, layout)*
@ -168,7 +169,7 @@ RB_WRAP(kbd_input)
layout = NULL;
}
if(!rb->kbd_input(buffer, LUAL_BUFFERSIZE, (unsigned short *)layout))
if(!rb->kbd_input(buffer, LUAL_BUFFERSIZE, (ucschar_t *)layout))
{
luaL_addstring(&b, buffer);
luaL_pushresult(&b);

4
apps/plugins/mpegplayer/mpegplayer.c
View file

@ -1073,8 +1073,8 @@ static void draw_oriented_alpha_bitmap_part(const unsigned char *src,
static void draw_putsxy_oriented(int x, int y, const char *str)
{
unsigned short ch;
unsigned short *ucs;
ucschar_t ch;
ucschar_t *ucs;
int ofs = MIN(x, 0);
struct font* pf = rb->font_get(osd.font);

4
apps/plugins/rockpaint.c
View file

@ -969,8 +969,8 @@ static void buffer_alpha_bitmap_part(
static void buffer_putsxyofs( fb_data *buf, int buf_width, int buf_height,
int x, int y, int ofs, const unsigned char *str )
{
unsigned short ch;
unsigned short *ucs;
ucschar_t ch;
ucschar_t *ucs;
struct font *pf = rb->font_get( FONT_UI );
if( !pf ) pf = rb->font_get( FONT_SYSFIXED );

4
apps/plugins/tagcache/tagcache.c
View file

@ -135,10 +135,10 @@ static void sleep_yield(void)
#define yield sleep_yield
}
/* make sure tag can be displayed by font pf*/
/* make sure tag can be displayed by font pf */
static bool text_is_displayable(struct font *pf, unsigned char *src)
{
unsigned short code;
ucschar_t code;
const unsigned char *ptr = src;
while(*ptr)
{

50
apps/plugins/text_viewer/tv_text_processor.c
View file

@ -41,7 +41,7 @@ static unsigned text_type = TV_TEXT_UNKNOWN;
static const unsigned char *end_ptr;
static unsigned short ucsbuf[TV_MAX_BLOCKS][TV_MAX_CHARS_PER_BLOCK];
static ucschar_t ucsbuf[TV_MAX_BLOCKS][TV_MAX_CHARS_PER_BLOCK];
static unsigned char utf8buf[TV_MAX_CHARS_PER_BLOCK * (2 * 3)];
static unsigned char *outbuf;
@ -54,11 +54,11 @@ static bool expand_extra_line = false;
/* when a line is divided, this value sets true. */
static bool is_break_line = false;
static unsigned short break_chars[] =
static unsigned short break_chars[] = // XXX promote to ucschar_t if we get a codepoint > 0xffff
{
0,
/* halfwidth characters */
'\t', '\n', 0x0b, 0x0c, ' ', '!', ',', '-', '.', ':', ';', '?', 0xb7,
'\t', '\n', 0x0b, 0x0c, ' ', '!', ',', '-', '.', ':', ';', '?', 0xb7,
/* fullwidth characters */
0x2010, /* hyphen */
0x3000, /* fullwidth space */
@ -76,7 +76,7 @@ static unsigned short break_chars[] =
};
/* the characters which is not judged as space with isspace() */
static unsigned short extra_spaces[] = { 0, 0x3000 };
static unsigned short extra_spaces[] = { 0, 0x3000 }; // XXX promote to ucschar_t if we get a codepoint > 0xffff
static int tv_glyph_width(int ch)
{
@ -93,7 +93,7 @@ static int tv_glyph_width(int ch)
return rb->font_get_width(rb->font_get(preferences->font_id), ch);
}
static unsigned char *tv_get_ucs(const unsigned char *str, unsigned short *ch)
static unsigned char *tv_get_ucs(const unsigned char *str, ucschar_t *ch)
{
int count = 1;
unsigned char utf8_tmp[3];
@ -148,7 +148,7 @@ static unsigned char *tv_get_ucs(const unsigned char *str, unsigned short *ch)
return (unsigned char *)str + count;
}
static void tv_decode2utf8(const unsigned short *ucs, int count)
static void tv_decode2utf8(const ucschar_t *ucs, int count)
{
int i;
@ -158,7 +158,7 @@ static void tv_decode2utf8(const unsigned short *ucs, int count)
*outbuf = '\0';
}
static bool tv_is_line_break_char(unsigned short ch)
static bool tv_is_line_break_char(ucschar_t ch)
{
size_t i;
@ -166,7 +166,7 @@ static bool tv_is_line_break_char(unsigned short ch)
if (preferences->word_mode == WM_CHOP)
return false;
for (i = 0; i < sizeof(break_chars)/sizeof(unsigned short); i++)
for (i = 0; i < sizeof(break_chars)/sizeof(ucschar_t); i++)
{
if (break_chars[i] == ch)
return true;
@ -174,14 +174,14 @@ static bool tv_is_line_break_char(unsigned short ch)
return false;
}
static bool tv_isspace(unsigned short ch)
static bool tv_isspace(ucschar_t ch)
{
size_t i;
if (ch < 128 && isspace(ch))
return true;
for (i = 0; i < sizeof(extra_spaces)/sizeof(unsigned short); i++)
for (i = 0; i < sizeof(extra_spaces)/sizeof(ucschar_t); i++)
{
if (extra_spaces[i] == ch)
return true;
@ -191,17 +191,17 @@ static bool tv_isspace(unsigned short ch)
static bool tv_is_break_line_join_mode(const unsigned char *next_str)
{
unsigned short ch;
ucschar_t ch;
tv_get_ucs(next_str, &ch);
return tv_isspace(ch);
}
static int tv_form_reflow_line(unsigned short *ucs, int chars)
static int tv_form_reflow_line(ucschar_t *ucs, int chars)
{
unsigned short new_ucs[TV_MAX_CHARS_PER_BLOCK];
unsigned short *p = new_ucs;
unsigned short ch;
ucschar_t new_ucs[TV_MAX_CHARS_PER_BLOCK];
ucschar_t *p = new_ucs;
ucschar_t ch;
int i;
int k;
int expand_spaces;
@ -262,15 +262,15 @@ static int tv_form_reflow_line(unsigned short *ucs, int chars)
}
}
rb->memcpy(ucs, new_ucs, sizeof(unsigned short) * TV_MAX_CHARS_PER_BLOCK);
rb->memcpy(ucs, new_ucs, sizeof(ucschar_t) * TV_MAX_CHARS_PER_BLOCK);
return indent_chars + nonspace_chars + expand_spaces;
}
static void tv_align_right(int *block_chars)
{
unsigned short *cur_text;
unsigned short *prev_text;
unsigned short ch;
ucschar_t *cur_text;
ucschar_t *prev_text;
ucschar_t ch;
int cur_block = block_count - 1;
int prev_block;
int cur_chars;
@ -335,9 +335,9 @@ static void tv_align_right(int *block_chars)
if (break_pos < prev_chars)
{
rb->memmove(cur_text + prev_chars - break_pos,
cur_text, block_chars[cur_block] * sizeof(unsigned short));
cur_text, block_chars[cur_block] * sizeof(ucschar_t));
rb->memcpy(cur_text, prev_text + break_pos,
(prev_chars - break_pos) * sizeof(unsigned short));
(prev_chars - break_pos) * sizeof(ucschar_t));
block_chars[prev_block] = break_pos;
block_chars[cur_block ] += prev_chars - break_pos;
@ -347,15 +347,15 @@ static void tv_align_right(int *block_chars)
}
}
static int tv_parse_text(const unsigned char *src, unsigned short *ucs,
static int tv_parse_text(const unsigned char *src, ucschar_t *ucs,
int *ucs_chars, bool is_indent)
{
const unsigned char *cur = src;
const unsigned char *next = src;
const unsigned char *line_break_ptr = NULL;
const unsigned char *line_end_ptr = NULL;
unsigned short ch = 0;
unsigned short prev_ch;
ucschar_t ch = 0;
ucschar_t prev_ch;
int chars = 0;
int gw;
int line_break_width = 0;
@ -480,7 +480,7 @@ static int tv_parse_text(const unsigned char *src, unsigned short *ucs,
int tv_create_formed_text(const unsigned char *src, ssize_t bufsize,
int block, bool is_multi, const unsigned char **dst)
{
unsigned short ch;
ucschar_t ch;
int chars[block_count];
int i;
int size = 0;

4
apps/plugins/zxbox/zxbox_keyb.c
View file

@ -326,7 +326,7 @@
struct keyboard_parameters {
const unsigned char* default_kbd;
int DEFAULT_LINES;
unsigned short kbd_buf[KBD_BUF_SIZE];
ucschar_t kbd_buf[KBD_BUF_SIZE];
int nchars;
int font_w;
int font_h;
@ -358,7 +358,7 @@ int zx_kbd_input(char* text/*, int buflen*/)
int editpos, len_utf8;
#endif
/* int statusbar_size = global_settings.statusbar ? STATUSBAR_HEIGHT : 0;*/
unsigned short ch/*, tmp, hlead = 0, hvowel = 0, htail = 0*/;
ucschar_t ch/*, tmp, hlead = 0, hvowel = 0, htail = 0*/;
/*bool hangul = false;*/
unsigned char *utf8;
const unsigned char *p;