len0rd/rockbox
1
0
Fork 1
mirror of https://github.com/Rockbox/rockbox.git synced 2025-11-09 13:12:37 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

unicode: Support characters beyond the first unicode plane

Browse source 
We used 16-bit variables to store the 'character code' everywhere but
this won't let us represent anything beyond U+FFFF.

This patch changes those variables to a custom type that can be 32 or 16
bits depending on the build, and adjusts numerous internal APIs and
datastructures to match.  This includes:

 * utf8decode() and friends
 * font manipulation, caching, rendering, and generation
 * on-screen keyboard
 * FAT filesystem (parsing and generating utf16 LFNs)
 * WIN32 simulator platform code

Note that this patch doesn't _enable_ >16bit unicode support; a followup
patch will turn that on for appropriate targets.

Appears to work on:

  * hosted linux, native, linux simulator in both 16/32-bit modes.

Needs testing on:

  * windows and macos simulator (16bit+32bit)

Change-Id: Iba111b27d2433019b6bff937cf1ebd2c4353a0e8
This commit is contained in:
Solomon Peachy 2024-12-17 08:55:21 -05:00
parent 2a88253426
commit a2c10f6189
44 changed files with 476 additions and 330 deletions
Download patch file Download diff file Expand all files Collapse all files

49
firmware/drivers/fat.c
View file

@ -747,6 +747,8 @@ static bool fatlong_parse_entry(struct fatlong_parse_state *lnparse,
/* so far so good; save entry information */
lnparse->ord = ord;
/* Treat entries as opaque 16-bit values;
utf8decode happens in fatlong_parse_finish() */
uint16_t *ucsp = fatent->ucssegs[ord - 1 + 5];
unsigned int i = longent_char_first();
@ -797,13 +799,24 @@ static bool fatlong_parse_finish(struct fatlong_parse_state *lnparse,
/* ensure the last segment is NULL-terminated if it is filled */
fatent->ucssegs[lnparse->ord_max + 5][0] = 0x0000;
for (uint16_t *ucsp = fatent->ucssegs[5], ucc = *ucsp;
ucc; ucc = *++ucsp)
unsigned long ucc; /* Decoded codepoint */
uint16_t *ucsp, ucs;
for (ucsp = fatent->ucssegs[5], ucs=*ucsp; ucs; ucs = *++ucsp)
{
/* end should be hit before ever seeing padding */
if (ucc == 0xffff)
if (ucs == 0xffff)
return false;
#ifdef UNICODE32
/* Check for a surrogate UTF16 pair */
if (ucs >= 0xd800 && ucs < 0xdc00 &&
*(ucsp+1) >= 0xdc00 && *(ucsp+1) < 0xe000) {
ucc = 0x10000 + (((ucs & 0x3ff) << 10) | (*(ucsp+1) & 0x3ff));
ucsp++;
} else
#endif
ucc = ucs;
if ((p = utf8encode(ucc, p)) - name > FAT_DIRENTRY_NAME_MAX)
return false;
}
@ -1612,12 +1625,27 @@ static int write_longname(struct bpb *fat_bpb, struct fat_filestr *parentstr,
for (unsigned long i = 0; i < ucspadlen; i++)
{
if (i < ucslen)
if (i < ucslen) {
#ifdef UNICODE32
ucschar_t tmp;
name = utf8decode(name, &tmp);
/* For codepoints > U+FFFF we will need to use a UTF16 surrogate
pair. 'ucslen' already takes this into account! */
if (tmp < 0x10000) {
ucsname[i] = tmp;
} else {
tmp -= 0x10000;
ucsname[i++] = 0xd800 | ((tmp >> 10) & 0x3ff); /* High */
ucsname[i] = 0xdc00 | (tmp & 0x3ff); /* Low */
}
#else
name = utf8decode(name, &ucsname[i]);
else if (i == ucslen)
#endif
} else if (i == ucslen) {
ucsname[i] = 0x0000; /* name doesn't fill last block */
else /* i > ucslen */
} else /* i > ucslen */ {
ucsname[i] = 0xffff; /* pad-out to end */
}
}
dc_lock_cache();
@ -1744,9 +1772,12 @@ static int add_dir_entry(struct bpb *fat_bpb, struct fat_filestr *parentstr,
create_dos_name(basisname, name, &n);
randomize_dos_name(shortname, basisname, &n);
/* one dir entry needed for every 13 characters of filename,
plus one entry for the short name */
ucslen = utf8length(name);
/* one dir entry needed for every 13 utf16 "code units"
of filename, plus one entry for the short name.
Keep in mind that a unicode character can take up to
two code units!
*/
ucslen = utf16len_utf8(name);
if (ucslen > 255)
FAT_ERROR(-2); /* name is too long */

10
firmware/drivers/lcd-bitmap-common.c
View file

@ -385,7 +385,7 @@ static void LCDFN(mono_bmp_part_helper)(const unsigned char *src, int src_x,
/* put a string at a given pixel position, skipping first ofs pixel columns */
static void LCDFN(putsxyofs)(int x, int y, int ofs, const unsigned char *str)
{
unsigned short *ucs;
ucschar_t *ucs;
struct viewport *vp = LCDFN(current_viewport);
font_lock(vp->font, true);
struct font* pf = font_get(vp->font);
@ -429,7 +429,7 @@ static void LCDFN(putsxyofs)(int x, int y, int ofs, const unsigned char *str)
bool is_rtl, is_diac;
const unsigned char *bits;
int width, base_width, base_ofs = 0;
const unsigned short next_ch = ucs[1];
const ucschar_t next_ch = ucs[1];
if (x >= vp->width)
break;
@ -447,7 +447,7 @@ static void LCDFN(putsxyofs)(int x, int y, int ofs, const unsigned char *str)
{
if (!rtl_next_non_diac_width)
{
const unsigned short *u;
const ucschar_t *u;
/* Jump to next non-diacritic char, and calc its width */
for (u = &ucs[1]; *u && IS_DIACRITIC(*u); u++);
@ -529,7 +529,7 @@ static void LCDFN(putsxyofs)(int x, int y, int ofs, const unsigned char *str)
/* put a string at a given pixel position, skipping first ofs pixel columns */
static void LCDFN(putsxyofs)(int x, int y, int ofs, const unsigned char *str)
{
unsigned short *ucs;
ucschar_t *ucs;
struct viewport *vp = LCDFN(current_viewport);
struct font* pf = font_get(vp->font);
const unsigned char *bits;
@ -567,7 +567,7 @@ static void LCDFN(putsxyofs)(int x, int y, int ofs, const unsigned char *str)
/* allow utf but no diacritics or rtl lang */
for (ucs = bidi_l2v(str, 1); *ucs; ucs++)
{
const unsigned short next_ch = ucs[1];
const ucschar_t next_ch = ucs[1];
if (x >= vp->width)
break;