keyboard: point-mode virtual keyboard for touchscreens

On large touchscreens the software keyboard can now run in "point mode". Keys are laid out finger-sized and tapped directly instead of being navigated with a cursor. Targets opt in with HAVE_KBD_POINT_MODE.

Key size is derived from LCD_DPI so a key is about 5mm across whatever the panel density, falling back to a fixed size where DPI is unknown.

A phone-style default layout in the UI font, across three flip pages of eight lines. Page two is page one shifted (i.e caps). Page three completes Latin-1 and adds additional accented letters common to the other European locales.

Space is drawn as an icon, since the UI font has no glyph for it.

Change-Id: Ibe2e305b3c22b1f9152358cf3864b9445b67b463
Co-Author: Claude Opus 4.8

apps/features.txt

@@ -82,6 +82,10 @@ lcd_sleep
 morse_input
 #endif
 
+#if defined(HAVE_KBD_POINT_MODE)
+kbd_point_mode
+#endif
+
 #if defined(HAVE_PITCHCONTROL)
 pitchscreen
 #endif

apps/recorder/keyboard.c

@@ -47,8 +47,23 @@
 #define DEFAULT_MARGIN 6
 #define KBD_BUF_SIZE 500
 
+/* Sentinel picker cell, drawn as an icon (the UI font has no glyph for it):
+ * KBD_SPACE inserts a space. */
+#define KBD_SPACE 0x2423
+
 #ifdef HAVE_TOUCHSCREEN
+/* Increase grid size so it responds better to taps */
+#ifdef HAVE_KBD_POINT_MODE
+#ifndef MIN_GRID_SIZE
+#if defined(LCD_DPI) && (LCD_DPI > 0)
+#define MIN_GRID_SIZE   ((LCD_DPI * 50 + 127) / 254)
+#else
+#define MIN_GRID_SIZE   44
+#endif
+#endif
+#else
 #define MIN_GRID_SIZE   16
+#endif
 #define GRID_SIZE(s, x)   \
         ((s) == SCREEN_MAIN && MIN_GRID_SIZE > (x) ? MIN_GRID_SIZE: (x))
 #endif
@@ -389,6 +404,29 @@ static ucschar_t get_kbd_ch(struct keyboard_parameters *pm, int x, int y)
     k = k * pm->max_chars + x;
     return (*pbuf != 0xFEFF && k < *pbuf)? pbuf[k+1]: ' ';
 }
+
+#ifdef HAVE_TOUCHSCREEN
+/* Number of real characters in the grid row drawn at picker row y. */
+static int get_kbd_row_len(struct keyboard_parameters *pm, int y)
+{
+    unsigned int n, i, k = pm->page*pm->lines + y;
+    ucschar_t *pbuf;
+    for (pbuf = pm->kbd_buf_ptr; (i = *pbuf) != 0xFEFF; pbuf += i + 1)
+    {
+        n = i ? (i + pm->max_chars - 1) / pm->max_chars : 1;
+        if (k < n)
+        {
+            int len = (int)i - (int)(k * pm->max_chars);
+            if (len > (int)pm->max_chars)
+                len = pm->max_chars;
+            return len < 0 ? 0 : len;
+        }
+        k -= n;
+    }
+    return 0;
+}
+#endif /* HAVE_TOUCHSCREEN */
+
 static void kbd_calc_pm_params(struct keyboard_parameters *pm,
                             struct screen *sc, struct edit_state *state);
 static void kbd_calc_vp_params(struct keyboard_parameters *pm,
@@ -430,10 +468,14 @@ int kbd_input(char* text, int buflen, ucschar_t *kbd)
     }
 
 #ifdef HAVE_TOUCHSCREEN
+    enum touchscreen_mode old_mode = touchscreen_get_mode();
+#ifdef HAVE_KBD_POINT_MODE
+    touchscreen_set_mode(TOUCHSCREEN_POINT);
+#else
     /* keyboard is unusuable in pointing mode so force 3x3 for now.
      * TODO - fix properly by using a bigger font and changing the layout */
-    enum touchscreen_mode old_mode = touchscreen_get_mode();
     touchscreen_set_mode(TOUCHSCREEN_BUTTON);
+#endif
 #endif
     /* initialize state */
     state.text = text;
@@ -458,6 +500,48 @@ int kbd_input(char* text, int buflen, ucschar_t *kbd)
             const unsigned char *p;
             int len = 0;
 
+#ifdef HAVE_KBD_POINT_MODE
+            if (touchscreen_get_mode() == TOUCHSCREEN_POINT)
+            {
+#if (LCD_WIDTH >= 480) && (LCD_HEIGHT >= 640)
+            /* Three pages of eight lines. Page two is page one shifted, cell
+             * for cell; page three completes Latin-1 and adds the accented
+             * letters common to the other European locales. The space bar
+             * occupies the same cells on every page. Ten columns need the
+             * width, and all eight lines have to be visible at once. */
+                p = "1234567890\n"
+                    "qwertyuiop\n"
+                    "asdfghjkl;\n"
+                    "zxcvbnm,./\n"
+                    "àáâäåçèéêë\n"
+                    "ìíîïñòóôöù\n"
+                    "úûüÿ'æøðþý\n"
+                    "`-=␣␣␣␣[]\\\n"
+                    "!@#$%^&*()\n"
+                    "QWERTYUIOP\n"
+                    "ASDFGHJKL:\n"
+                    "ZXCVBNM<>?\n"
+                    "ÀÁÂÄÅÇÈÉÊË\n"
+                    "ÌÍÎÏÑÒÓÔÖÙ\n"
+                    "ÚÛÜŸ\"ÆØÐÞÝ\n"
+                    "~_+␣␣␣␣{}|\n"
+                    "¡¢£¤¥¦§¨©ª\n"
+                    "«¬®¯°±²³´µ\n"
+                    "¶·¸¹º»¼½¾¿\n"
+                    "×÷ßÃãÕõ€Œœ\n"
+                    "ĐđČčĎďĚěŇň\n"
+                    "ŘřŠšŤťŮůŽž\n"
+                    "ĄąĆćĘꣳŃń\n"
+                    "ŚśŹ␣␣␣␣źŻż";
+                pm->default_lines = 8;
+                pm->max_line_len = 10;
+#else
+# error "HAVE_KBD_POINT_MODE: no point-mode layout for this screen size"
+#endif
+            }
+            else
+#endif
+            {
 #if LCD_WIDTH >= 160 && LCD_HEIGHT >= 96
             struct screen *sc = &screens[l];
 
@@ -528,6 +612,7 @@ int kbd_input(char* text, int buflen, ucschar_t *kbd)
                 pm->max_line_len = 18;
             }
 #endif
+            }
             pbuf = pm->kbd_buf;
             while (*p)
             {
@@ -849,6 +934,11 @@ static void kbd_calc_pm_params(struct keyboard_parameters *pm,
                                 (touchscreen_get_mode() == TOUCHSCREEN_POINT));
 #endif
 
+#ifdef HAVE_TOUCHSCREEN
+    if (touchscreen_get_mode() == TOUCHSCREEN_POINT)
+        pm->curfont = sc->getuifont();
+    else
+#endif
     pm->curfont = pm->default_lines ? FONT_SYSFIXED : sc->getuifont();
     font = font_get(pm->curfont);
     pm->font_h = font->height;
@@ -907,6 +997,11 @@ static void kbd_calc_vp_params(struct keyboard_parameters *pm,
     icon_w = get_icon_width(sc->screen_type);
 
     sc_w = vp->width; /**sc->getwidth();**/
+#ifdef HAVE_TOUCHSCREEN
+    if (touchscreen_get_mode() == TOUCHSCREEN_POINT)
+        pm->font_w = sc_w / pm->max_line_len;   /* fill width, allow >10 cols */
+    else
+#endif
     if (pm->font_w < sc_w / pm->max_line_len)
         pm->font_w = sc_w / pm->max_line_len;
     pm->max_chars = sc_w / pm->font_w;
@@ -1075,22 +1170,47 @@ static void kbd_draw_picker(struct keyboard_parameters *pm,
         {
             for (i = 0; i < pm->max_chars; i++)
             {
+                int cx = i*pm->font_w, cy = j*pm->font_h;
                 ch = get_kbd_ch(pm, i, j);
+                if (ch == KBD_SPACE)
+                {
+                    /* space bar: a bar with short upturned ends. Cells join;
+                     * the run's first/last cell adds the vertical end tick. */
+                    int th = MAX(2, pm->font_h/14);
+                    int by = cy + (pm->font_h*2)/3;
+                    int tk = pm->font_h/6;
+                    int rx = cx + pm->font_w - th;
+                    sc->fillrect(cx, by, pm->font_w, th);
+                    if (i == 0 || get_kbd_ch(pm, i-1, j) != KBD_SPACE)
+                        sc->fillrect(cx, by - tk, th, tk + th);
+                    if (get_kbd_ch(pm, i+1, j) != KBD_SPACE)
+                        sc->fillrect(rx, by - tk, th, tk + th);
+                    continue;
+                }
                 utf8 = utf8encode(ch, outline);
                 *utf8 = 0;
 
                 sc->getstringsize(outline, &w, &h);
-                sc->putsxy(i*pm->font_w + (pm->font_w-w) / 2,
-                           j*pm->font_h + (pm->font_h-h) / 2, outline);
+                sc->putsxy(cx + (pm->font_w-w) / 2,
+                           cy + (pm->font_h-h) / 2, outline);
             }
         }
 
         if (!pm->line_edit)
         {
-            /* highlight the key that has focus */
+            /* highlight the focused key; the space bar spans several cells and
+             * highlights as one */
+            int hx = pm->x, hw = 1;
+            if (get_kbd_ch(pm, hx, pm->y) == KBD_SPACE)
+            {
+                while (hx > 0 && get_kbd_ch(pm, hx - 1, pm->y) == KBD_SPACE)
+                    hx--;
+                while (get_kbd_ch(pm, hx + hw, pm->y) == KBD_SPACE)
+                    hw++;
+            }
             sc->set_drawmode(DRMODE_COMPLEMENT);
-            sc->fillrect(pm->font_w*pm->x, pm->font_h*pm->y,
-                         pm->font_w, pm->font_h);
+            sc->fillrect(pm->font_w*hx, pm->font_h*pm->y,
+                         pm->font_w*hw, pm->font_h);
             sc->set_drawmode(DRMODE_SOLID);
         }
     }
@@ -1337,8 +1457,8 @@ static void kbd_draw_buttons(struct keyboard_parameters *pm, struct screen *sc)
     /* draw buttons */
     if (pm->pages > 1)
     {
-        /* button to flip page. */
-        vp.y = pm->lines*pm->font_h;
+        /* button to flip page */
+        vp.y = pm->kbd_viewports[eKBD_VP_PICKER].y + pm->lines*pm->font_h;
         sc->hline(0, sc_w - 1, 0);
         sc->putsxy(0, text_y, ">");
     }
@@ -1378,22 +1498,32 @@ static int keyboard_touchscreen(struct keyboard_parameters *pm,
 #endif
     if (x < 0 || y < 0)
         return ACTION_NONE;
-    if (y < pm->lines*pm->font_h)
+    /* The picker grid is drawn in its own viewport, below the edit line, so
+     * translate the absolute touch into that viewport (see kbd_draw_picker). */
+    struct viewport *picker_vp = &pm->kbd_viewports[eKBD_VP_PICKER];
+    int picker_x = x - picker_vp->x;
+    int picker_y = y - picker_vp->y;
+    if (picker_y >= 0 && picker_y < pm->lines*pm->font_h)
     {
-        if (x/pm->font_w < pm->max_chars)
+        int row = picker_y/pm->font_h;
+        /* ignore taps past the end of a short row */
+        if (picker_x >= 0 && picker_x/pm->font_w < get_kbd_row_len(pm, row))
         {
             /* picker area */
             state->changed = CHANGED_PICKER;
-            pm->x = x/pm->font_w;
-            pm->y = y/pm->font_h;
+            pm->x = picker_x/pm->font_w;
+            pm->y = row;
             pm->line_edit = false;
             if (button == BUTTON_REL)
                 return ACTION_KBD_SELECT;
         }
     }
-    else if (y < pm->main_y - pm->keyboard_margin)
+    else if (pm->pages > 1 && picker_y >= pm->lines*pm->font_h &&
+             picker_y < pm->lines*pm->font_h +
+                        GRID_SIZE(sc->screen_type, sc->getcharheight()))
     {
-        /* button to flip page */
+        /* flip-page button, drawn just below the grid; test the same band in
+         * picker-relative coords so ">" is tappable. */
         if (button == BUTTON_REL)
             return ACTION_KBD_PAGE_FLIP;
     }
@@ -1432,6 +1562,9 @@ static void kbd_insert_selected(struct keyboard_parameters *pm,
     /* find input char */
     ucschar_t ch = get_kbd_ch(pm, pm->x, pm->y);
 
+    if (ch == KBD_SPACE)
+        ch = ' ';
+
     /* check for hangul input */
     if (ch >= 0x3131 && ch <= 0x3163)
     {
@@ -1552,6 +1685,18 @@ static void kbd_move_picker_horizontal(struct keyboard_parameters *pm,
     state->changed = CHANGED_PICKER;
 
     pm->x += dir;
+    /* The space bar spans several cells but highlights and inserts as one key,
+     * so step over the whole run instead of stopping inside it, which looks
+     * like the cursor is stuck. */
+    if (pm->x >= 0 && pm->x < pm->max_chars &&
+        get_kbd_ch(pm, pm->x, pm->y) == KBD_SPACE &&
+        get_kbd_ch(pm, pm->x - dir, pm->y) == KBD_SPACE)
+    {
+        while (pm->x >= 0 && pm->x < pm->max_chars &&
+               get_kbd_ch(pm, pm->x, pm->y) == KBD_SPACE)
+            pm->x += dir;
+    }
+
     if (pm->x < 0)
     {
         if (!global_settings.list_wraparound && pm->page == 0)

firmware/export/config/hibyr1.h

@@ -37,6 +37,8 @@
 #ifndef BOOTLOADER
 #define HAVE_BUTTON_DATA
 #define HAVE_TOUCHSCREEN
+/* Finger-sized keys, tapped directly, instead of a navigated cursor */
+#define HAVE_KBD_POINT_MODE
 #endif
 
 #ifndef CONFIG_BACKLIGHT_FADING

firmware/export/config/hibyr3proii.h

@@ -37,6 +37,8 @@
 #ifndef BOOTLOADER
 #define HAVE_BUTTON_DATA
 #define HAVE_TOUCHSCREEN
+/* Finger-sized keys, tapped directly, instead of a navigated cursor */
+#define HAVE_KBD_POINT_MODE
 #endif
 
 #ifndef CONFIG_BACKLIGHT_FADING

manual/rockbox_interface/browsing_and_playing.tex

@@ -211,6 +211,16 @@ The virtual keyboard can be easily changed by making a text file
 with the required layout. More information on how to achieve this can be found
 on the Rockbox website at \wikilink{LoadableKeyboardLayouts}.
 
+\opt{kbd_point_mode}{
+  On this player the keyboard is laid out for direct touch where the keys are sized
+  for a fingertip and are tapped individually, rather than being selected
+  with the 3\,\texttimes\,3 grid used on other touchscreen targets. By default the
+  characters are spread over three pages with lower case and accented
+  letters, their upper-case forms in the same positions, and a third page
+  of punctuation and additional European characters. Tap
+  \textbf{\textgreater} below the keys to change page.
+}
+
 \opt{morse_input}{
   Also you can switch to Morse code input mode by changing the
   \setting{Use Morse Code Input} setting%