From 26fd39d9e094bb2b3f8a580828bfdf1a0e371b2e Mon Sep 17 00:00:00 2001 From: Charlie Gordon Date: Thu, 16 Mar 2017 09:16:31 +0000 Subject: [PATCH] unicode: improve support for wide and combining code points - qe_isaccent() recognises all combining code points (to be refined) - add computer generated width table unicode_width.h - display difference in hex in compare-windows to distinguish space and nbsp, normal and combined form for accented letters. - display multiple combining code points in what-cursor-position (C-x =) - attach combining code points to preceeding space in display_char_bidir - handle accents specificaaly in flush_fragment - handle combining code-points in screen backstore via indirection table this transient table can be inspected via describe-screen --- Makefile | 3 + charset.c | 32 ++- extras.c | 5 +- qe.c | 125 +++++++---- qe.h | 9 +- tty.c | 189 ++++++++++++++--- unicode_width.h | 550 ++++++++++++++++++++++++++++++++++++++++++++++++ unihex.c | 8 +- 8 files changed, 824 insertions(+), 97 deletions(-) create mode 100644 unicode_width.h diff --git a/Makefile b/Makefile index 5633158..297e0cf 100644 --- a/Makefile +++ b/Makefile @@ -302,6 +302,7 @@ $(TOBJS_DIR)/modules.txt: $(TSRCS) Makefile sed s/qe_module_init/qe_module_declare/ >> $@ $(OBJS_DIR)/cfb.o: cfb.c cfb.h fbfrender.h +$(OBJS_DIR)/charset.o: charset.c unicode_width.h $(OBJS_DIR)/charsetjis.o: charsetjis.c charsetjis.def $(OBJS_DIR)/fbfrender.o: fbfrender.c fbfrender.h libfbf.h $(OBJS_DIR)/qe.o: qe.c parser.c qeconfig.h qfribidi.h variables.h @@ -309,6 +310,7 @@ $(OBJS_DIR)/qfribidi.o: qfribidi.c qfribidi.h $(OBJS_DIR)/clang.o: clang.c rust.c swift.c icon.c groovy.c virgil.c $(XOBJS_DIR)/cfb.o: cfb.c cfb.h fbfrender.h +$(XOBJS_DIR)/charset.o: charset.c unicode_width.h $(XOBJS_DIR)/charsetjis.o: charsetjis.c charsetjis.def $(XOBJS_DIR)/fbfrender.o: fbfrender.c fbfrender.h libfbf.h $(XOBJS_DIR)/qe.o: qe.c parser.c qeconfig.h qfribidi.h variables.h @@ -316,6 +318,7 @@ $(XOBJS_DIR)/qfribidi.o: qfribidi.c qfribidi.h $(XOBJS_DIR)/clang.o: clang.c rust.c swift.c icon.c groovy.c virgil.c $(TOBJS_DIR)/cfb.o: cfb.c cfb.h fbfrender.h +$(TOBJS_DIR)/charset.o: charset.c unicode_width.h $(TOBJS_DIR)/charsetjis.o: charsetjis.c charsetjis.def $(TOBJS_DIR)/fbfrender.o: fbfrender.c fbfrender.h libfbf.h $(TOBJS_DIR)/qe.o: qe.c parser.c qeconfig.h qfribidi.h variables.h diff --git a/charset.c b/charset.c index 1bb765a..6cb97c5 100644 --- a/charset.c +++ b/charset.c @@ -27,28 +27,12 @@ struct QECharset *first_charset; /* Unicode utilities */ /* Compute tty width of unicode characters. This is a modified - * implementation of wcwidth() from Markus Kuhn. We do not handle non - * spacing and enclosing combining characters and control chars. - */ - -/* XXX: This table is incomplete, should compute from UnicodeData.txt - * via a specialized utility + * implementation of wcwidth() from Markus Kuhn. We handle most + * non spacing and enclosing combining characters. */ static unsigned int const unicode_glyph_ranges[] = { - 0x10FF, 1, 0x115f, 2, /* 0: Hangul Jamo */ - 0x2328, 1, 0x232a, 2, /* 2: wide Angle brackets */ - 0x2E7F, 1, 0x2efd, 2, /* 4: CJK Radicals */ - 0x2EFF, 1, 0x303e, 2, /* 6: Kangxi Radicals */ - 0x303F, 1, 0x4dbf, 2, /* 8: CJK */ - 0x4DFF, 1, 0xa4cf, 2, /* 10: CJK */ - 0xABFF, 1, 0xd7a3, 2, /* 12: Hangul Syllables */ - 0xF8FF, 1, 0xfaff, 2, /* 14: CJK Compatibility Ideographs */ - 0xFDFF, 1, 0xFE1F, 2, /* 16: */ - 0xFE2F, 1, 0xfe6f, 2, /* 18: CJK Compatibility Forms */ - 0xFEFF, 1, 0xff5f, 2, /* 20: Fullwidth Forms */ - 0xFFDF, 1, 0xffe6, 2, /* 22: */ - 0x1FFFF, 1, 0x3fffd, 2, /* 24: CJK Compatibility */ - UINT_MAX, 1, /* 26: catchall */ +/* width table is produced by unitable.c */ +#include "unicode_width.h" }; static const unsigned int *unicode_glyph_range_index[0x20]; @@ -344,6 +328,14 @@ int utf8_encode(char *q0, int c) return q - q0; } +char *utf8_char_to_string(char *buf, int c) { + char *p = buf; + if (qe_isaccent(c)) + *p++ = ' '; + p[utf8_encode(p, c)] = '\0'; + return buf; +} + int utf8_to_unicode(unsigned int *dest, int dest_length, const char *str) { const char *p; diff --git a/extras.c b/extras.c index fe2cc41..138e70e 100644 --- a/extras.c +++ b/extras.c @@ -31,6 +31,7 @@ void do_compare_windows(EditState *s, int argval) EditState *s2; int offset1, offset2, size1, size2, ch1, ch2; int tries, resync = 0; + char buf1[MAX_CHAR_BYTES + 2], buf2[MAX_CHAR_BYTES + 2]; s1 = s; /* Should use same internal function as for next_window */ @@ -119,7 +120,9 @@ void do_compare_windows(EditState *s, int argval) s1->offset - save1, s2->offset - save2); break; } - put_status(s, "Difference: %c <-> %c", ch1, ch2); + put_status(s, "Difference: '%s' [0x%02X] <-> '%s' [0x%02X]", + utf8_char_to_string(buf1, ch1), ch1, + utf8_char_to_string(buf2, ch2), ch2); break; } if (ch1 != EOF) { diff --git a/qe.c b/qe.c index c7915b8..6e00bd8 100644 --- a/qe.c +++ b/qe.c @@ -2490,24 +2490,28 @@ void do_count_lines(EditState *s) void do_what_cursor_position(EditState *s) { char buf[256]; + int accents[6]; buf_t outbuf, *out; int line_num, col_num; - int c, c2, cc, offset1, offset2, off; + int offset1, off; + int c, cc; + int i, n; out = buf_init(&outbuf, buf, sizeof(buf)); if (s->offset < s->b->total_size) { c = eb_nextc(s->b, s->offset, &offset1); - c2 = eb_nextc(s->b, offset1, &offset2); - if (c == '\n' || !qe_isaccent(c2)) { - c2 = 0; - offset2 = offset1; + n = 0; + if (c != '\n' && !s->unihex_mode) { + while (n < countof(accents) + && qe_isaccent(cc = eb_nextc(s->b, offset1, &off))) { + accents[n++] = cc; + offset1 = off; + } } if (s->b->eol_type == EOL_MAC) { - if (c == '\r') - c = '\n'; - else - if (c == '\n') - c = '\r'; + /* CR and LF are swapped in old style Mac buffers */ + if (c == '\r' || c == '\n') + c ^= '\r' ^ '\n'; } buf_puts(out, "char:"); if (c < 32 || c == 127) { @@ -2516,30 +2520,37 @@ void do_what_cursor_position(EditState *s) if (c < 127 || c >= 160) { buf_put_byte(out, ' '); buf_put_byte(out, '\''); - if (c == '\\' || c == '\'') + if (c == '\\' || c == '\'') { buf_put_byte(out, '\\'); + } + if (qe_isaccent(c)) { + buf_putc_utf8(out, ' '); + } buf_putc_utf8(out, c); - if (c2) - buf_putc_utf8(out, c2); + for (i = 0; i < n; i++) { + buf_put_byte(out, ' '); + buf_putc_utf8(out, accents[i]); + } buf_put_byte(out, '\''); } - if (c < 0x100 && !c2) - buf_printf(out, " \\%03o", c); - buf_printf(out, " %d", c); - if (c2) - buf_printf(out, "/%d", c2); + if (n == 0) { + if (c < 0x100) { + buf_printf(out, " \\%03o", c); + } + buf_printf(out, " %d", c); + } buf_printf(out, " 0x%02x", c); - if (c2) - buf_printf(out, "/0x%02x", c2); - + for (i = 0; i < n; i++) { + buf_printf(out, "/0x%02x", accents[i]); + } /* Display buffer bytes if char is encoded */ - off = s->offset; - cc = eb_read_one_byte(s->b, off++); - if (cc != c || c2 || off != offset2) { - buf_printf(out, " [%02X", cc); - while (off < offset2) { - cc = eb_read_one_byte(s->b, off++); - buf_printf(out, " %02X", cc); + if (offset1 != s->offset + 1) { + int sep = '['; + buf_put_byte(out, ' '); + for (off = s->offset; off < offset1; off++) { + cc = eb_read_one_byte(s->b, off); + buf_printf(out, "%c%02X", sep, cc); + sep = ' '; } buf_put_byte(out, ']'); } @@ -3512,7 +3523,7 @@ static void flush_fragment(DisplayState *s) int display_char_bidir(DisplayState *s, int offset1, int offset2, int embedding_level, int ch) { - int space, style, istab; + int space, style, istab, isaccent; EditState *e; style = s->style; @@ -3539,18 +3550,43 @@ int display_char_bidir(DisplayState *s, int offset1, int offset2, space = (ch == ' '); istab = (ch == '\t'); + isaccent = qe_isaccent(ch); /* a fragment is a part of word where style/embedding_level do not change. For TAB, only one fragment containing it is sent */ - if ((s->fragment_index >= MAX_WORD_SIZE) || - istab || - (s->fragment_index >= 1 && - (space != s->last_space || - style != s->last_style || - embedding_level != s->last_embedding_level))) { - /* flush the current fragment if needed */ - flush_fragment(s); + if (s->fragment_index >= 1) { + if (s->fragment_index >= MAX_WORD_SIZE || + istab || + space != s->last_space || + style != s->last_style || + embedding_level != s->last_embedding_level) { + /* flush the current fragment if needed */ + if (isaccent && s->fragment_chars[s->fragment_index - 1] == ' ') { + /* separate last space to make it part of the next word */ + int off1, off2, cur_hex; + --s->fragment_index; + off1 = s->fragment_offsets[s->fragment_index][0]; + off2 = s->fragment_offsets[s->fragment_index][1]; + cur_hex = s->fragment_hex_mode[s->fragment_index]; + flush_fragment(s); + s->fragment_chars[s->fragment_index] = ' '; + s->fragment_offsets[s->fragment_index][0] = off1; + s->fragment_offsets[s->fragment_index][1] = off2; + s->fragment_hex_mode[s->fragment_index] = cur_hex; + s->fragment_index++; + } else { + flush_fragment(s); + } + } } + if (isaccent && s->fragment_index == 0) { + /* prepend a space if fragment starts with an accent */ + s->fragment_chars[s->fragment_index] = ' '; + s->fragment_offsets[s->fragment_index][0] = offset1; + s->fragment_offsets[s->fragment_index][1] = offset2; + s->fragment_hex_mode[s->fragment_index] = s->cur_hex_mode; + s->fragment_index++; + } /* store the char and its embedding level */ s->fragment_chars[s->fragment_index] = ch; s->fragment_offsets[s->fragment_index][0] = offset1; @@ -4075,13 +4111,16 @@ int text_display_line(EditState *s, DisplayState *ds, int offset) c = '\n'; display_printf(ds, offset0, offset, "^%c", ('@' + c) & 127); } else - if (c > MAX_UNICODE_DISPLAY) { + if (c >= 128 + && (s->qe_state->show_unicode == 1 || + c == 0xfeff || /* Display BOM as \uFEFF to make it explicit */ + c > MAX_UNICODE_DISPLAY)) { /* display unsupported unicode code points as hex */ - display_printf(ds, offset0, offset, "\\U%08x", c); - } else - if (c >= 256 && (s->qe_state->show_unicode == 1 || c == 0xfeff)) { - /* Display BOM as \uFEFF to make it explicit */ - display_printf(ds, offset0, offset, "\\u%04x", c); + if (c > 0xffff) { + display_printf(ds, offset0, offset, "\\U%08x", c); + } else { + display_printf(ds, offset0, offset, "\\u%04x", c); + } } else { display_char_bidir(ds, offset0, offset, embedding_level, c); } diff --git a/qe.h b/qe.h index ef33b59..bf5821f 100644 --- a/qe.h +++ b/qe.h @@ -295,10 +295,6 @@ static inline int qe_isword(int c) { /* XXX: any unicode char >= 128 is considered as word. */ return qe_isalnum_(c) || (c >= 128); } -static inline int qe_isaccent(int c) { - /* XXX: only check Latin combining marks */ - return qe_inrange(c, 0x300, 0x362); -} static inline int qe_toupper(int c) { return (qe_inrange(c, 'a', 'z') ? c + 'A' - 'a' : c); } @@ -617,6 +613,7 @@ extern unsigned char const utf8_length[256]; static inline int utf8_is_trailing_byte(int c) { return (c & 0xC0) == 0x80; } int utf8_encode(char *q, int c); int utf8_decode(const char **pp); +char *utf8_char_to_string(char *buf, int c); int utf8_to_unicode(unsigned int *dest, int dest_length, const char *str); void charset_completion(CompleteState *cp); @@ -638,6 +635,10 @@ u8 *encode_8bit(QECharset *charset, u8 *q, int c); int unicode_tty_glyph_width(unsigned int ucs); +static inline int qe_isaccent(int c) { + return c >= 0x300 && unicode_tty_glyph_width(c) == 0; +} + /* arabic.c */ int arab_join(unsigned int *line, unsigned int *ctog, int len); diff --git a/tty.c b/tty.c index 9898ea3..7d335ef 100644 --- a/tty.c +++ b/tty.c @@ -42,7 +42,10 @@ typedef uint64_t TTYChar; #define TTYCHAR_GETFG(cc) ((uint32_t)((cc) >> 32) & 0xFF) #define TTYCHAR_GETBG(cc) ((uint32_t)((cc) >> (32 + 8)) & 0xFF) #define TTYCHAR_DEFAULT TTYCHAR(' ', 7, 0) +#define TTYCHAR_COMB 0x200000 +#define TTYCHAR_BAD 0xFFFD #define TTYCHAR_NONE 0xFFFFFFFF +#define COMB_CACHE_SIZE 2048 #else typedef unsigned int TTYChar; #define TTYCHAR(ch,fg,bg) ((ch) | ((fg) << 16) | ((bg) << 24)) @@ -52,7 +55,9 @@ typedef unsigned int TTYChar; #define TTYCHAR_GETFG(cc) (((cc) >> 16) & 0xFF) #define TTYCHAR_GETBG(cc) (((cc) >> 24) & 0xFF) #define TTYCHAR_DEFAULT TTYCHAR(' ', 7, 0) +#define TTYCHAR_BAD 0xFFFD #define TTYCHAR_NONE 0xFFFF +#define COMB_CACHE_SIZE 1 #endif #if defined(CONFIG_UNLOCKIO) @@ -104,6 +109,7 @@ typedef struct TTYState { #define USE_BOLD_AS_BRIGHT 2 #define USE_BLINK_AS_BRIGHT 4 #define USE_ERASE_END_OF_LINE 8 + unsigned int comb_cache[COMB_CACHE_SIZE]; } TTYState; static void tty_resize(int sig); @@ -1027,10 +1033,8 @@ static void tty_term_close_font(qe__unused__ QEditScreen *s, QEFont **fontp) static inline int tty_term_glyph_width(qe__unused__ QEditScreen *s, unsigned int ucs) { - /* XXX: should support combining marks */ - /* fast test for majority of non-wide scripts */ - if (ucs < 0x1100) + if (ucs < 0x300) return 1; return unicode_tty_glyph_width(ucs); @@ -1051,14 +1055,102 @@ static void tty_term_text_metrics(QEditScreen *s, qe__unused__ QEFont *font, metrics->width = x; } +#if MAX_UNICODE_DISPLAY > 0xFFFF + +static unsigned int comb_cache_add(TTYState *ts, const unsigned int *seq, int len) { + unsigned int *ip; + for (ip = ts->comb_cache; *ip; ip += *ip & 0xFFFF) { + if (*ip == len + 1U && !memcmp(ip + 1, seq, len * sizeof(*ip))) { + return TTYCHAR_COMB + (ip - ts->comb_cache); + } + } + for (ip = ts->comb_cache; *ip; ip += *ip & 0xFFFF) { + if (*ip >= 0x10001U + len) { + /* found free slot */ + if (*ip > 0x10001U + len) { + /* split free block */ + ip[len + 1] = *ip - (len + 1); + } + break; + } + } + if (*ip == 0) { + if ((ip - ts->comb_cache) + len + 1 >= countof(ts->comb_cache)) { + return TTYCHAR_BAD; + } + ip[len + 1] = 0; + } + *ip = len + 1; + memcpy(ip + 1, seq, len * sizeof(*ip)); + return TTYCHAR_COMB + (ip - ts->comb_cache); +} + +static void comb_cache_clean(TTYState *ts, const TTYChar *screen, int len) { + unsigned int *ip; + + if (ts->comb_cache[0] == 0) + return; + + for (ip = ts->comb_cache; *ip != 0; ip += *ip & 0xFFFF) { + *ip |= 0x10000; + } + ip = ts->comb_cache; + for (int i = 0; i < len; i++) { + int ch = TTYCHAR_GETCH(screen[i]); + if (ch >= TTYCHAR_COMB && ch < TTYCHAR_COMB + countof(ts->comb_cache) - 1) { + ip[ch - TTYCHAR_COMB] &= ~0x10000; + } + } + for (; *ip != 0; ip += *ip & 0xFFFF) { + if (*ip & 0x10000) { + while (ip[*ip & 0xFFFF] & 0x10000) { + /* coalesce subsequent free blocks */ + *ip += ip[*ip & 0xFFFF] & 0xFFFF; + } + if (ip[*ip & 0xFFFF] == 0) { + /* truncate free list */ + *ip = 0; + break; + } + } + } +} + +static void comb_cache_describe(QEditScreen *s, EditBuffer *b) { + TTYState *ts = s->priv_data; + unsigned int *ip; + + eb_printf(b, "Unicode combination cache:\n\n"); + + for (ip = ts->comb_cache; *ip != 0; ip += *ip & 0xFFFF) { + if (*ip & 0x10000) { + eb_printf(b, " FREE %d\n", (*ip & 0xFFFF) - 1); + } else { + eb_printf(b, " %06X %d:", + (unsigned int)(TTYCHAR_COMB + (ip - ts->comb_cache)), + (*ip & 0xFFFF) - 1); + for (unsigned int i = 1; i < (*ip & 0xFFFF); i++) { + eb_printf(b, " %04X", ip[i]); + } + eb_printf(b, "\n"); + } + } +} +#else +#define comb_cache_add(s, p, n) TTYCHAR_BAD +#define comb_cache_clean(s, p, n) +#define comb_cache_describe(s, b) +#endif + static void tty_term_draw_text(QEditScreen *s, qe__unused__ QEFont *font, - int x, int y, const unsigned int *str, int len, + int x, int y, const unsigned int *str0, int len, QEColor color) { TTYState *ts = s->priv_data; TTYChar *ptr; int fgcolor, w, n; unsigned int cc; + const unsigned int *str = str0; if (y < s->clip_y1 || y >= s->clip_y2 || x >= s->clip_x2) return; @@ -1076,37 +1168,60 @@ static void tty_term_draw_text(QEditScreen *s, qe__unused__ QEFont *font, w = tty_term_glyph_width(s, cc); x += w; if (x >= s->clip_x1) { - /* now we are on the screen. need to put spaces for - wide chars */ + /* pad partially clipped wide char with spaces */ n = x; - if (s->clip_x2 < n) + if (n > s->clip_x2) n = s->clip_x2; n -= s->clip_x1; for (; n > 0; n--) { *ptr = TTYCHAR(' ', fgcolor, TTYCHAR_GETBG(*ptr)); ptr++; } + /* skip combining code points if any */ + while (len > 0 && tty_term_glyph_width(s, *str) == 0) { + len--; + str++; + } break; } } } else { ptr += x; } - for (; len > 0; len--) { - cc = *str++; + for (; len > 0; len--, str++) { + cc = *str; w = tty_term_glyph_width(s, cc); - /* XXX: would need to put spaces for wide chars */ - if (x + w > s->clip_x2) + if (x + w > s->clip_x2) { + /* pad partially clipped wide char with spaces */ + while (x < s->clip_x2) { + *ptr = TTYCHAR(' ', fgcolor, TTYCHAR_GETBG(*ptr)); + ptr++; + x++; + } break; - *ptr = TTYCHAR(cc, fgcolor, TTYCHAR_GETBG(*ptr)); - ptr++; - n = w - 1; - while (n > 0) { - *ptr = TTYCHAR(TTYCHAR_NONE, fgcolor, TTYCHAR_GETBG(*ptr)); - ptr++; - n--; } - x += w; + if (w == 0) { + if (str == str0) + continue; + /* allocate synthetic glyph for multicharacter combination */ + int nacc = 1; + while (nacc < len && tty_term_glyph_width(s, str[nacc]) == 0) + nacc++; + cc = comb_cache_add(ts, str - 1, nacc + 1); + str += nacc - 1; + len -= nacc - 1; + ptr[-1] = TTYCHAR(cc, fgcolor, TTYCHAR_GETBG(ptr[-1])); + } else { + *ptr = TTYCHAR(cc, fgcolor, TTYCHAR_GETBG(*ptr)); + ptr++; + x += w; + while (w > 1) { + /* put placeholders for wide chars */ + *ptr = TTYCHAR(TTYCHAR_NONE, fgcolor, TTYCHAR_GETBG(*ptr)); + ptr++; + w--; + } + } } } @@ -1161,12 +1276,6 @@ static void tty_term_flush(QEditScreen *s) if (ptr1 == ptr2) continue; - /* if first modified char is an accent, backtrack on letter */ - if (qe_isaccent(TTYCHAR_GETCH(*ptr1)) - || qe_isaccent(TTYCHAR_GETCH(ptr1[shadow]))) { - ptr1--; - } - /* quickly scan for last difference on row: * the first difference on row at ptr1 is before ptr2 * so we do not need a test on ptr2 > ptr1 @@ -1214,7 +1323,7 @@ static void tty_term_flush(QEditScreen *s) /* Go to the first difference */ /* CG: this probably does not work if there are * double-width glyphs on the row in front of this - * difference + * difference (actually it should) */ TTY_FPRINTF(s->STDOUT, "\033[%d;%dH", y + 1, (int)(ptr1 - ptr + 1)); @@ -1304,7 +1413,24 @@ static void tty_term_flush(QEditScreen *s) } TTY_PUTC(ch - 32, s->STDOUT); } - } else { + } else +#if MAX_UNICODE_DISPLAY > 0xFFFF + if (ch >= TTYCHAR_COMB && ch < TTYCHAR_COMB + COMB_CACHE_SIZE - 1) { + u8 buf[10], *q; + unsigned int *ip = ts->comb_cache + (ch - TTYCHAR_COMB); + int ncc = *ip++; + + if (ncc < 0x300) { + while (ncc-- > 1) { + q = s->charset->encode_func(s->charset, buf, *ip++); + if (q) { + TTY_FWRITE(buf, 1, q - buf, s->STDOUT); + } + } + } + } else +#endif + { u8 buf[10], *q; int nc; @@ -1365,6 +1491,13 @@ static void tty_term_flush(QEditScreen *s) ts->cursor_y + 1, ts->cursor_x + 1); } fflush(s->STDOUT); + + /* Update combination cache from screen. Should do this before redisplay */ + comb_cache_clean(ts, ts->screen, ts->screen_size); +} + +static void tty_term_describe(QEditScreen *s, EditBuffer *b) { + comb_cache_describe(s, b); } static QEDisplay tty_dpy = { @@ -1390,7 +1523,7 @@ static QEDisplay tty_dpy = { NULL, /* dpy_bmp_lock */ NULL, /* dpy_bmp_unlock */ NULL, /* dpy_full_screen */ - NULL, /* dpy__describe */ + tty_term_describe, NULL, /* next */ }; diff --git a/unicode_width.h b/unicode_width.h new file mode 100644 index 0000000..80e40ab --- /dev/null +++ b/unicode_width.h @@ -0,0 +1,550 @@ + 0x002FF, 1, + 0x0036F, 0, + 0x00482, 1, /* 0400-04FF Cyrillic */ + 0x00489, 0, + 0x00590, 1, /* 0590-05FF Hebrew */ + 0x005BD, 0, + 0x005BE, 1, + 0x005BF, 0, + 0x005C0, 1, + 0x005C2, 0, + 0x005C3, 1, + 0x005C5, 0, + 0x005C6, 1, + 0x005C7, 0, + 0x0060F, 1, /* 0600-06FF Arabic */ + 0x0061A, 0, + 0x0064A, 1, + 0x0065F, 0, + 0x0066F, 1, + 0x00670, 0, + 0x006D5, 1, + 0x006DC, 0, + 0x006DE, 1, + 0x006E4, 0, + 0x006E6, 1, + 0x006E8, 0, + 0x006E9, 1, + 0x006ED, 0, + 0x00710, 1, /* 0700-074F Syriac */ + 0x00711, 0, + 0x0072F, 1, + 0x0074A, 0, + 0x007A5, 1, /* 0780-07BF Thaana */ + 0x007B0, 0, + 0x007EA, 1, /* 07C0-07FF NKo */ + 0x007F3, 0, + 0x00815, 1, /* 0800-083F Samaritan */ + 0x00819, 0, + 0x0081A, 1, + 0x00823, 0, + 0x00824, 1, + 0x00827, 0, + 0x00828, 1, + 0x0082D, 0, + 0x00858, 1, /* 0840-085F Mandaic */ + 0x0085B, 0, + 0x008E2, 1, /* 08A0-08FF Arabic Extended-A */ + 0x00903, 0, /* 0900-097F Devanagari */ + 0x00939, 1, + 0x0093C, 0, + 0x0093D, 1, + 0x0094F, 0, + 0x00950, 1, + 0x00957, 0, + 0x00961, 1, + 0x00963, 0, + 0x00980, 1, /* 0980-09FF Bengali */ + 0x00983, 0, + 0x009BB, 1, + 0x009BC, 0, + 0x009BD, 1, + 0x009C4, 0, + 0x009C6, 1, + 0x009C8, 0, + 0x009CA, 1, + 0x009CD, 0, + 0x009D6, 1, + 0x009D7, 0, + 0x009E1, 1, + 0x009E3, 0, + 0x00A00, 1, /* 0A00-0A7F Gurmukhi */ + 0x00A03, 0, + 0x00A3B, 1, + 0x00A3C, 0, + 0x00A3D, 1, + 0x00A42, 0, + 0x00A46, 1, + 0x00A48, 0, + 0x00A4A, 1, + 0x00A4D, 0, + 0x00A50, 1, + 0x00A51, 0, + 0x00A6F, 1, + 0x00A71, 0, + 0x00A74, 1, + 0x00A75, 0, + 0x00A80, 1, /* 0A80-0AFF Gujarati */ + 0x00A83, 0, + 0x00ABB, 1, + 0x00ABC, 0, + 0x00ABD, 1, + 0x00AC5, 0, + 0x00AC6, 1, + 0x00AC9, 0, + 0x00ACA, 1, + 0x00ACD, 0, + 0x00AE1, 1, + 0x00AE3, 0, + 0x00B00, 1, /* 0B00-0B7F Oriya */ + 0x00B03, 0, + 0x00B3B, 1, + 0x00B3C, 0, + 0x00B3D, 1, + 0x00B44, 0, + 0x00B46, 1, + 0x00B48, 0, + 0x00B4A, 1, + 0x00B4D, 0, + 0x00B55, 1, + 0x00B57, 0, + 0x00B61, 1, + 0x00B63, 0, + 0x00B81, 1, /* 0B80-0BFF Tamil */ + 0x00B82, 0, + 0x00BBD, 1, + 0x00BC2, 0, + 0x00BC5, 1, + 0x00BC8, 0, + 0x00BC9, 1, + 0x00BCD, 0, + 0x00BD6, 1, + 0x00BD7, 0, + 0x00BFF, 1, + 0x00C03, 0, /* 0C00-0C7F Telugu */ + 0x00C3D, 1, + 0x00C44, 0, + 0x00C45, 1, + 0x00C48, 0, + 0x00C49, 1, + 0x00C4D, 0, + 0x00C54, 1, + 0x00C56, 0, + 0x00C61, 1, + 0x00C63, 0, + 0x00C80, 1, /* 0C80-0CFF Kannada */ + 0x00C83, 0, + 0x00CBB, 1, + 0x00CBC, 0, + 0x00CBD, 1, + 0x00CC4, 0, + 0x00CC5, 1, + 0x00CC8, 0, + 0x00CC9, 1, + 0x00CCD, 0, + 0x00CD4, 1, + 0x00CD6, 0, + 0x00CE1, 1, + 0x00CE3, 0, + 0x00D00, 1, /* 0D00-0D7F Malayalam */ + 0x00D03, 0, + 0x00D3D, 1, + 0x00D44, 0, + 0x00D45, 1, + 0x00D48, 0, + 0x00D49, 1, + 0x00D4D, 0, + 0x00D56, 1, + 0x00D57, 0, + 0x00D61, 1, + 0x00D63, 0, + 0x00D81, 1, /* 0D80-0DFF Sinhala */ + 0x00D83, 0, + 0x00DC9, 1, + 0x00DCA, 0, + 0x00DCE, 1, + 0x00DD4, 0, + 0x00DD5, 1, + 0x00DD6, 0, + 0x00DD7, 1, + 0x00DDF, 0, + 0x00DF1, 1, + 0x00DF3, 0, + 0x00E30, 1, /* 0E00-0E7F Thai */ + 0x00E31, 0, + 0x00E33, 1, + 0x00E3A, 0, + 0x00E46, 1, + 0x00E4E, 0, + 0x00EB0, 1, /* 0E80-0EFF Lao */ + 0x00EB1, 0, + 0x00EB3, 1, + 0x00EB9, 0, + 0x00EBA, 1, + 0x00EBC, 0, + 0x00EC7, 1, + 0x00ECD, 0, + 0x00F17, 1, /* 0F00-0FFF Tibetan */ + 0x00F19, 0, + 0x00F34, 1, + 0x00F35, 0, + 0x00F36, 1, + 0x00F37, 0, + 0x00F38, 1, + 0x00F39, 0, + 0x00F3D, 1, + 0x00F3F, 0, + 0x00F70, 1, + 0x00F84, 0, + 0x00F85, 1, + 0x00F87, 0, + 0x00F8C, 1, + 0x00F97, 0, + 0x00F98, 1, + 0x00FBC, 0, + 0x00FC5, 1, + 0x00FC6, 0, + 0x0102A, 1, /* 1000-109F Myanmar */ + 0x0103E, 0, + 0x01055, 1, + 0x01059, 0, + 0x0105D, 1, + 0x01060, 0, + 0x01061, 1, + 0x01064, 0, + 0x01066, 1, + 0x0106D, 0, + 0x01070, 1, + 0x01074, 0, + 0x01081, 1, + 0x0108D, 0, + 0x0108E, 1, + 0x0108F, 0, + 0x01099, 1, + 0x0109D, 0, + 0x010FF, 1, + 0x0115F, 2, /* 1100-11FF Hangul Jamo */ + 0x011A2, 1, + 0x011A7, 2, + 0x011F9, 1, + 0x011FF, 2, + 0x0135C, 1, /* 1200-137F Ethiopic */ + 0x0135F, 0, + 0x01711, 1, /* 1700-171F Tagalog */ + 0x01714, 0, + 0x01731, 1, /* 1720-173F Hanunoo */ + 0x01734, 0, + 0x01751, 1, /* 1740-175F Buhid */ + 0x01753, 0, + 0x01771, 1, /* 1760-177F Tagbanwa */ + 0x01773, 0, + 0x017B3, 1, /* 1780-17FF Khmer */ + 0x017D3, 0, + 0x017DC, 1, + 0x017DD, 0, + 0x0180A, 1, /* 1800-18AF Mongolian */ + 0x0180D, 0, + 0x018A8, 1, + 0x018A9, 0, + 0x0191F, 1, /* 1900-194F Limbu */ + 0x0192B, 0, + 0x0192F, 1, + 0x0193B, 0, + 0x01A16, 1, /* 1A00-1A1F Buginese */ + 0x01A1B, 0, + 0x01A54, 1, /* 1A20-1AAF Tai Tham */ + 0x01A5E, 0, + 0x01A5F, 1, + 0x01A7C, 0, + 0x01A7E, 1, + 0x01A7F, 0, + 0x01AFF, 1, + 0x01B04, 0, /* 1B00-1B87 Balinese */ + 0x01B33, 1, + 0x01B44, 0, + 0x01B6A, 1, + 0x01B73, 0, + 0x01B7F, 1, + 0x01B82, 0, + 0x01B7F, 1, + 0x01B82, 0, /* 1B80-1BBF Sundanese */ + 0x01BA0, 1, + 0x01BAD, 0, + 0x01BE5, 1, /* 1BC0-1BFF Batak */ + 0x01BF3, 0, + 0x01C23, 1, /* 1C00-1C4F Lepcha */ + 0x01C37, 0, + 0x01CCF, 1, + 0x01CD2, 0, /* 1CD0-1CFF Vedic Extensions */ + 0x01CD3, 1, + 0x01CE8, 0, + 0x01CEC, 1, + 0x01CED, 0, + 0x01CF1, 1, + 0x01CF4, 0, + 0x01CF7, 1, + 0x01CF9, 0, + 0x01DBF, 1, + 0x01DF5, 0, /* 1DC0-1DFF Combining Diacritical Marks Supplement */ + 0x01DFB, 1, + 0x01DFF, 0, + 0x0200A, 1, /* 2000-206F General Punctuation */ + 0x0200D, 0, + 0x020CF, 1, + 0x020F0, 0, /* 20D0-20FF Combining Diacritical Marks for Symbols */ + 0x02328, 1, /* 2300-23FF Miscellaneous Technical */ + 0x0232A, 2, + 0x02CEE, 1, /* 2C80-2CFF Coptic */ + 0x02CF1, 0, + 0x02D7E, 1, /* 2D30-2D7F Tifinagh */ + 0x02D7F, 0, + 0x02DDF, 1, + 0x02DFF, 0, + 0x02E7F, 1, + 0x02E99, 2, /* 2E80-2EFF CJK Radicals Supplement */ + 0x02E9A, 1, + 0x02EF3, 2, + 0x02EFF, 1, + 0x02FD5, 2, /* 2F00-2FDF Kangxi Radicals */ + 0x02FEF, 1, + 0x02FFB, 2, /* 2FF0-2FFF Ideographic Description Characters */ + 0x02FFF, 1, + 0x03029, 2, /* 3000-303F CJK Symbols and Punctuation */ + 0x0302F, 0, + 0x0303E, 2, + 0x03040, 1, /* 3040-309F Hiragana */ + 0x03096, 2, + 0x03098, 1, + 0x0309A, 0, + 0x030FF, 2, + 0x03104, 1, /* 3100-312F Bopomofo */ + 0x0312D, 2, + 0x03130, 1, /* 3130-318F Hangul Compatibility Jamo */ + 0x0318E, 2, + 0x0318F, 1, + 0x031BA, 2, /* 31A0-31BF Bopomofo Extended */ + 0x031BF, 1, + 0x031E3, 2, /* 31C0-31EF CJK Strokes */ + 0x031EF, 1, + 0x0321E, 2, /* 3200-32FF Enclosed CJK Letters and Months */ + 0x0321F, 1, + 0x03247, 2, + 0x0324F, 1, + 0x032FE, 2, + 0x032FF, 1, + 0x04DBF, 2, + 0x04DFF, 1, + 0x0A48C, 2, /* A000-A48F Yi Syllables */ + 0x0A48F, 1, + 0x0A4C6, 2, /* A490-A4CF Yi Radicals */ + 0x0A66E, 1, /* A640-A69F Cyrillic Extended-B */ + 0x0A672, 0, + 0x0A673, 1, + 0x0A67D, 0, + 0x0A69D, 1, + 0x0A69F, 0, + 0x0A6EF, 1, /* A6A0-A6FF Bamum */ + 0x0A6F1, 0, + 0x0A801, 1, /* A800-A82F Syloti Nagri */ + 0x0A802, 0, + 0x0A805, 1, + 0x0A806, 0, + 0x0A80A, 1, + 0x0A80B, 0, + 0x0A822, 1, + 0x0A827, 0, + 0x0A87F, 1, + 0x0A881, 0, /* A880-A8DF Saurashtra */ + 0x0A8B3, 1, + 0x0A8C4, 0, + 0x0A8DF, 1, + 0x0A8F1, 0, /* A8E0-A8FF Devanagari Extended */ + 0x0A925, 1, /* A900-A92F Kayah Li */ + 0x0A92D, 0, + 0x0A946, 1, /* A930-A95F Rejang */ + 0x0A953, 0, + 0x0A95F, 1, + 0x0A97C, 2, /* A960-A97F Hangul Jamo Extended-A */ + 0x0A97F, 1, + 0x0A983, 0, /* A980-A9DF Javanese */ + 0x0A9B2, 1, + 0x0A9C0, 0, + 0x0AA28, 1, /* AA00-AA5F Cham */ + 0x0AA36, 0, + 0x0AA42, 1, + 0x0AA43, 0, + 0x0AA4B, 1, + 0x0AA4D, 0, + 0x0AA7A, 1, /* AA60-AA7F Myanmar Extended-A */ + 0x0AA7D, 0, + 0x0AAAF, 1, /* AA80-AADF Tai Viet */ + 0x0AAB0, 0, + 0x0AAB1, 1, + 0x0AAB4, 0, + 0x0AAB6, 1, + 0x0AAB8, 0, + 0x0AABD, 1, + 0x0AABF, 0, + 0x0AAC0, 1, + 0x0AAC1, 0, + 0x0AAEA, 1, /* AAE0-AAFF Meetei Mayek Extensions */ + 0x0AAEF, 0, + 0x0AAF4, 1, + 0x0AAF6, 0, + 0x0ABE2, 1, /* ABC0-ABFF Meetei Mayek */ + 0x0ABEA, 0, + 0x0ABEB, 1, + 0x0ABED, 0, + 0x0ABFF, 1, + 0x0D7A3, 2, /* AC00-D7AF Hangul Syllables */ + 0x0D7FF, 1, + 0x0DFFF, 3, + 0x0F86F, 1, /* E000-F8FF Private Use Area */ + 0x0F87F, 0, + 0x0F8FF, 1, + 0x0FAFF, 2, + 0x0FB1D, 1, /* FB00-FB4F Alphabetic Presentation Forms */ + 0x0FB1E, 0, + 0x0FDFF, 1, + 0x0FE0F, 0, + 0x0FE19, 2, /* FE10-FE1F Vertical Forms */ + 0x0FE1F, 1, + 0x0FE2F, 0, + 0x0FE52, 2, /* FE50-FE6F Small Form Variants */ + 0x0FE53, 1, + 0x0FE66, 2, + 0x0FE67, 1, + 0x0FE6B, 2, + 0x0FEFE, 1, /* FE70-FEFF Arabic Presentation Forms-B */ + 0x0FEFF, 0, + 0x0FF00, 1, /* FF00-FFEF Halfwidth and Fullwidth Forms */ + 0x0FF60, 2, + 0x0FF9D, 1, + 0x0FF9F, 0, + 0x0FFDF, 1, + 0x0FFE6, 2, + 0x101FC, 1, /* 101D0-107FF Alphabetic and syllabic LTR scripts and sets of symbols */ + 0x101FD, 0, + 0x102DF, 1, + 0x102E0, 0, + 0x10375, 1, + 0x1037A, 0, + 0x10A00, 1, /* 10800-10FFF Alphabetic and syllabic RTL scripts */ + 0x10A03, 0, + 0x10A04, 1, + 0x10A06, 0, + 0x10A0B, 1, + 0x10A0F, 0, + 0x10A37, 1, + 0x10A3A, 0, + 0x10A3E, 1, + 0x10A3F, 0, + 0x10AE4, 1, + 0x10AE6, 0, + 0x10FFF, 1, + 0x11002, 0, /* 11000-11FFF Brahmic scripts */ + 0x11037, 1, + 0x11046, 0, + 0x1107E, 1, + 0x11082, 0, + 0x110AF, 1, + 0x110BA, 0, + 0x110FF, 1, + 0x11102, 0, + 0x11126, 1, + 0x11134, 0, + 0x11172, 1, + 0x11173, 0, + 0x1117F, 1, + 0x11182, 0, + 0x111B2, 1, + 0x111C0, 0, + 0x111C9, 1, + 0x111CC, 0, + 0x1122B, 1, + 0x11237, 0, + 0x112DE, 1, + 0x112EA, 0, + 0x112FF, 1, + 0x11303, 0, + 0x1133B, 1, + 0x1133C, 0, + 0x1133D, 1, + 0x11344, 0, + 0x11346, 1, + 0x11348, 0, + 0x1134A, 1, + 0x1134D, 0, + 0x11356, 1, + 0x11357, 0, + 0x11361, 1, + 0x11363, 0, + 0x11365, 1, + 0x1136C, 0, + 0x1136F, 1, + 0x11374, 0, + 0x114AF, 1, + 0x114C3, 0, + 0x115AE, 1, + 0x115B5, 0, + 0x115B7, 1, + 0x115C0, 0, + 0x115DB, 1, + 0x115DD, 0, + 0x1162F, 1, + 0x11640, 0, + 0x116AA, 1, + 0x116B7, 0, + 0x1171C, 1, + 0x1172B, 0, + 0x16AEF, 1, /* 16000-16FFF Recently-devised scripts */ + 0x16AF4, 0, + 0x16B2F, 1, + 0x16B36, 0, + 0x16F50, 1, + 0x16F7E, 0, + 0x16F8E, 1, + 0x16F92, 0, + 0x1AFFF, 1, /* 17000-1B4FF Large East Asian scripts */ + 0x1B001, 2, + 0x1BC9C, 1, /* 1BC00-1BFFF Shorthands */ + 0x1BC9E, 0, + 0x1D164, 1, /* 1D000-1DFFF Notational systems */ + 0x1D169, 0, + 0x1D16C, 1, + 0x1D172, 0, + 0x1D17A, 1, + 0x1D182, 0, + 0x1D184, 1, + 0x1D18B, 0, + 0x1D1A9, 1, + 0x1D1AD, 0, + 0x1D241, 1, + 0x1D244, 0, + 0x1D9FF, 1, + 0x1DA36, 0, + 0x1DA3A, 1, + 0x1DA6C, 0, + 0x1DA74, 1, + 0x1DA75, 0, + 0x1DA83, 1, + 0x1DA84, 0, + 0x1DA9A, 1, + 0x1DA9F, 0, + 0x1DAA0, 1, + 0x1DAAF, 0, + 0x1E8CF, 1, /* 1E800-1EFFF RTL scripts */ + 0x1E8D6, 0, + 0x1F1FF, 1, /* 1F100-1F2FF Alphanumeric and ideographic sets */ + 0x1F202, 2, + 0x1F20F, 1, + 0x1F23A, 2, + 0x1F23F, 1, + 0x1F248, 2, + 0x1F24F, 1, + 0x1F251, 2, + 0x1FFFF, 1, + 0xDFFFF, 2, + 0xE00FF, 1, + 0xE01EF, 0, + 0xFFFFFFFF, 1, /* catch all */ diff --git a/unihex.c b/unihex.c index ed78ec2..f4fdefc 100644 --- a/unihex.c +++ b/unihex.c @@ -145,7 +145,13 @@ static int unihex_display_line(EditState *s, DisplayState *ds, int offset) offset2 = offset1 = -1; } } - display_char(ds, offset1, offset2, b); + if (qe_isaccent(b)) { + /* insert space to make accent stand on its own */ + display_char(ds, offset1, offset2, ' '); + display_char(ds, -1, -1, b); + } else { + display_char(ds, offset1, offset2, b); + } /* spacing out single width glyphs may be less readable */ if (unicode_tty_glyph_width(b) < 2) { display_char(ds, -1, -1, ' ');