diff --git a/TODO.md b/TODO.md index 35b7e5b..9315b35 100644 --- a/TODO.md +++ b/TODO.md @@ -63,6 +63,8 @@ ``` an EditState would have an embedded QECursor that contains the buffer and offset an EditBuffer could also have an embedded QECursor with no EditState +* use hash tables for command and variable names +* sort key binding tables? ### Charsets / Unicode / Bidir @@ -104,13 +106,7 @@ jisx0213-2004-std.txt sjis-0213-2004-std.txt MAC-CYRILLIC.TXT MAC-GREEK.TXT MAC-ICELAND.TXT MAC-TURKISH.TXT koi8_ru.cp APL-ISO-IR-68.TXT GSM0338.TXT SGML.TXT -* accented letters on macOS -* combining unicode glyphs produce bogus cursor positions - example: `V M-'` this problem occurs if no combined glyph exists. - qemacs does not take into account combination performed by the terminal. - Terminal glyph width of 0 should be supported. * deal with accents in filenames (macOS uses combining accents encoded as UTF-8) -* fix backspace on combining glyphs ### Windowing / Display @@ -197,6 +193,7 @@ insert_window_left() deletes some left-most windows * add hook on file change * handle files starting with re: * check file permissions. +* handle filenames with embedded spaces * use trick for entering spaces in filename prompts without completion * fix `s->offset` reset to 0 upon `C-x C-f newfile ENT C-x 2 C-x b ENT` * insert-file: load via separate buffer with charset conversion @@ -213,7 +210,6 @@ insert_window_left() deletes some left-most windows * `find-file-read-only-other-window` on `C-x 4 r` * `save-modified-buffers` on `C-x s` * `find-file`: should support scp syntax for remote loading -* handle filenames with embedded spaces * avoid error in new file * actually load file in `find-file-noselect` * should update symbolic links times when saving files @@ -533,6 +529,7 @@ insert_window_left() deletes some left-most windows * html/xml: fix colorizer for multi-line tags and attributes * [BUG] xml: crash bug on **johnmacfarlane.net/texmath.xhtml** * `html-mode`: support hex entities +* add syntax based wrapping mode for very long lines * distribute libqhtml as a separate project * OPTIMIZE `eb_nextc` et al or always duplicate box content (big speed improvement). diff --git a/qfribidi.h b/qfribidi.h index f51b5a2..ac57e4a 100644 --- a/qfribidi.h +++ b/qfribidi.h @@ -30,36 +30,36 @@ typedef int FriBidiChar; typedef enum { /* do not change the order of these constants since 'property_val' depends on them */ - FRIBIDI_TYPE_LTR, /* Strong left to right */ - FRIBIDI_TYPE_RTL, /* Right to left characters */ - FRIBIDI_TYPE_WL, /* Weak left to right */ - FRIBIDI_TYPE_WR, /* Weak right to left */ - FRIBIDI_TYPE_EN, /* European digit */ - FRIBIDI_TYPE_ES, /* European number separator */ - FRIBIDI_TYPE_ET, /* European number terminator */ - FRIBIDI_TYPE_AN, /* Arabic digit */ - FRIBIDI_TYPE_CS, /* Common Separator */ - FRIBIDI_TYPE_BS, /* Block separator */ - FRIBIDI_TYPE_SS, /* Segment separator */ - FRIBIDI_TYPE_WS, /* Whitespace */ - FRIBIDI_TYPE_AL, /* Arabic characters */ - FRIBIDI_TYPE_NSM, /* Non spacing mark */ - FRIBIDI_TYPE_BN, - FRIBIDI_TYPE_ON, /* Other Neutral */ - FRIBIDI_TYPE_LRE, /* Left-To-Right embedding */ - FRIBIDI_TYPE_RLE, /* Right-To-Left embedding */ - FRIBIDI_TYPE_PDF, /* Pop directional override */ - FRIBIDI_TYPE_LRO, /* Left-To-Right override */ - FRIBIDI_TYPE_RLO, /* Right-To-Left override */ + FRIBIDI_TYPE_LTR, /* Strong Left-to-Right */ + FRIBIDI_TYPE_RTL, /* Right-to-left characters */ + FRIBIDI_TYPE_WL, /* Weak left to right */ + FRIBIDI_TYPE_WR, /* Weak right to left */ + FRIBIDI_TYPE_EN, /* European Numeral */ + FRIBIDI_TYPE_ES, /* European number Separator */ + FRIBIDI_TYPE_ET, /* European number Terminator */ + FRIBIDI_TYPE_AN, /* Arabic Numeral */ + FRIBIDI_TYPE_CS, /* Common Separator */ + FRIBIDI_TYPE_BS, /* Block Separator */ + FRIBIDI_TYPE_SS, /* Segment Separator */ + FRIBIDI_TYPE_WS, /* Whitespace */ + FRIBIDI_TYPE_AL, /* Arabic Letter */ + FRIBIDI_TYPE_NSM, /* Non Spacing Mark */ + FRIBIDI_TYPE_BN, /* Boundary Neutral */ + FRIBIDI_TYPE_ON, /* Other Neutral */ + FRIBIDI_TYPE_LRE, /* Left-to-Right Embedding */ + FRIBIDI_TYPE_RLE, /* Right-to-Left Embedding */ + FRIBIDI_TYPE_PDF, /* Pop Directional Flag */ + FRIBIDI_TYPE_LRO, /* Left-to-Right Override */ + FRIBIDI_TYPE_RLO, /* Right-to-Left Override */ /* The following are only used internally */ FRIBIDI_TYPE_SOT, FRIBIDI_TYPE_EOT, FRIBIDI_TYPE_N, FRIBIDI_TYPE_E, - FRIBIDI_TYPE_CTL, /* Control units */ - FRIBIDI_TYPE_EO, /* Control units */ - FRIBIDI_TYPE_NULL, /* type record is to be deleted */ + FRIBIDI_TYPE_CTL, /* Control units */ + FRIBIDI_TYPE_EO, /* Control units */ + FRIBIDI_TYPE_NULL, /* type record is to be deleted */ FRIBIDI_TYPE_L = FRIBIDI_TYPE_LTR, FRIBIDI_TYPE_R = FRIBIDI_TYPE_RTL, FRIBIDI_TYPE_CM = FRIBIDI_TYPE_ON + 2, @@ -68,7 +68,7 @@ typedef enum { /*====================================================================== // Typedef for the run-length list. //----------------------------------------------------------------------*/ -typedef struct _TypeLink { +typedef struct TypeLink { FriBidiCharType type; int pos; int len; diff --git a/tools/cptoqe.c b/tools/cptoqe.c index b1366fa..6b4124d 100644 --- a/tools/cptoqe.c +++ b/tools/cptoqe.c @@ -156,7 +156,7 @@ static void handle_cp(FILE *f0, const char *name, const char *fname, int lineno) } } - c1 = strtol_c(p, &p, 16); + c1 = strtol(p, &p, 16); if (!isspace((unsigned char)*p)) { /* ignore ranges such as "0x20-0x7e idem" */ continue; @@ -166,7 +166,7 @@ static void handle_cp(FILE *f0, const char *name, const char *fname, int lineno) /* unknown */ /* continue; */ } - c2 = strtol_c(p, &p, 16); + c2 = strtol(p, &p, 16); if (c1 >= 256) { fprintf(stderr, "%s:%d: ERROR %d %d\n", filename, lineno, c1, c2); continue; diff --git a/tools/kmaptoqe.c b/tools/kmaptoqe.c index 6127a20..13a460c 100644 --- a/tools/kmaptoqe.c +++ b/tools/kmaptoqe.c @@ -455,8 +455,7 @@ static inline char *skipspaces(char *p) { return p; } -static int getcp(char *p, char **pp) -{ +static int getcp(char *p, char **pp) { if (*p == '\0') { return -1; } else @@ -477,16 +476,17 @@ static int getcp(char *p, char **pp) ((p[2] - '0') << 0); } else { *pp = p + 1; - return *p; + return (unsigned char)*p; } } else if ((*p & 0xe0) == 0xc0 && (p[1] & 0xc0) == 0x80) { - /* UTF-8 character */ + /* 2 byte UTF-8 encoded glyph */ *pp = p + 2; - return ((*p - 0xc0) << 6) | ((p[1] - 0x80) << 0); + return ((*p & 0x1f) << 6) | ((p[1] & 0x3f) << 0); } else { + /* single byte */ *pp = p + 1; - return *p; + return (unsigned char)*p; } } diff --git a/tools/ligtoqe.c b/tools/ligtoqe.c index e4f3bf9..33eb5a0 100644 --- a/tools/ligtoqe.c +++ b/tools/ligtoqe.c @@ -108,7 +108,7 @@ static void help(void) printf("usage: ligtoqe [-u] sourcefile.lig output_ligature_file\n" "Build a ligature resource file for qemacs\n" "\n" - "-u : output an UTF8 encode summy on stdout\n"); + "-u : output a UTF8 encoded summary on stdout\n"); exit(1); } diff --git a/tools/unicode_gen.c b/tools/unicode_gen.c index ceee586..a185f0b 100644 --- a/tools/unicode_gen.c +++ b/tools/unicode_gen.c @@ -152,6 +152,8 @@ typedef struct genstate_t { FILE *fc; int ind, argc; char **argv; + int use_names; + unsigned int filter_start, filter_end; } genstate_t; #define has_arg(gp) ((gp)->ind < (gp)->argc) @@ -625,30 +627,13 @@ static void set_raw_tty(void) { } static int make_tty_width_table(genstate_t *gp) { - unsigned int filter_start = 0x20, filter_end = CHARCODE_MAX; unsigned int code; unsigned char *width; - int narg = 0; if (getenv("QELEVEL")) { fprintf(stderr, "cannot run in quick emacs shell buffer\n"); return 1; } - for (; has_arg(gp); next_arg(gp)) { - char *arg = peek_arg(gp); - if (isdigit((unsigned char)*arg)) { - switch (narg++) { - case 0: - filter_start = strtoul(arg, NULL, 16); - continue; - case 1: - filter_end = strtoul(arg, NULL, 16); - continue; - } - } - break; - } - width = calloc(sizeof(*width), CHARCODE_MAX + 1); /* set default widths */ set_range(width, 0, CHARCODE_MAX, 1); @@ -663,7 +648,7 @@ static int make_tty_width_table(genstate_t *gp) { setvbuf(stdin, NULL, _IONBF, 0); setvbuf(stderr, NULL, _IONBF, 0); - for (code = filter_start; code <= filter_end; code++) { + for (code = gp->filter_start; code <= gp->filter_end; code++) { char buf[10]; char line[32]; int len, c, n, y, x; @@ -1106,7 +1091,7 @@ static int make_wcwidth_variant_table(genstate_t *gp) { X(CS, "Common Separator") \ X(BS, "Block Separator") \ X(SS, "Segment Separator") \ - X(WS, "WhiteSpace") \ + X(WS, "Whitespace") \ X(AL, "Arabic Letter") \ X(NSM, "Non Spacing Mark") \ X(BN, "Boundary Neutral") \ @@ -1167,15 +1152,9 @@ static int make_bidir_table(genstate_t *gp) { char line[1024]; FILE *fp; unsigned char *bidir_type; - int use_names = 0; int c, lineno, pos, count, count1, count2, last_bidir; unsigned long code, code1, code2; - if (has_arg(gp) && !strcmp(peek_arg(gp), "-a")) { - use_names = 1; - next_arg(gp); - } - if (!(fp = open_unicode_file(gp->unicode_dir, gp->unicode_version, "UnicodeData.txt", filename, sizeof filename))) @@ -1267,11 +1246,19 @@ static int make_bidir_table(genstate_t *gp) { count = count1 + count2 - 1; /* output the enum values */ - fprintf(gp->fc, "\n" "enum bidir_class {\n"); -#define X(e,s) fprintf(gp->fc, " %s, /* %s */\n", #e, s); + /* public type with package name prefixes */ +#define X(e,s) fprintf(gp->fi, " QE_BIDIR_%s, /* %s */\n", #e, s); + fprintf(gp->fi, "enum qe_bidir_class {\n"); BIDIR_CLASSES(X); + fprintf(gp->fi, "};\n"); #undef X + + /* private type with shorter aliases */ +#define X(e,s) fprintf(gp->fc, " %s = QE_BIDIR_%s, /* %s */\n", #e, #e, s); + fprintf(gp->fc, "\n" "enum bidir_class {\n"); + BIDIR_CLASSES(X); fprintf(gp->fc, "};\n"); +#undef X /* output a direct table for single byte code points */ pos = 0; @@ -1279,7 +1266,7 @@ static int make_bidir_table(genstate_t *gp) { fprintf(gp->fc, "\n" "static const unsigned char bidir_table_00[%d] = {", 256); for (code = 0; code <= 255; code++) { if (!(pos++ % 8)) fprintf(gp->fc, "\n "); - if (use_names) { + if (gp->use_names) { fprintf(gp->fc, " %3s,", bidir_class_names[bidir_type[code]]); } else { fprintf(gp->fc, " 0x%02x,", bidir_type[code]); @@ -1289,7 +1276,7 @@ static int make_bidir_table(genstate_t *gp) { /* output a composite table for other code points */ fprintf(gp->fc, "\n" "static const unsigned int bidir_table[%d] = {", count2); - if (use_names) + if (gp->use_names) fprintf(gp->fc, "\n" "#define X(c,v) (((c) << 8) | (v))"); pos = 0; last_bidir = -1; @@ -1297,7 +1284,7 @@ static int make_bidir_table(genstate_t *gp) { if (last_bidir != bidir_type[code]) { /* output combined table */ if (!(pos++ % 4)) fprintf(gp->fc, "\n "); - if (use_names) { + if (gp->use_names) { fprintf(gp->fc, " X(0x%06lx, %3s),", code, bidir_class_names[bidir_type[code]]); } else { fprintf(gp->fc, " 0x%08lx,", (code << 8) | bidir_type[code]); @@ -1305,7 +1292,7 @@ static int make_bidir_table(genstate_t *gp) { last_bidir = bidir_type[code]; } } - if (use_names) + if (gp->use_names) fprintf(gp->fc, "\n" "#undef X"); fprintf(gp->fc, "\n};\n"); #if 0 @@ -1327,7 +1314,7 @@ static int make_bidir_table(genstate_t *gp) { for (code = 0; code <= CHARCODE_MAX; code++) { if (last_bidir != bidir_type[code]) { if (!(pos++ % 8)) fprintf(gp->fc, "\n "); - if (use_names) { + if (gp->use_names) { fprintf(gp->fc, " %3s,", bidir_class_names[bidir_type[code]]); } else { fprintf(gp->fc, " 0x%02x,", bidir_type[code]); @@ -1336,6 +1323,67 @@ static int make_bidir_table(genstate_t *gp) { } } fprintf(gp->fc, "\n};\n"); +#endif + /* use a binary lookup loop */ + // XXX: this is inefficient and slow + // should use multi-level indirect tables +#if 00 + enum qe_bidir_class qe_bidir_get_type(unsigned int ch) { + int a, b; + if (ch < 256) + return bidir_table_00[ch]; + if (ch > CHARCODE_MAX) + return LTR; + a = 0; + b = countof(bidir_table) - 1; + while (a < b) { + int m = (a + b) >> 1; + if (ch < bidir_table[m] >> 8) + b = m; + else + a = m; + } + return bidir_table[a] & 0xFF; + } + + /* version for test with ASCII chars */ + static inline enum qe_bidir_class qe_bidir_get_type_test(unsigned int ch) { + if (ch >= 'A' && ch <= 'Z') + return QE_BIDIR_RTL; + else + return qe_bidir_get_type(ch); + } +#else + /* function implementation */ + fprintf(gp->fc, "\n"); + fprintf(gp->fc, "enum qe_bidir_class qe_bidir_get_type(unsigned int ch) {\n"); + fprintf(gp->fc, " int a, b;\n"); + fprintf(gp->fc, " if (ch < 256)\n"); + fprintf(gp->fc, " return bidir_table_00[ch];\n"); + fprintf(gp->fc, " if (ch > CHARCODE_MAX)\n"); + fprintf(gp->fc, " return LTR;\n"); + fprintf(gp->fc, " a = 0;\n"); + fprintf(gp->fc, " b = countof(bidir_table) - 1;\n"); + fprintf(gp->fc, " while (a < b) {\n"); + fprintf(gp->fc, " int m = (a + b) >> 1;\n"); + fprintf(gp->fc, " if (ch < bidir_table[m] >> 8)\n"); + fprintf(gp->fc, " b = m;\n"); + fprintf(gp->fc, " else\n"); + fprintf(gp->fc, " a = m;\n"); + fprintf(gp->fc, " }\n"); + fprintf(gp->fc, " return bidir_table[a] & 0xFF;\n"); + fprintf(gp->fc, "}\n"); + /* function prototypes */ + fprintf(gp->fi, "\n"); + fprintf(gp->fi, "enum qe_bidir_class qe_bidir_get_type(unsigned int ch);\n"); + fprintf(gp->fi, "\n"); + fprintf(gp->fi, "/* version for test with ASCII chars */\n"); + fprintf(gp->fi, "static inline enum qe_bidir_class qe_bidir_get_type_test(unsigned int ch) {\n"); + fprintf(gp->fi, " if (ch >= 'A' && ch <= 'Z')\n"); + fprintf(gp->fi, " return QE_BIDIR_RTL;\n"); + fprintf(gp->fi, " else\n"); + fprintf(gp->fi, " return qe_bidir_get_type(ch);\n"); + fprintf(gp->fi, "}\n"); #endif free(bidir_type); return 0; @@ -1360,7 +1408,11 @@ static int usage(const char *name) { } int main(int argc, char *argv[]) { - genstate_t gp[1] = {{ NULL, "unidata", NULL, NULL, 0, stdout, stdout, 1, argc, argv }}; + genstate_t gp[1] = {{ + NULL, "unidata", NULL, NULL, 0, + stdout, stdout, 1, argc, argv, + 0, 0x20, CHARCODE_MAX + }}; int tasks = 0; #define MAKE_WCWIDTH_TABLE (1 << 0) #define MAKE_TTY_WIDTH_TABLE (1 << 1) @@ -1386,6 +1438,10 @@ int main(int argc, char *argv[]) { } else if (strequal(arg, "-b")) { tasks |= MAKE_BIDIR_TABLE; + if (has_arg(gp) && !strcmp(peek_arg(gp), "-a")) { + gp->use_names = 1; + next_arg(gp); + } } else if (strequal(arg, "-w")) { tasks |= MAKE_WCWIDTH_TABLE; @@ -1395,7 +1451,12 @@ int main(int argc, char *argv[]) { } else if (strequal(arg, "-W")) { tasks |= MAKE_TTY_WIDTH_TABLE; + if (has_arg(gp) && isdigit((unsigned char)*peek_arg(gp))) + gp->filter_start = strtoul(get_arg(gp), NULL, 16); + if (has_arg(gp) && isdigit((unsigned char)*peek_arg(gp))) + gp->filter_end = strtoul(get_arg(gp), NULL, 16); } else { + fprintf(stderr, "%s: unknown option %s\n", NAME, arg); return usage(NAME); } } diff --git a/unicode_join.c b/unicode_join.c index 839b847..83d70d5 100644 --- a/unicode_join.c +++ b/unicode_join.c @@ -143,7 +143,7 @@ int expand_ligature(unsigned int *buf, int c) } /* apply all the ligature rules in logical order. Always return a - smaller buffer */ + smaller or equal buffer */ static int unicode_ligature(unsigned int *buf_out, unsigned int *pos_L_to_V, int len) @@ -152,6 +152,7 @@ static int unicode_ligature(unsigned int *buf_out, unsigned int *q; const unsigned short *lig; /* CG: C99 variable-length arrays may be too large */ + // XXX: why do we need a local copy? unsigned int buf[len]; blockcpy(buf, buf_out, len); @@ -167,7 +168,7 @@ static int unicode_ligature(unsigned int *buf_out, goto found; } /* fast test to eliminate common cases */ - if (i == (len - 1)) + if (i == len - 1) goto nolig; l2 = buf[i + 1]; if (l1 <= 0x7f && l2 <= 0x7f) @@ -224,20 +225,21 @@ static int unicode_ligature(unsigned int *buf_out, static int unicode_classify(unsigned int *buf, int len) { - int i, mask, c; + int i, mask; mask = 0; for (i = 0; i < len; i++) { - c = buf[i]; + unsigned int c = buf[i]; if (c <= 0x7f) /* latin1 fast handling */ continue; mask |= UNICODE_NONASCII; - if (c >= 0x2000) /* fast test for non handled scripts */ - continue; - if (c >= 0x600 && c <= 0x6ff) - mask |= UNICODE_ARABIC; - else if (c >= 0x900 && c <= 0x97f) - mask |= UNICODE_INDIC; + if (c < 0xA00) { + if ((c & ~0xff) == 0x600) /* 0600..06FF */ + mask |= UNICODE_ARABIC; + else + if ((c & ~0x7f) == 0x900) /* 0900..097F */ + mask |= UNICODE_INDIC; + } } return mask; } @@ -265,8 +267,8 @@ static void bidi_reverse_buf(unsigned int *str, int len) } /* Convert a string of unicode characters to a string of glyphs. We - suppose that the font implements a minimum number of standard - ligatures chars. The string is reversed if 'reversed' is set to + assume that the font implements a minimum number of standard + ligature chars. The string is reversed if 'reversed' is set to deal with the bidir case. 'char_to_glyph_pos' gives the index of the first glyph associated to a given character of the source buffer. */