Small fixes

- fix warnings in tools/cptoqe.c
- fix signed char issue in tools/kmaptoqe.c
- fix optional argument parsing in tools/unicode_gen.c
- generate qe_bidir_get_type() and qe_bidir_get_type_test()
  functions for bidir_tables target
This commit is contained in:
Charlie Gordon 2022-10-01 15:35:23 +02:00
commit fd76eae931
7 changed files with 146 additions and 86 deletions

11
TODO.md
View file

@ -63,6 +63,8 @@
```
an EditState would have an embedded QECursor that contains the buffer and offset
an EditBuffer could also have an embedded QECursor with no EditState
* use hash tables for command and variable names
* sort key binding tables?
### Charsets / Unicode / Bidir
@ -104,13 +106,7 @@
jisx0213-2004-std.txt sjis-0213-2004-std.txt
MAC-CYRILLIC.TXT MAC-GREEK.TXT MAC-ICELAND.TXT MAC-TURKISH.TXT
koi8_ru.cp APL-ISO-IR-68.TXT GSM0338.TXT SGML.TXT
* accented letters on macOS
* combining unicode glyphs produce bogus cursor positions
example: `V M-'` this problem occurs if no combined glyph exists.
qemacs does not take into account combination performed by the terminal.
Terminal glyph width of 0 should be supported.
* deal with accents in filenames (macOS uses combining accents encoded as UTF-8)
* fix backspace on combining glyphs
### Windowing / Display
@ -197,6 +193,7 @@ insert_window_left() deletes some left-most windows
* add hook on file change
* handle files starting with re:
* check file permissions.
* handle filenames with embedded spaces
* use trick for entering spaces in filename prompts without completion
* fix `s->offset` reset to 0 upon `C-x C-f newfile ENT C-x 2 C-x b ENT`
* insert-file: load via separate buffer with charset conversion
@ -213,7 +210,6 @@ insert_window_left() deletes some left-most windows
* `find-file-read-only-other-window` on `C-x 4 r`
* `save-modified-buffers` on `C-x s`
* `find-file`: should support scp syntax for remote loading
* handle filenames with embedded spaces
* avoid error in new file
* actually load file in `find-file-noselect`
* should update symbolic links times when saving files
@ -533,6 +529,7 @@ insert_window_left() deletes some left-most windows
* html/xml: fix colorizer for multi-line tags and attributes
* [BUG] xml: crash bug on **johnmacfarlane.net/texmath.xhtml**
* `html-mode`: support hex entities
* add syntax based wrapping mode for very long lines
* distribute libqhtml as a separate project
* OPTIMIZE `eb_nextc` et al or always duplicate box content (big speed improvement).

View file

@ -30,36 +30,36 @@ typedef int FriBidiChar;
typedef enum {
/* do not change the order of these constants since 'property_val'
depends on them */
FRIBIDI_TYPE_LTR, /* Strong left to right */
FRIBIDI_TYPE_RTL, /* Right to left characters */
FRIBIDI_TYPE_WL, /* Weak left to right */
FRIBIDI_TYPE_WR, /* Weak right to left */
FRIBIDI_TYPE_EN, /* European digit */
FRIBIDI_TYPE_ES, /* European number separator */
FRIBIDI_TYPE_ET, /* European number terminator */
FRIBIDI_TYPE_AN, /* Arabic digit */
FRIBIDI_TYPE_CS, /* Common Separator */
FRIBIDI_TYPE_BS, /* Block separator */
FRIBIDI_TYPE_SS, /* Segment separator */
FRIBIDI_TYPE_WS, /* Whitespace */
FRIBIDI_TYPE_AL, /* Arabic characters */
FRIBIDI_TYPE_NSM, /* Non spacing mark */
FRIBIDI_TYPE_BN,
FRIBIDI_TYPE_ON, /* Other Neutral */
FRIBIDI_TYPE_LRE, /* Left-To-Right embedding */
FRIBIDI_TYPE_RLE, /* Right-To-Left embedding */
FRIBIDI_TYPE_PDF, /* Pop directional override */
FRIBIDI_TYPE_LRO, /* Left-To-Right override */
FRIBIDI_TYPE_RLO, /* Right-To-Left override */
FRIBIDI_TYPE_LTR, /* Strong Left-to-Right */
FRIBIDI_TYPE_RTL, /* Right-to-left characters */
FRIBIDI_TYPE_WL, /* Weak left to right */
FRIBIDI_TYPE_WR, /* Weak right to left */
FRIBIDI_TYPE_EN, /* European Numeral */
FRIBIDI_TYPE_ES, /* European number Separator */
FRIBIDI_TYPE_ET, /* European number Terminator */
FRIBIDI_TYPE_AN, /* Arabic Numeral */
FRIBIDI_TYPE_CS, /* Common Separator */
FRIBIDI_TYPE_BS, /* Block Separator */
FRIBIDI_TYPE_SS, /* Segment Separator */
FRIBIDI_TYPE_WS, /* Whitespace */
FRIBIDI_TYPE_AL, /* Arabic Letter */
FRIBIDI_TYPE_NSM, /* Non Spacing Mark */
FRIBIDI_TYPE_BN, /* Boundary Neutral */
FRIBIDI_TYPE_ON, /* Other Neutral */
FRIBIDI_TYPE_LRE, /* Left-to-Right Embedding */
FRIBIDI_TYPE_RLE, /* Right-to-Left Embedding */
FRIBIDI_TYPE_PDF, /* Pop Directional Flag */
FRIBIDI_TYPE_LRO, /* Left-to-Right Override */
FRIBIDI_TYPE_RLO, /* Right-to-Left Override */
/* The following are only used internally */
FRIBIDI_TYPE_SOT,
FRIBIDI_TYPE_EOT,
FRIBIDI_TYPE_N,
FRIBIDI_TYPE_E,
FRIBIDI_TYPE_CTL, /* Control units */
FRIBIDI_TYPE_EO, /* Control units */
FRIBIDI_TYPE_NULL, /* type record is to be deleted */
FRIBIDI_TYPE_CTL, /* Control units */
FRIBIDI_TYPE_EO, /* Control units */
FRIBIDI_TYPE_NULL, /* type record is to be deleted */
FRIBIDI_TYPE_L = FRIBIDI_TYPE_LTR,
FRIBIDI_TYPE_R = FRIBIDI_TYPE_RTL,
FRIBIDI_TYPE_CM = FRIBIDI_TYPE_ON + 2,
@ -68,7 +68,7 @@ typedef enum {
/*======================================================================
// Typedef for the run-length list.
//----------------------------------------------------------------------*/
typedef struct _TypeLink {
typedef struct TypeLink {
FriBidiCharType type;
int pos;
int len;

View file

@ -156,7 +156,7 @@ static void handle_cp(FILE *f0, const char *name, const char *fname, int lineno)
}
}
c1 = strtol_c(p, &p, 16);
c1 = strtol(p, &p, 16);
if (!isspace((unsigned char)*p)) {
/* ignore ranges such as "0x20-0x7e idem" */
continue;
@ -166,7 +166,7 @@ static void handle_cp(FILE *f0, const char *name, const char *fname, int lineno)
/* unknown */
/* continue; */
}
c2 = strtol_c(p, &p, 16);
c2 = strtol(p, &p, 16);
if (c1 >= 256) {
fprintf(stderr, "%s:%d: ERROR %d %d\n", filename, lineno, c1, c2);
continue;

View file

@ -455,8 +455,7 @@ static inline char *skipspaces(char *p) {
return p;
}
static int getcp(char *p, char **pp)
{
static int getcp(char *p, char **pp) {
if (*p == '\0') {
return -1;
} else
@ -477,16 +476,17 @@ static int getcp(char *p, char **pp)
((p[2] - '0') << 0);
} else {
*pp = p + 1;
return *p;
return (unsigned char)*p;
}
} else
if ((*p & 0xe0) == 0xc0 && (p[1] & 0xc0) == 0x80) {
/* UTF-8 character */
/* 2 byte UTF-8 encoded glyph */
*pp = p + 2;
return ((*p - 0xc0) << 6) | ((p[1] - 0x80) << 0);
return ((*p & 0x1f) << 6) | ((p[1] & 0x3f) << 0);
} else {
/* single byte */
*pp = p + 1;
return *p;
return (unsigned char)*p;
}
}

View file

@ -108,7 +108,7 @@ static void help(void)
printf("usage: ligtoqe [-u] sourcefile.lig output_ligature_file\n"
"Build a ligature resource file for qemacs\n"
"\n"
"-u : output an UTF8 encode summy on stdout\n");
"-u : output a UTF8 encoded summary on stdout\n");
exit(1);
}

View file

@ -152,6 +152,8 @@ typedef struct genstate_t {
FILE *fc;
int ind, argc;
char **argv;
int use_names;
unsigned int filter_start, filter_end;
} genstate_t;
#define has_arg(gp) ((gp)->ind < (gp)->argc)
@ -625,30 +627,13 @@ static void set_raw_tty(void) {
}
static int make_tty_width_table(genstate_t *gp) {
unsigned int filter_start = 0x20, filter_end = CHARCODE_MAX;
unsigned int code;
unsigned char *width;
int narg = 0;
if (getenv("QELEVEL")) {
fprintf(stderr, "cannot run in quick emacs shell buffer\n");
return 1;
}
for (; has_arg(gp); next_arg(gp)) {
char *arg = peek_arg(gp);
if (isdigit((unsigned char)*arg)) {
switch (narg++) {
case 0:
filter_start = strtoul(arg, NULL, 16);
continue;
case 1:
filter_end = strtoul(arg, NULL, 16);
continue;
}
}
break;
}
width = calloc(sizeof(*width), CHARCODE_MAX + 1);
/* set default widths */
set_range(width, 0, CHARCODE_MAX, 1);
@ -663,7 +648,7 @@ static int make_tty_width_table(genstate_t *gp) {
setvbuf(stdin, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
for (code = filter_start; code <= filter_end; code++) {
for (code = gp->filter_start; code <= gp->filter_end; code++) {
char buf[10];
char line[32];
int len, c, n, y, x;
@ -1106,7 +1091,7 @@ static int make_wcwidth_variant_table(genstate_t *gp) {
X(CS, "Common Separator") \
X(BS, "Block Separator") \
X(SS, "Segment Separator") \
X(WS, "WhiteSpace") \
X(WS, "Whitespace") \
X(AL, "Arabic Letter") \
X(NSM, "Non Spacing Mark") \
X(BN, "Boundary Neutral") \
@ -1167,15 +1152,9 @@ static int make_bidir_table(genstate_t *gp) {
char line[1024];
FILE *fp;
unsigned char *bidir_type;
int use_names = 0;
int c, lineno, pos, count, count1, count2, last_bidir;
unsigned long code, code1, code2;
if (has_arg(gp) && !strcmp(peek_arg(gp), "-a")) {
use_names = 1;
next_arg(gp);
}
if (!(fp = open_unicode_file(gp->unicode_dir, gp->unicode_version,
"UnicodeData.txt",
filename, sizeof filename)))
@ -1267,11 +1246,19 @@ static int make_bidir_table(genstate_t *gp) {
count = count1 + count2 - 1;
/* output the enum values */
fprintf(gp->fc, "\n" "enum bidir_class {\n");
#define X(e,s) fprintf(gp->fc, " %s, /* %s */\n", #e, s);
/* public type with package name prefixes */
#define X(e,s) fprintf(gp->fi, " QE_BIDIR_%s, /* %s */\n", #e, s);
fprintf(gp->fi, "enum qe_bidir_class {\n");
BIDIR_CLASSES(X);
fprintf(gp->fi, "};\n");
#undef X
/* private type with shorter aliases */
#define X(e,s) fprintf(gp->fc, " %s = QE_BIDIR_%s, /* %s */\n", #e, #e, s);
fprintf(gp->fc, "\n" "enum bidir_class {\n");
BIDIR_CLASSES(X);
fprintf(gp->fc, "};\n");
#undef X
/* output a direct table for single byte code points */
pos = 0;
@ -1279,7 +1266,7 @@ static int make_bidir_table(genstate_t *gp) {
fprintf(gp->fc, "\n" "static const unsigned char bidir_table_00[%d] = {", 256);
for (code = 0; code <= 255; code++) {
if (!(pos++ % 8)) fprintf(gp->fc, "\n ");
if (use_names) {
if (gp->use_names) {
fprintf(gp->fc, " %3s,", bidir_class_names[bidir_type[code]]);
} else {
fprintf(gp->fc, " 0x%02x,", bidir_type[code]);
@ -1289,7 +1276,7 @@ static int make_bidir_table(genstate_t *gp) {
/* output a composite table for other code points */
fprintf(gp->fc, "\n" "static const unsigned int bidir_table[%d] = {", count2);
if (use_names)
if (gp->use_names)
fprintf(gp->fc, "\n" "#define X(c,v) (((c) << 8) | (v))");
pos = 0;
last_bidir = -1;
@ -1297,7 +1284,7 @@ static int make_bidir_table(genstate_t *gp) {
if (last_bidir != bidir_type[code]) {
/* output combined table */
if (!(pos++ % 4)) fprintf(gp->fc, "\n ");
if (use_names) {
if (gp->use_names) {
fprintf(gp->fc, " X(0x%06lx, %3s),", code, bidir_class_names[bidir_type[code]]);
} else {
fprintf(gp->fc, " 0x%08lx,", (code << 8) | bidir_type[code]);
@ -1305,7 +1292,7 @@ static int make_bidir_table(genstate_t *gp) {
last_bidir = bidir_type[code];
}
}
if (use_names)
if (gp->use_names)
fprintf(gp->fc, "\n" "#undef X");
fprintf(gp->fc, "\n};\n");
#if 0
@ -1327,7 +1314,7 @@ static int make_bidir_table(genstate_t *gp) {
for (code = 0; code <= CHARCODE_MAX; code++) {
if (last_bidir != bidir_type[code]) {
if (!(pos++ % 8)) fprintf(gp->fc, "\n ");
if (use_names) {
if (gp->use_names) {
fprintf(gp->fc, " %3s,", bidir_class_names[bidir_type[code]]);
} else {
fprintf(gp->fc, " 0x%02x,", bidir_type[code]);
@ -1336,6 +1323,67 @@ static int make_bidir_table(genstate_t *gp) {
}
}
fprintf(gp->fc, "\n};\n");
#endif
/* use a binary lookup loop */
// XXX: this is inefficient and slow
// should use multi-level indirect tables
#if 00
enum qe_bidir_class qe_bidir_get_type(unsigned int ch) {
int a, b;
if (ch < 256)
return bidir_table_00[ch];
if (ch > CHARCODE_MAX)
return LTR;
a = 0;
b = countof(bidir_table) - 1;
while (a < b) {
int m = (a + b) >> 1;
if (ch < bidir_table[m] >> 8)
b = m;
else
a = m;
}
return bidir_table[a] & 0xFF;
}
/* version for test with ASCII chars */
static inline enum qe_bidir_class qe_bidir_get_type_test(unsigned int ch) {
if (ch >= 'A' && ch <= 'Z')
return QE_BIDIR_RTL;
else
return qe_bidir_get_type(ch);
}
#else
/* function implementation */
fprintf(gp->fc, "\n");
fprintf(gp->fc, "enum qe_bidir_class qe_bidir_get_type(unsigned int ch) {\n");
fprintf(gp->fc, " int a, b;\n");
fprintf(gp->fc, " if (ch < 256)\n");
fprintf(gp->fc, " return bidir_table_00[ch];\n");
fprintf(gp->fc, " if (ch > CHARCODE_MAX)\n");
fprintf(gp->fc, " return LTR;\n");
fprintf(gp->fc, " a = 0;\n");
fprintf(gp->fc, " b = countof(bidir_table) - 1;\n");
fprintf(gp->fc, " while (a < b) {\n");
fprintf(gp->fc, " int m = (a + b) >> 1;\n");
fprintf(gp->fc, " if (ch < bidir_table[m] >> 8)\n");
fprintf(gp->fc, " b = m;\n");
fprintf(gp->fc, " else\n");
fprintf(gp->fc, " a = m;\n");
fprintf(gp->fc, " }\n");
fprintf(gp->fc, " return bidir_table[a] & 0xFF;\n");
fprintf(gp->fc, "}\n");
/* function prototypes */
fprintf(gp->fi, "\n");
fprintf(gp->fi, "enum qe_bidir_class qe_bidir_get_type(unsigned int ch);\n");
fprintf(gp->fi, "\n");
fprintf(gp->fi, "/* version for test with ASCII chars */\n");
fprintf(gp->fi, "static inline enum qe_bidir_class qe_bidir_get_type_test(unsigned int ch) {\n");
fprintf(gp->fi, " if (ch >= 'A' && ch <= 'Z')\n");
fprintf(gp->fi, " return QE_BIDIR_RTL;\n");
fprintf(gp->fi, " else\n");
fprintf(gp->fi, " return qe_bidir_get_type(ch);\n");
fprintf(gp->fi, "}\n");
#endif
free(bidir_type);
return 0;
@ -1360,7 +1408,11 @@ static int usage(const char *name) {
}
int main(int argc, char *argv[]) {
genstate_t gp[1] = {{ NULL, "unidata", NULL, NULL, 0, stdout, stdout, 1, argc, argv }};
genstate_t gp[1] = {{
NULL, "unidata", NULL, NULL, 0,
stdout, stdout, 1, argc, argv,
0, 0x20, CHARCODE_MAX
}};
int tasks = 0;
#define MAKE_WCWIDTH_TABLE (1 << 0)
#define MAKE_TTY_WIDTH_TABLE (1 << 1)
@ -1386,6 +1438,10 @@ int main(int argc, char *argv[]) {
} else
if (strequal(arg, "-b")) {
tasks |= MAKE_BIDIR_TABLE;
if (has_arg(gp) && !strcmp(peek_arg(gp), "-a")) {
gp->use_names = 1;
next_arg(gp);
}
} else
if (strequal(arg, "-w")) {
tasks |= MAKE_WCWIDTH_TABLE;
@ -1395,7 +1451,12 @@ int main(int argc, char *argv[]) {
} else
if (strequal(arg, "-W")) {
tasks |= MAKE_TTY_WIDTH_TABLE;
if (has_arg(gp) && isdigit((unsigned char)*peek_arg(gp)))
gp->filter_start = strtoul(get_arg(gp), NULL, 16);
if (has_arg(gp) && isdigit((unsigned char)*peek_arg(gp)))
gp->filter_end = strtoul(get_arg(gp), NULL, 16);
} else {
fprintf(stderr, "%s: unknown option %s\n", NAME, arg);
return usage(NAME);
}
}

View file

@ -143,7 +143,7 @@ int expand_ligature(unsigned int *buf, int c)
}
/* apply all the ligature rules in logical order. Always return a
smaller buffer */
smaller or equal buffer */
static int unicode_ligature(unsigned int *buf_out,
unsigned int *pos_L_to_V,
int len)
@ -152,6 +152,7 @@ static int unicode_ligature(unsigned int *buf_out,
unsigned int *q;
const unsigned short *lig;
/* CG: C99 variable-length arrays may be too large */
// XXX: why do we need a local copy?
unsigned int buf[len];
blockcpy(buf, buf_out, len);
@ -167,7 +168,7 @@ static int unicode_ligature(unsigned int *buf_out,
goto found;
}
/* fast test to eliminate common cases */
if (i == (len - 1))
if (i == len - 1)
goto nolig;
l2 = buf[i + 1];
if (l1 <= 0x7f && l2 <= 0x7f)
@ -224,20 +225,21 @@ static int unicode_ligature(unsigned int *buf_out,
static int unicode_classify(unsigned int *buf, int len)
{
int i, mask, c;
int i, mask;
mask = 0;
for (i = 0; i < len; i++) {
c = buf[i];
unsigned int c = buf[i];
if (c <= 0x7f) /* latin1 fast handling */
continue;
mask |= UNICODE_NONASCII;
if (c >= 0x2000) /* fast test for non handled scripts */
continue;
if (c >= 0x600 && c <= 0x6ff)
mask |= UNICODE_ARABIC;
else if (c >= 0x900 && c <= 0x97f)
mask |= UNICODE_INDIC;
if (c < 0xA00) {
if ((c & ~0xff) == 0x600) /* 0600..06FF */
mask |= UNICODE_ARABIC;
else
if ((c & ~0x7f) == 0x900) /* 0900..097F */
mask |= UNICODE_INDIC;
}
}
return mask;
}
@ -265,8 +267,8 @@ static void bidi_reverse_buf(unsigned int *str, int len)
}
/* Convert a string of unicode characters to a string of glyphs. We
suppose that the font implements a minimum number of standard
ligatures chars. The string is reversed if 'reversed' is set to
assume that the font implements a minimum number of standard
ligature chars. The string is reversed if 'reversed' is set to
deal with the bidir case. 'char_to_glyph_pos' gives the index of
the first glyph associated to a given character of the source
buffer. */