From 13f0d3bec92963a7cd3d5d794cb9446cd3d140aa Mon Sep 17 00:00:00 2001 From: Charlie Gordon Date: Sat, 14 Dec 2024 20:27:25 +0100 Subject: [PATCH] Improve C-like language colorizers (#53) * add `CLANG_C_TYPES` flag for generic C types * add `CLANG_C_KEYWORDS` flag for generic C keywords * add `CLANG_T_TYPES` flag for generic `xxx_t` types * more explicit regex testing * support for `u`, `U` and `u64` encoding prefixes * improve comment detection in diff output for js and c3 * change `get_c_identifier` API for consistency * add `is_c_identifier_start()` and `is_c_identifier_part()` * move `qe_haslower` to **util.c** * update C2 language keyword and type lists * improve Pike colorizer --- LICENSE | 2 +- lang/clang.c | 533 ++++++++++++++++++++++++++++----------------------- lang/clang.h | 3 +- lang/rust.c | 4 +- qe-manual.md | 27 +-- util.c | 14 ++ util.h | 1 + 7 files changed, 331 insertions(+), 253 deletions(-) diff --git a/LICENSE b/LICENSE index 2cf3abb..1622f5f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ Quick Emacs (QEmacs) Copyright (c) 2000-2002 Fabrice Bellard -Copyright (c) 2000-2023 Charlie Gordon +Copyright (c) 2000-2024 Charlie Gordon Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/lang/clang.c b/lang/clang.c index ae328cb..67ac907 100644 --- a/lang/clang.c +++ b/lang/clang.c @@ -27,16 +27,30 @@ #include "clang.h" /* C mode options */ +#define CLANG_C_TYPES 0x00080 /* add C types to syn->types list */ +#define CLANG_C_KEYWORDS 0x00100 /* add C keywords to syn->keywords list */ #define CLANG_LEX 0x00200 #define CLANG_YACC 0x00400 -#define CLANG_REGEX 0x00800 -#define CLANG_WLITERALS 0x01000 -#define CLANG_PREPROC 0x02000 -#define CLANG_CAP_TYPE 0x04000 /* Mixed case initial cap is type */ -#define CLANG_STR3 0x08000 /* Support """strings""" */ -#define CLANG_LINECONT 0x10000 /* support \ as line continuation */ +#define CLANG_REGEX 0x00800 /* recognize / delimited regular expressions */ +#define CLANG_WLITERALS 0x01000 /* recognize L, u, U, u8 string prefix */ +#define CLANG_PREPROC 0x02000 /* colorize preprocessing directives */ +#define CLANG_CAP_TYPE 0x04000 /* Mixed case initial cap is type */ +#define CLANG_STR3 0x08000 /* Support """strings""" */ +#define CLANG_LINECONT 0x10000 /* support \ as line continuation */ #define CLANG_NEST_COMMENTS 0x20000 /* block comments are nested */ -#define CLANG_CC 0x13100 /* all C language features */ +#define CLANG_T_TYPES 0x40000 /* _t suffix indicates type identifier */ + +/* FIXME: need flags for + '@' in identifiers(start / next), + '$' in identifiers(start / next), + '-' in identifiers next + '#' as comment char + UnicodeIDStart and UnicodeIDContinue in identifiers + */ + +/* all C language features */ +#define CLANG_CC (CLANG_LINECONT | CLANG_WLITERALS | CLANG_PREPROC | \ + CLANG_C_KEYWORDS | CLANG_C_TYPES | CLANG_T_TYPES) static const char c_keywords[] = { "auto|break|case|const|continue|default|do|else|enum|extern|for|goto|" @@ -70,69 +84,84 @@ static const char c_extensions[] = { "h.in|c.in|" /* preprocessed C input (should use more generic approach) */ }; -int get_c_identifier(char *buf, int buf_size, const char32_t *p, int flavor) { +static int is_c_identifier_start(char32_t c, int flavor) { + // should accept unicode escape sequence, UnicodeIDStart + return (qe_isalpha_(c) + || c == '$' + || (c == '@' && flavor != CLANG_PIKE) + || (flavor == CLANG_RUST && c >= 128)); +} + +static int is_c_identifier_part(char32_t c, int flavor) { + // should accept unicode escape sequence, UnicodeIDContinue, ZWJ or ZWNJ + return (qe_isalnum_(c) + || (c == '-' && flavor == CLANG_CSS) + || (flavor == CLANG_RUST && c >= 128)); +} + +// FIXME: should merge into ustr_get_identifier() +int get_c_identifier(char *dest, int size, char32_t c, + const char32_t *str, int i0, int n, int flavor) +{ /*@API utils - Grab a C ASCII identifier from a char32_t buffer for a given flavor. - @argument `buf` a pointer to the destination array - @argument `buf_size` the length of the destination array in bytes - @argument `p` a valid pointer to an array of codepoints + Grab an identifier from a `char32_t` buffer for a given C flavor, + accept non-ASCII identifiers and encode in UTF-8. + @argument `dest` a pointer to the destination array + @argument `size` the length of the destination array in bytes + @argument `c` the initial code point or `0` if none + @argument `str` a valid pointer to an array of codepoints + @argument `i` the index to the next codepoint + @argument `n` the length of the codepoint array @argument `flavor` the language variant for identifier syntax @return the number of codepoints used in the source array. @note `dest` can be a null pointer if `size` is `0`. - @note non-ASCII codepoints are accepted for CLANG_RUST but are not UTF-8 encoded */ - char32_t c; - int i, j; + int pos = 0, i = i0; - i = j = 0; - c = p[i]; - if (qe_isalpha_(c) - || c == '$' - || (c == '@' && flavor != CLANG_PIKE) - || (flavor == CLANG_RUST && c >= 128)) { - for (;;) { - if (j < buf_size - 1) { - /* XXX: UTF-8 bug */ - buf[j++] = (c < 0xFF) ? c : 0xFF; - } - i++; - c = p[i]; - if (c == '-' && flavor == CLANG_CSS) - continue; - if (qe_isalnum_(c)) - continue; - if (flavor == CLANG_RUST && c >= 128) - continue; - if (c == ':' && p[i + 1] == ':' - && flavor == CLANG_CPP - && qe_isalpha_(p[i + 2])) { - if (j < buf_size - 2) { - buf[j++] = c; - buf[j++] = c; - } - i += 2; - c = p[i]; - continue; - } - break; + if (c == 0) { + if (!(i < n && is_c_identifier_start(c = str[i++], flavor))) { + if (size > 0) + *dest = '\0'; + return 0; } } - buf[j] = '\0'; - return i; -} - -static int qe_haslower(const char *str) { - /*@API utils - Check if a C string contains has ASCII lowercase letters. - @argument `str` a valid pointer to a C string - @return a boolean value, non zero if and only if the string contains - ASCII lowercase letters. - */ - while (*str) { - if (qe_islower((unsigned char)*str++)) - return 1; + for (;; i++) { + if (c < 128) { + if (pos + 1 < size) { + dest[pos++] = c; + } + } else { + char buf[6]; + int len = utf8_encode(buf, c); + if (pos + len < size) { + memcpy(dest + pos, buf, len); + pos += len; + } else { + size = pos + 1; + } + } + if (i >= n) + break; + c = str[i]; + if (!is_c_identifier_part(c, flavor)) { + /* include ugly c++ :: separator in identifier */ + if (c == ':' && str[i + 1] == ':' + && (flavor == CLANG_CPP || flavor == CLANG_C3) + && is_c_identifier_start(str[i + 2], flavor)) { + if (pos + 1 < size) + dest[pos++] = ':'; + if (pos + 1 < size) + dest[pos++] = ':'; + i += 2; + c = str[i]; + } else { + break; + } + } } - return 0; + if (pos < size) + dest[pos] = '\0'; + return i - i0; } enum { @@ -176,7 +205,7 @@ static void c_colorize_line(QEColorizeContext *cp, { int i = 0, start, i1, i2, indent, level; int style, style0, style1, type_decl, tag; - char32_t c, prev, delim; + char32_t c, delim; char kbuf[64]; int mode_flags = syn->colorize_flags; int flavor = (mode_flags & CLANG_FLAVOR); @@ -219,8 +248,8 @@ static void c_colorize_line(QEColorizeContext *cp, while (i < n) { start = i; + reswitch: c = str[i++]; - switch (c) { case '*': /* lone star at the beginning of a line in a shell buffer @@ -232,7 +261,7 @@ static void c_colorize_line(QEColorizeContext *cp, i--; goto parse_comment2; } - goto normal; + break; case '/': if (str[i] == '*') { /* C style multi-line comment */ @@ -303,52 +332,53 @@ static void c_colorize_line(QEColorizeContext *cp, SET_STYLE(sbuf, start, i, C_STYLE_COMMENT); continue; } - /* XXX: should use more context to tell regex from divide */ - prev = ' '; - for (i1 = start; i1 > indent; ) { - prev = str[--i1]; - if (!qe_isblank(prev)) - break; - } - if ((mode_flags & CLANG_REGEX) - && !qe_findchar("])", prev) - && (qe_findchar(" [({},;=<>!~^&|*/%?:", prev) - || sbuf[i1] == C_STYLE_KEYWORD - || (str[i] != ' ' && (str[i] != '=' || str[i + 1] != ' ') - && !(qe_isalnum(prev) || prev == ')')))) { - /* parse regex */ - state |= IN_C_REGEX; - delim = '/'; - parse_regex: - style = C_STYLE_REGEX; - while (i < n) { - c = str[i++]; - if (c == '\\') { - if (i < n) { - i += 1; - } - } else - if (state & IN_C_CHARCLASS) { - if (c == ']') { - state &= ~IN_C_CHARCLASS; - } - /* ECMA 5: ignore '/' inside char classes */ - } else { - if (c == '[') { - state |= IN_C_CHARCLASS; - } else - if (c == delim) { - while (qe_isalnum_(str[i])) { - i++; + if (mode_flags & CLANG_REGEX) { + /* XXX: should use more context to tell regex from divide */ + char32_t prev = ' '; + for (i1 = start; i1 > indent; ) { + prev = str[--i1]; + if (!qe_isblank(prev)) + break; + } + if (!qe_findchar("])", prev) + && (qe_findchar(" [({},;=<>!~^&|*/%?:", prev) + || sbuf[i1] == C_STYLE_KEYWORD + || (str[i] != ' ' && (str[i] != '=' || str[i + 1] != ' ') + && !(qe_isalnum(prev) || prev == ')')))) { + /* parse regex */ + state |= IN_C_REGEX; + delim = '/'; + parse_regex: + style = C_STYLE_REGEX; + while (i < n) { + c = str[i++]; + if (c == '\\') { + if (i < n) { + i += 1; + } + } else + if (state & IN_C_CHARCLASS) { + if (c == ']') { + state &= ~IN_C_CHARCLASS; + } + /* ECMA 5: ignore '/' inside char classes */ + } else { + if (c == '[') { + state |= IN_C_CHARCLASS; + } else + if (c == delim) { + while (qe_isalnum_(str[i])) { + i++; + } + state &= ~IN_C_REGEX; + style = style0; + break; } - state &= ~IN_C_REGEX; - style = style0; - break; } } + SET_STYLE(sbuf, start, i, C_STYLE_REGEX); + continue; } - SET_STYLE(sbuf, start, i, C_STYLE_REGEX); - continue; } break; case '%': @@ -364,15 +394,6 @@ static void c_colorize_line(QEColorizeContext *cp, SET_STYLE(sbuf, start, i, C_STYLE_PREPROCESS); break; } - if (mode_flags & CLANG_PREPROC) { - state |= IN_C_PREPROCESS; - style = style0 = C_STYLE_PREPROCESS; - } - if (flavor == CLANG_D) { - /* only #line is supported, but can occur anywhere */ - state |= IN_C_PREPROCESS; - style = style0 = C_STYLE_PREPROCESS; - } if (flavor == CLANG_AWK || flavor == CLANG_PHP || flavor == CLANG_LIMBO || flavor == CLANG_SQUIRREL) { goto parse_comment1; @@ -382,32 +403,38 @@ static void c_colorize_line(QEColorizeContext *cp, goto parse_regex; } if (flavor == CLANG_HAXE || flavor == CLANG_CBANG) { - i += get_c_identifier(kbuf, countof(kbuf), str + i, flavor); + i += get_c_identifier(kbuf, countof(kbuf), 0, str, i, n, flavor); // XXX: check for proper preprocessor directive? SET_STYLE(sbuf, start, i, C_STYLE_PREPROCESS); continue; } if (flavor == CLANG_PIKE) { + int klen; if (str[i] == '\"') { i++; - goto parse_string; + goto parse_string; // FIXME: accept embedded newlines } + if (str[i] == '(') { + // FIXME: parse literal strings until `#)` + } + if (str[i] == '[') { + // FIXME: parse literal strings until `#]` + } + if (str[i] == '{') { + // FIXME: parse literal strings until `#}` + } + if (ustr_match_keyword(str + i, "string", &klen)) { + /* Pike's version of #embed */ + style = C_STYLE_PREPROCESS; + i += klen; + break; + } + } + if (mode_flags & CLANG_PREPROC) { state |= IN_C_PREPROCESS; style = style0 = C_STYLE_PREPROCESS; } break; - case 'L': /* wide character and string literals */ - if (mode_flags & CLANG_WLITERALS) { - if (str[i] == '\'') { - i++; - goto parse_string_q; - } - if (str[i] == '\"') { - i++; - goto parse_string; - } - } - goto normal; // case 'r': /* XXX: D language r" wysiwyg chars " */ // case 'X': @@ -424,6 +451,7 @@ static void c_colorize_line(QEColorizeContext *cp, delim = '\''; goto string; case '`': + // FIXME: Support `operator in Pike if (flavor == CLANG_SCALA || flavor == CLANG_GMSCRIPT) { /* scala quoted identifier */ while (i < n) { @@ -464,9 +492,9 @@ static void c_colorize_line(QEColorizeContext *cp, /* ignore escape sequences and newlines */ state |= IN_C_STRING_D; // XXX: IN_RAW_STRING style1 = C_STYLE_STRING; - delim = str[i]; + delim = str[i++]; style = style1; - for (i++; i < n;) { + while (i < n) { c = str[i++]; if (c == delim) { if (str[i] == c) { @@ -588,11 +616,15 @@ static void c_colorize_line(QEColorizeContext *cp, SET_STYLE(sbuf, start, i, C_STYLE_NUMBER); continue; } - if (qe_isalpha_(c) || c == '$' || (c == '@' && flavor != CLANG_PIKE)) { - /* XXX: should support :: */ - i = start + get_c_identifier(kbuf, countof(kbuf), str + start, flavor); + if (is_c_identifier_start(c, flavor)) { + i += get_c_identifier(kbuf, countof(kbuf), c, str, i, n, flavor); + if (str[i] == '\'' || str[i] == '\"') { + /* check for encoding prefix */ + if ((mode_flags & CLANG_WLITERALS) && strfind("L|u|U|u8", kbuf)) + goto reswitch; + } if (strfind(syn->keywords, kbuf) - || ((mode_flags & CLANG_CC) && strfind(c_keywords, kbuf)) + || ((mode_flags & CLANG_C_KEYWORDS) && strfind(c_keywords, kbuf)) || ((flavor == CLANG_CSS) && str[i] == ':')) { SET_STYLE(sbuf, start, i, C_STYLE_KEYWORD); continue; @@ -610,11 +642,9 @@ static void c_colorize_line(QEColorizeContext *cp, if ((start == 0 || str[start - 1] != '.') && (!qe_findchar(".(:", str[i]) || flavor == CLANG_PIKE) && (sreg_match(syn->types, kbuf, 1) - || ((mode_flags & CLANG_CC) && strfind(c_types, kbuf)) - || (((mode_flags & CLANG_CC) || (flavor == CLANG_D)) && - strend(kbuf, "_t", NULL)) - || ((mode_flags & CLANG_CAP_TYPE) && - qe_isupper(c) && qe_haslower(kbuf)) + || ((mode_flags & CLANG_C_TYPES) && strfind(c_types, kbuf)) + || ((mode_flags & CLANG_T_TYPES) && strend(kbuf, "_t", NULL)) + || ((mode_flags & CLANG_CAP_TYPE) && qe_isupper(c) && qe_haslower(kbuf)) || (flavor == CLANG_HAXE && qe_isupper(c) && qe_haslower(kbuf) && (start == 0 || !qe_findchar("(", str[start - 1]))))) { /* if not cast, assume type declaration */ @@ -759,7 +789,7 @@ static int c_line_has_label(EditState *s, const char32_t *buf, int len, const QETermStyle *sbuf) { char kbuf[64]; - int i, j, style; + int i, style; i = cp_skip_blanks(buf, 0, len); @@ -770,10 +800,10 @@ static int c_line_has_label(EditState *s, const char32_t *buf, int len, || style == C_STYLE_PREPROCESS) return 0; - j = get_c_identifier(kbuf, countof(kbuf), buf + i, CLANG_C); + i += get_c_identifier(kbuf, countof(kbuf), 0, buf, i, len, CLANG_C); if (style == C_STYLE_KEYWORD && strfind("case|default", kbuf)) return 1; - i = cp_skip_blanks(buf, i + j, len); + i = cp_skip_blanks(buf, i, len); return (buf[i] == ':'); } @@ -1367,8 +1397,8 @@ ModeDef c_mode = { .mode_probe = c_mode_probe, .colorize_func = c_colorize_line, .colorize_flags = CLANG_C | CLANG_CC, - .keywords = c_keywords, - .types = c_types, + .keywords = "", //c_keywords, + .types = "", //c_types, .indent_func = c_indent_line, .auto_indent = 1, }; @@ -1509,24 +1539,31 @@ static ModeDef carbon_mode = { /*---------------- C2 language ----------------*/ static const char c2_keywords[] = { - // should remove C keywords: - //"extern|static|typedef|long|short|signed|unsigned|" - /* new C2 keywords */ - "module|import|as|public|local|type|func|nil|elemsof|" - /* boolean values */ - "false|true|" + // Module related + "module|import|as|public|" + // Types -> c2_types + // Type related + "auto|asm|cast|const|elemsof|enum|enum_min|enum_max|" + "false|fn|local|nil|offsetof|to_container|public|" + "sizeof|struct|template|true|type|union|volatile|" + // Control flow related + "break|case|continue|default|do|else|fallthrough|" + "for|goto|if|return|switch|sswitch|while|" + // other + "assert|static_assert" }; static const char c2_types[] = { - "bool|int8|int16|int32|int64|uint8|uint16|uint32|uint64|" - "float32|float64|" + "bool|i8|i16|i32|i64|u8|u16|u32|u64|isize|usize|f32|f64|void|" + "reg8|reg16|reg32|reg64|" + "char" // same as i8, a design error }; static ModeDef c2_mode = { .name = "C2", - .extensions = "c2|c2h|c2t", + .extensions = "c2|c2h|c2i|c2t", .colorize_func = c_colorize_line, - .colorize_flags = CLANG_C2 | CLANG_CC, + .colorize_flags = CLANG_C2 | CLANG_PREPROC | CLANG_CAP_TYPE, .keywords = c2_keywords, .types = c2_types, .indent_func = c_indent_line, @@ -1677,55 +1714,55 @@ static int is_js_identifier_part(char32_t c) { return (qe_isalnum_(c) || c == '$' || c >= 128); } +// FIXME: should merge into ustr_get_identifier() static int get_js_identifier(char *dest, int size, char32_t c, - const char32_t *str, int i, int n) + const char32_t *str, int i0, int n) { /*@API utils - Grab an identifier from a char32_t buffer, accept non-ASCII identifiers - and encode in UTF-8. + Grab an identifier from a `char32_t` buffer, + accept non-ASCII identifiers and encode in UTF-8. @argument `dest` a pointer to the destination array @argument `size` the length of the destination array in bytes @argument `c` the initial code point or `0` if none @argument `str` a valid pointer to an array of codepoints - @argument `i` the index to the next codepoint + @argument `i0` the index to the next codepoint @argument `n` the length of the codepoint array @return the number of codepoints used in the source array. @note `dest` can be a null pointer if `size` is `0`. */ - int pos = 0, j = i; + int pos = 0, i = i0; if (c == 0) { - if (!(j < n && is_js_identifier_start(c = str[j++]))) { + if (!(i < n && is_js_identifier_start(c = str[i++]))) { if (size > 0) *dest = '\0'; return 0; } } - - for (;; j++) { + for (;; i++) { if (c < 128) { - if (pos < size - 1) { + if (pos + 1 < size) { dest[pos++] = c; } } else { char buf[6]; - int i1, len = utf8_encode(buf, c); - for (i1 = 0; i1 < len; i1++) { - if (pos < size - 1) { - dest[pos++] = buf[i1]; - } + int len = utf8_encode(buf, c); + if (pos + len < size) { + memcpy(dest + pos, buf, len); + pos += len; + } else { + size = pos + 1; } } - if (j >= n) + if (i >= n) break; - c = str[j]; - // should use unicode_alpha test + c = str[i]; if (!is_js_identifier_part(c)) break; } if (pos < size) dest[pos] = '\0'; - return j - i; + return i - i0; } static void js_colorize_line(QEColorizeContext *cp, @@ -1734,7 +1771,7 @@ static void js_colorize_line(QEColorizeContext *cp, { int i = 0, start, i1, indent; int style, tag, level; - char32_t c, prev, delim; + char32_t c, delim; char kbuf[64]; int mode_flags = syn->colorize_flags; int flavor = (mode_flags & CLANG_FLAVOR); @@ -1773,12 +1810,18 @@ static void js_colorize_line(QEColorizeContext *cp, while (i < n) { start = i; c = str[i++]; - switch (c) { case '*': - if (start == indent && cp->partial_file) + /* lone star at the beginning of a line in a shell buffer + * is treated as a comment start. This improves colorization + * of diff and git output. + */ + if (start == indent && cp->partial_file + && (i == n || str[i] == ' ' || str[i] == '/')) { + i--; goto parse_comment2; - goto normal; + } + continue; case '/': if (str[i] == '*') { /* C style multi-line comment */ @@ -1815,64 +1858,68 @@ static void js_colorize_line(QEColorizeContext *cp, i = n; break; } - /* XXX: should use more context to tell regex from divide */ - prev = ' '; - for (i1 = start; i1 > indent; ) { - prev = str[--i1]; - if (!qe_isblank(prev)) - break; - } - if ((mode_flags & CLANG_REGEX) - && !qe_findchar("])", prev) - && (qe_findchar(" [({},;=<>!~^&|*/%?:", prev) - || sbuf[i1] == C_STYLE_KEYWORD - || (str[i] != ' ' && (str[i] != '=' || str[i + 1] != ' ') - && !(qe_isalnum(prev) || prev == ')')))) { - /* parse regex */ - state |= IN_C_REGEX; - delim = '/'; - parse_regex: - style = C_STYLE_REGEX; - while (i < n) { - c = str[i++]; - if (c == '\\') { - if (i < n) { - i += 1; - } - } else - if (state & IN_C_CHARCLASS) { - if (c == ']') { - state &= ~IN_C_CHARCLASS; - } - /* ECMA 5: ignore '/' inside char classes */ - } else { - if (c == '[') { - state |= IN_C_CHARCLASS; - } else - if (c == delim) { - while (qe_isalnum_(str[i])) { - i++; + if (mode_flags & CLANG_REGEX) { + /* XXX: should use more context to tell regex from divide */ + char32_t prev = ' '; + for (i1 = start; i1 > indent; ) { + prev = str[--i1]; + if (!qe_isblank(prev)) + break; + } + if (!qe_findchar("])", prev) + && (qe_findchar(" [({},;=<>!~^&|*/%?:", prev) + || sbuf[i1] == C_STYLE_KEYWORD + || (str[i] != ' ' && (str[i] != '=' || str[i + 1] != ' ') + && !(qe_isalnum(prev) || prev == ')')))) { + /* parse regex */ + state |= IN_C_REGEX; + delim = '/'; + parse_regex: + style = C_STYLE_REGEX; + while (i < n) { + c = str[i++]; + if (c == '\\') { + if (i < n) { + i += 1; + } + } else + if (state & IN_C_CHARCLASS) { + if (c == ']') { + state &= ~IN_C_CHARCLASS; + } + /* ECMA 5: ignore '/' inside char classes */ + } else { + if (c == '[') { + state |= IN_C_CHARCLASS; + } else + if (c == delim) { + while (qe_isalnum_(str[i])) { + i++; + } + state &= ~IN_C_REGEX; + break; } - state &= ~IN_C_REGEX; - break; } } + break; } - break; } continue; case '#': /* preprocessor */ - /* v8: #include */ - if (start == 0 && - (str[i] == '!' || - (flavor == CLANG_V8 && - ustrstart(str + i + 1, "include", NULL)))) - { + if (start == 0 && str[i] == '!') { /* recognize a shebang comment line */ style = C_STYLE_PREPROCESS; i = n; break; } + if (flavor == CLANG_V8 && start == 0 + && ustrstart(str + i + 1, "include", NULL)) { + /* v8: #include */ + // FIXME: handle multiline comments + style = C_STYLE_PREPROCESS; + i = n; + break; + } continue; case '@': /* annotations */ i += get_js_identifier(kbuf, countof(kbuf), c, str, i, n); @@ -1968,7 +2015,6 @@ static void js_colorize_line(QEColorizeContext *cp, tag = 0; continue; default: - normal: if (qe_isdigit(c)) { /* XXX: should parse actual number syntax */ /* decimal, binary, octal and hexadecimal literals: @@ -2546,7 +2592,8 @@ static ModeDef d_mode = { .name = "D", .extensions = "d|di", .colorize_func = c_colorize_line, - .colorize_flags = CLANG_D, + /* only #line is supported, but can occur anywhere */ + .colorize_flags = CLANG_D | CLANG_PREPROC | CLANG_T_TYPES, .keywords = d_keywords, .types = d_types, .indent_func = c_indent_line, @@ -2783,15 +2830,17 @@ static ModeDef dart_mode = { /*---------------- Pike programming language ----------------*/ static const char pike_keywords[] = { - "break|case|catch|class|constant|continue|default|do|else|enum|extern|" + "auto|break|case|catch|class|constant|continue|const|default|do|else|enum|extern|" "final|for|foreach|gauge|global|if|import|inherit|inline|" - "lambda|local|nomask|optional|predef|" - "private|protected|public|return|sscanf|static|switch|typedef|typeof|" - "while|__attribute__|__deprecated__|__func__|" + "lambda|local|optional|predef|private|protected|public|" + "return|sscanf|static|switch|throw|typedef|typeof|while|" + "_Static_assert|__async__|__attribute__|__deprecated__|" + "__experimental__|__func__|__generic__|__generator__|__weak__|" + "__unused__|__unknown__" }; static const char pike_types[] = { - "array|float|int|string|function|mapping|multiset|mixed|object|program|" + "array|float|int|string|function|mapping|mixed|multiset|object|program|" "variant|void|" }; @@ -2799,7 +2848,7 @@ static ModeDef pike_mode = { .name = "Pike", .extensions = "pike", .colorize_func = c_colorize_line, - .colorize_flags = CLANG_PIKE, + .colorize_flags = CLANG_PIKE | CLANG_PREPROC, .keywords = pike_keywords, .types = pike_types, .indent_func = c_indent_line, @@ -4401,8 +4450,8 @@ static void c3_colorize_line(QEColorizeContext *cp, while (i < n) { start = i; + reswitch: c = str[i++]; - switch (c) { case '*': if ((state & IN_C3_CONTRACTS) && str[i] == '>') { @@ -4411,9 +4460,16 @@ static void c3_colorize_line(QEColorizeContext *cp, style = C3_STYLE_PREPROCESS; break; } - if (start == indent && cp->partial_file) + /* lone star at the beginning of a line in a shell buffer + * is treated as a comment start. This improves colorization + * of diff and git output. + */ + if (start == indent && cp->partial_file + && (i == n || str[i] == ' ' || str[i] == '/')) { + i--; goto parse_comment2; - goto normal; + } + continue; case '/': if (str[i] == '*') { /* C style multi-line comment */ @@ -4524,9 +4580,8 @@ static void c3_colorize_line(QEColorizeContext *cp, case '(': case '{': tag = 0; - break; + continue; default: - normal: if (qe_isdigit(c)) { /* XXX: should parse actual number syntax */ /* decimal, binary, octal and hexadecimal literals: @@ -4543,7 +4598,11 @@ static void c3_colorize_line(QEColorizeContext *cp, i += get_js_identifier(kbuf, countof(kbuf), c, str, i, n); if (cp->state_only) continue; - + if (str[i] == '\'' || str[i] == '\"') { + /* check for encoding prefix */ + if (strfind("x|b64", kbuf)) + goto reswitch; + } /* keywords used as object property tags are regular identifiers. * `default` is always considered a keyword as the context cannot be * determined precisely by this simplistic lexical parser */ diff --git a/lang/clang.h b/lang/clang.h index d85f10a..1c2a333 100644 --- a/lang/clang.h +++ b/lang/clang.h @@ -92,7 +92,8 @@ enum { CLANG_FLAVOR = 0x7F, }; -int get_c_identifier(char *buf, int buf_size, const char32_t *p, int flavor); +int get_c_identifier(char *dest, int size, char32_t c, + const char32_t *str, int i, int n, int flavor); void c_indent_line(EditState *s, int offset0); #endif /* CLANG_H */ diff --git a/lang/rust.c b/lang/rust.c index 7f39238..a461e12 100644 --- a/lang/rust.c +++ b/lang/rust.c @@ -221,9 +221,7 @@ static void rust_colorize_line(QEColorizeContext *cp, /* identifiers match: * "[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*" */ - klen = get_c_identifier(kbuf, countof(kbuf), - str + start, CLANG_RUST); - i = start + klen; + i += get_c_identifier(kbuf, countof(kbuf), c, str, i, n, CLANG_RUST); if (str[i] == '!' && (str[i + 1] == '(' || strequal(kbuf, "macro_rules"))) { diff --git a/qe-manual.md b/qe-manual.md index 5b50576..f06da58 100644 --- a/qe-manual.md +++ b/qe-manual.md @@ -1965,15 +1965,22 @@ Convert a character to its numerical value as a hexadecimal digit Return the numerical value, or `-1` if the character is not a hex digit -### `int get_c_identifier(char *buf, int buf_size, const char32_t *p, int flavor);` +### `int get_c_identifier(char *dest, int size, char32_t c, const char32_t *str, int i0, int n, int flavor);` -Grab a C ASCII identifier from a char32_t buffer for a given flavor. +Grab an identifier from a `char32_t` buffer for a given C flavor, +accept non-ASCII identifiers and encode in UTF-8. -* argument `buf` a pointer to the destination array +* argument `dest` a pointer to the destination array -* argument `buf_size` the length of the destination array in bytes +* argument `size` the length of the destination array in bytes -* argument `p` a valid pointer to an array of codepoints +* argument `c` the initial code point or `0` if none + +* argument `str` a valid pointer to an array of codepoints + +* argument `i` the index to the next codepoint + +* argument `n` the length of the codepoint array * argument `flavor` the language variant for identifier syntax @@ -1981,8 +1988,6 @@ Return the number of codepoints used in the source array. Note: `dest` can be a null pointer if `size` is `0`. -Note: non-ASCII codepoints are accepted for CLANG_RUST but are not UTF-8 encoded - ### `int32_t get_i8(const uint8_t *tab);` Get a signed 8-bit value from a memory area @@ -2023,10 +2028,10 @@ Return the value read from memory Note: the value must be stored in memory in the native byte order -### `int get_js_identifier(char *dest, int size, char32_t c, const char32_t *str, int i, int n);` +### `int get_js_identifier(char *dest, int size, char32_t c, const char32_t *str, int i0, int n);` -Grab an identifier from a char32_t buffer, accept non-ASCII identifiers -and encode in UTF-8. +Grab an identifier from a `char32_t` buffer, +accept non-ASCII identifiers and encode in UTF-8. * argument `dest` a pointer to the destination array @@ -2036,7 +2041,7 @@ and encode in UTF-8. * argument `str` a valid pointer to an array of codepoints -* argument `i` the index to the next codepoint +* argument `i0` the index to the next codepoint * argument `n` the length of the codepoint array diff --git a/util.c b/util.c index 7212b45..5de835a 100644 --- a/util.c +++ b/util.c @@ -716,6 +716,20 @@ void qe_strtolower(char *buf, int size, const char *str) { } } +int qe_haslower(const char *str) { + /*@API utils + Check if a C string contains has ASCII lowercase letters. + @argument `str` a valid pointer to a C string + @return a boolean value, non zero if and only if the string contains + ASCII lowercase letters. + */ + while (*str) { + if (qe_islower((unsigned char)*str++)) + return 1; + } + return 0; +} + int memfind(const char *list, const char *s, int len) { /*@API utils.string Find a string fragment in a list of words separated by `|`. diff --git a/util.h b/util.h index d34da07..5c45ded 100644 --- a/util.h +++ b/util.h @@ -382,6 +382,7 @@ int qe_skip_spaces(const char **pp); int qe_strcollate(const char *s1, const char *s2); int qe_strtobool(const char *s, int def); void qe_strtolower(char *buf, int buf_size, const char *str); +int qe_haslower(const char *str); int memfind(const char *list, const char *p, int len); int strfind(const char *list, const char *s); int strxfind(const char *list, const char *s);