diff --git a/buffer.c b/buffer.c index c2acba6..39f732b 100644 --- a/buffer.c +++ b/buffer.c @@ -498,6 +498,9 @@ EditBuffer *eb_new(const char *name, int flags) if (flags & BF_UTF8) { eb_set_charset(b, &charset_utf8, b->eol_type); + } else + if (flags & BF_RAW) { + eb_set_charset(b, &charset_raw, EOL_UNIX); } else { /* CG: default charset should be selectable */ eb_set_charset(b, &charset_8859_1, b->eol_type); diff --git a/charset.c b/charset.c index c11ce02..5bdd799 100644 --- a/charset.c +++ b/charset.c @@ -98,6 +98,37 @@ static const unsigned char utf8_first_code_mask[7] = { 0, 0, 0x1f, 0xf, 0x7, 0x3, 0x1, }; +/********************************************************/ +/* raw */ + +static void decode_raw_init(CharsetDecodeState *s) +{ + s->table = table_idem; +} + +static u8 *encode_raw(__unused__ QECharset *charset, u8 *p, int c) +{ + if (c <= 0xff) { + *p++ = c; + return p; + } else { + return NULL; + } +} + +QECharset charset_raw = { + "raw", + "binary|none", + decode_raw_init, + decode_8bit, + encode_raw, + charset_get_pos_8bit, + charset_get_chars_8bit, + charset_goto_char_8bit, + charset_goto_line_8bit, + 1, 0, 0, 10, 0, 0, NULL, NULL, +}; + /********************************************************/ /* 8859-1 */ @@ -1072,7 +1103,7 @@ void detect_eol_type_32bit(const u8 *buf, int size, /* detect the charset. Actually only UTF8 is detected */ QECharset *detect_charset(const u8 *buf, int size, EOLType *eol_typep) { - int i, l, c, has_utf8; + int i, l, c, has_utf8, has_binary; has_utf8 = 0; for (i = 0; i < size;) { @@ -1145,10 +1176,41 @@ QECharset *detect_charset(const u8 *buf, int size, EOLType *eol_typep) #endif /* Should detect iso-2220-jp upon \033$@ and \033$B, but jis * support is not selected in tiny build + * XXX: should use charset probe functions. */ - /* CG: should use a state variable for default charset */ - detect_eol_type_8bit(buf, size, &charset_8859_1, eol_typep); - return &charset_8859_1; + + has_binary = 0; + { + static const uint32_t magic = (1 << '\b') | (1 << '\t') | (1 << '\f') | + (1 << '\n') | (1 << '\r') | (1 << '\033') | + (1 << 0x0e) | (1 << 0x0f) | (1 << 0x1f); + + for (i = 0; i < size; i++) { + c = buf[i]; + if (c < 32 && !(magic & (1 << c))) + has_binary += 1; + } + } + if (has_binary) { + *eol_typep = EOL_UNIX; + return &charset_raw; + } + + detect_eol_type_8bit(buf, size, &charset_raw, eol_typep); + + if (*eol_typep == EOL_DOS) { + /* XXX: default DOS files to Latin1, should be selectable */ + return &charset_8859_1; + } +#ifndef CONFIG_TINY + if (*eol_typep == EOL_MAC) { + /* XXX: default MAC files to Mac_roman, should be selectable */ + /* XXX: should use probe functions */ + return &charset_mac_roman; + } +#endif + /* XXX: should use a state variable for default charset */ + return &charset_utf8; } /********************************************************/ @@ -1347,6 +1409,7 @@ void charset_init(void) for (i = 0xc0; i < 0xfe; i++) table_utf8[i] = ESCAPE_CHAR; + qe_register_charset(&charset_raw); qe_register_charset(&charset_8859_1); qe_register_charset(&charset_vt100); qe_register_charset(&charset_7bit); diff --git a/charsetmore.c b/charsetmore.c index 36bd65f..aa31f25 100644 --- a/charsetmore.c +++ b/charsetmore.c @@ -1159,7 +1159,7 @@ static const unsigned short table_mac_roman[128] = { 0x00af, 0x02d8, 0x02d9, 0x02da, 0x00b8, 0x02dd, 0x02db, 0x02c7, }; -static QECharset charset_mac_roman = { +QECharset charset_mac_roman = { "mac-roman", "x-mac|mac", decode_8bit_init, diff --git a/qe.c b/qe.c index 08f06e6..509696e 100644 --- a/qe.c +++ b/qe.c @@ -1875,6 +1875,31 @@ QECharset *read_charset(EditState *s, const char *charset_str, return charset; } +void do_show_coding_system(EditState *s) +{ + put_status(s, "Buffer charset is now %s%s", s->b->charset->name, + s->b->eol_type == EOL_DOS ? "-dos" : + s->b->eol_type == EOL_MAC ? "-mac" : "-unix"); +} + +void do_set_auto_coding(EditState *s, int verbose) +{ + u8 buf[4097]; + int buf_size; + EditBuffer *b = s->b; + EOLType eol_type = b->eol_type; + QECharset *charset; + + buf_size = eb_read(b, 0, buf, sizeof(buf)); + eol_type = b->eol_type; + /* XXX: detect_charset returns a default charset */ + charset = detect_charset(buf, buf_size, &eol_type); + eb_set_charset(b, charset, eol_type); + if (verbose) { + do_show_coding_system(s); + } +} + void do_set_buffer_file_coding_system(EditState *s, const char *charset_str) { QECharset *charset; @@ -1885,7 +1910,7 @@ void do_set_buffer_file_coding_system(EditState *s, const char *charset_str) if (!charset) return; eb_set_charset(s->b, charset, eol_type); - put_status(s, "Charset is now %s for this buffer", s->b->charset->name); + do_show_coding_system(s); } /* convert the charset of a buffer to another charset */ @@ -3242,7 +3267,7 @@ static int bidir_compute_attributes(TypeLink *list_tab, int max_size, int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size, int *offsetp, int line_num) { - int len, l, line, col, offset; + int len, l, line, col, offset, bom; int colorize_state; /* invalidate cache if needed */ @@ -3272,7 +3297,8 @@ int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size, for (l = s->colorize_nb_valid_lines; l <= line_num; l++) { len = eb_get_line(s->b, buf, buf_size, &offset); - s->colorize_func(buf, len, &colorize_state, 1); + bom = (len > 0 && buf[0] == 0xFEFF); + s->colorize_func(buf + bom, len - bom, &colorize_state, 1); s->colorize_states[l] = colorize_state; } } @@ -3280,7 +3306,8 @@ int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size, /* compute line color */ colorize_state = s->colorize_states[line_num]; len = eb_get_line(s->b, buf, buf_size, offsetp); - s->colorize_func(buf, len, &colorize_state, 0); + bom = (len > 0 && buf[0] == 0xFEFF); + s->colorize_func(buf + bom, len - bom, &colorize_state, 0); /* XXX: if state is same as previous, minimize invalid region? */ s->colorize_states[line_num + 1] = colorize_state; @@ -3504,7 +3531,8 @@ int text_display(EditState *s, DisplayState *ds, int offset) /* currently, we cannot display these chars */ display_printf(ds, offset0, offset, "\\U%08x", c); } else - if (c >= 256 && s->qe_state->show_unicode == 1) { + if (c >= 256 && (s->qe_state->show_unicode == 1 || c == 0xfeff)) { + /* Display BOM as \uFEFF to make it explicit */ display_printf(ds, offset0, offset, "\\u%04x", c); } else { display_char_bidir(ds, offset0, offset, embedding_level, c); diff --git a/qe.h b/qe.h index ff7ea8f..c8fa7ed 100644 --- a/qe.h +++ b/qe.h @@ -506,10 +506,14 @@ struct QECharset { }; extern QECharset *first_charset; -extern QECharset charset_utf8, charset_8859_1; /* predefined charsets */ +/* predefined charsets */ +extern QECharset charset_raw; +extern QECharset charset_8859_1; +extern QECharset charset_utf8; extern QECharset charset_vt100; /* used for the tty output */ extern QECharset charset_ucs2le, charset_ucs2be; extern QECharset charset_ucs4le, charset_ucs4be; +extern QECharset charset_mac_roman; typedef enum EOLType { EOL_UNIX = 0, @@ -1760,6 +1764,8 @@ void do_yank_pop(EditState *s); void do_exchange_point_and_mark(EditState *s); QECharset *read_charset(EditState *s, const char *charset_str, EOLType *eol_typep); +void do_show_coding_system(EditState *s); +void do_set_auto_coding(EditState *s, int verbose); void do_set_buffer_file_coding_system(EditState *s, const char *charset_str); void do_convert_buffer_file_coding_system(EditState *s, const char *charset_str); diff --git a/qeconfig.h b/qeconfig.h index 1b62fd1..a56a3e4 100644 --- a/qeconfig.h +++ b/qeconfig.h @@ -381,6 +381,8 @@ static CmdDef basic_commands[] = { CMD2( KEY_NONE, KEY_NONE, "set-mode", do_set_mode, ESs, "s{Set mode: }[mode]") + CMD1( KEY_NONE, KEY_NONE, + "set-auto-coding", do_set_auto_coding, 1) /* tab & indent */ CMD2( KEY_NONE, KEY_NONE,