Improve charset detection and handling
* add charset_raw for binary files * improve charset detection for ambiguous cases * add do_show_coding_system() * add do_set_auto_coding() to (re)select the best coding system * handle BOM mark: display as \ufeff and ignore for syntax coloring
This commit is contained in:
Charlie Gordon
parent
61b3358740
commit
05ab3dc386
6 changed files with 113 additions and 11 deletions
3
buffer.c
3
buffer.c
|
|
@ -498,6 +498,9 @@ EditBuffer *eb_new(const char *name, int flags)
|
|||
|
||||
if (flags & BF_UTF8) {
|
||||
eb_set_charset(b, &charset_utf8, b->eol_type);
|
||||
} else
|
||||
if (flags & BF_RAW) {
|
||||
eb_set_charset(b, &charset_raw, EOL_UNIX);
|
||||
} else {
|
||||
/* CG: default charset should be selectable */
|
||||
eb_set_charset(b, &charset_8859_1, b->eol_type);
|
||||
|
|
|
|||
71
charset.c
71
charset.c
|
|
@ -98,6 +98,37 @@ static const unsigned char utf8_first_code_mask[7] = {
|
|||
0, 0, 0x1f, 0xf, 0x7, 0x3, 0x1,
|
||||
};
|
||||
|
||||
/********************************************************/
|
||||
/* raw */
|
||||
|
||||
static void decode_raw_init(CharsetDecodeState *s)
|
||||
{
|
||||
s->table = table_idem;
|
||||
}
|
||||
|
||||
static u8 *encode_raw(__unused__ QECharset *charset, u8 *p, int c)
|
||||
{
|
||||
if (c <= 0xff) {
|
||||
*p++ = c;
|
||||
return p;
|
||||
} else {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
QECharset charset_raw = {
|
||||
"raw",
|
||||
"binary|none",
|
||||
decode_raw_init,
|
||||
decode_8bit,
|
||||
encode_raw,
|
||||
charset_get_pos_8bit,
|
||||
charset_get_chars_8bit,
|
||||
charset_goto_char_8bit,
|
||||
charset_goto_line_8bit,
|
||||
1, 0, 0, 10, 0, 0, NULL, NULL,
|
||||
};
|
||||
|
||||
/********************************************************/
|
||||
/* 8859-1 */
|
||||
|
||||
|
|
@ -1072,7 +1103,7 @@ void detect_eol_type_32bit(const u8 *buf, int size,
|
|||
/* detect the charset. Actually only UTF8 is detected */
|
||||
QECharset *detect_charset(const u8 *buf, int size, EOLType *eol_typep)
|
||||
{
|
||||
int i, l, c, has_utf8;
|
||||
int i, l, c, has_utf8, has_binary;
|
||||
|
||||
has_utf8 = 0;
|
||||
for (i = 0; i < size;) {
|
||||
|
|
@ -1145,10 +1176,41 @@ QECharset *detect_charset(const u8 *buf, int size, EOLType *eol_typep)
|
|||
#endif
|
||||
/* Should detect iso-2220-jp upon \033$@ and \033$B, but jis
|
||||
* support is not selected in tiny build
|
||||
* XXX: should use charset probe functions.
|
||||
*/
|
||||
/* CG: should use a state variable for default charset */
|
||||
detect_eol_type_8bit(buf, size, &charset_8859_1, eol_typep);
|
||||
return &charset_8859_1;
|
||||
|
||||
has_binary = 0;
|
||||
{
|
||||
static const uint32_t magic = (1 << '\b') | (1 << '\t') | (1 << '\f') |
|
||||
(1 << '\n') | (1 << '\r') | (1 << '\033') |
|
||||
(1 << 0x0e) | (1 << 0x0f) | (1 << 0x1f);
|
||||
|
||||
for (i = 0; i < size; i++) {
|
||||
c = buf[i];
|
||||
if (c < 32 && !(magic & (1 << c)))
|
||||
has_binary += 1;
|
||||
}
|
||||
}
|
||||
if (has_binary) {
|
||||
*eol_typep = EOL_UNIX;
|
||||
return &charset_raw;
|
||||
}
|
||||
|
||||
detect_eol_type_8bit(buf, size, &charset_raw, eol_typep);
|
||||
|
||||
if (*eol_typep == EOL_DOS) {
|
||||
/* XXX: default DOS files to Latin1, should be selectable */
|
||||
return &charset_8859_1;
|
||||
}
|
||||
#ifndef CONFIG_TINY
|
||||
if (*eol_typep == EOL_MAC) {
|
||||
/* XXX: default MAC files to Mac_roman, should be selectable */
|
||||
/* XXX: should use probe functions */
|
||||
return &charset_mac_roman;
|
||||
}
|
||||
#endif
|
||||
/* XXX: should use a state variable for default charset */
|
||||
return &charset_utf8;
|
||||
}
|
||||
|
||||
/********************************************************/
|
||||
|
|
@ -1347,6 +1409,7 @@ void charset_init(void)
|
|||
for (i = 0xc0; i < 0xfe; i++)
|
||||
table_utf8[i] = ESCAPE_CHAR;
|
||||
|
||||
qe_register_charset(&charset_raw);
|
||||
qe_register_charset(&charset_8859_1);
|
||||
qe_register_charset(&charset_vt100);
|
||||
qe_register_charset(&charset_7bit);
|
||||
|
|
|
|||
|
|
@ -1159,7 +1159,7 @@ static const unsigned short table_mac_roman[128] = {
|
|||
0x00af, 0x02d8, 0x02d9, 0x02da, 0x00b8, 0x02dd, 0x02db, 0x02c7,
|
||||
};
|
||||
|
||||
static QECharset charset_mac_roman = {
|
||||
QECharset charset_mac_roman = {
|
||||
"mac-roman",
|
||||
"x-mac|mac",
|
||||
decode_8bit_init,
|
||||
|
|
|
|||
38
qe.c
38
qe.c
|
|
@ -1875,6 +1875,31 @@ QECharset *read_charset(EditState *s, const char *charset_str,
|
|||
return charset;
|
||||
}
|
||||
|
||||
void do_show_coding_system(EditState *s)
|
||||
{
|
||||
put_status(s, "Buffer charset is now %s%s", s->b->charset->name,
|
||||
s->b->eol_type == EOL_DOS ? "-dos" :
|
||||
s->b->eol_type == EOL_MAC ? "-mac" : "-unix");
|
||||
}
|
||||
|
||||
void do_set_auto_coding(EditState *s, int verbose)
|
||||
{
|
||||
u8 buf[4097];
|
||||
int buf_size;
|
||||
EditBuffer *b = s->b;
|
||||
EOLType eol_type = b->eol_type;
|
||||
QECharset *charset;
|
||||
|
||||
buf_size = eb_read(b, 0, buf, sizeof(buf));
|
||||
eol_type = b->eol_type;
|
||||
/* XXX: detect_charset returns a default charset */
|
||||
charset = detect_charset(buf, buf_size, &eol_type);
|
||||
eb_set_charset(b, charset, eol_type);
|
||||
if (verbose) {
|
||||
do_show_coding_system(s);
|
||||
}
|
||||
}
|
||||
|
||||
void do_set_buffer_file_coding_system(EditState *s, const char *charset_str)
|
||||
{
|
||||
QECharset *charset;
|
||||
|
|
@ -1885,7 +1910,7 @@ void do_set_buffer_file_coding_system(EditState *s, const char *charset_str)
|
|||
if (!charset)
|
||||
return;
|
||||
eb_set_charset(s->b, charset, eol_type);
|
||||
put_status(s, "Charset is now %s for this buffer", s->b->charset->name);
|
||||
do_show_coding_system(s);
|
||||
}
|
||||
|
||||
/* convert the charset of a buffer to another charset */
|
||||
|
|
@ -3242,7 +3267,7 @@ static int bidir_compute_attributes(TypeLink *list_tab, int max_size,
|
|||
int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size,
|
||||
int *offsetp, int line_num)
|
||||
{
|
||||
int len, l, line, col, offset;
|
||||
int len, l, line, col, offset, bom;
|
||||
int colorize_state;
|
||||
|
||||
/* invalidate cache if needed */
|
||||
|
|
@ -3272,7 +3297,8 @@ int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size,
|
|||
|
||||
for (l = s->colorize_nb_valid_lines; l <= line_num; l++) {
|
||||
len = eb_get_line(s->b, buf, buf_size, &offset);
|
||||
s->colorize_func(buf, len, &colorize_state, 1);
|
||||
bom = (len > 0 && buf[0] == 0xFEFF);
|
||||
s->colorize_func(buf + bom, len - bom, &colorize_state, 1);
|
||||
s->colorize_states[l] = colorize_state;
|
||||
}
|
||||
}
|
||||
|
|
@ -3280,7 +3306,8 @@ int generic_get_colorized_line(EditState *s, unsigned int *buf, int buf_size,
|
|||
/* compute line color */
|
||||
colorize_state = s->colorize_states[line_num];
|
||||
len = eb_get_line(s->b, buf, buf_size, offsetp);
|
||||
s->colorize_func(buf, len, &colorize_state, 0);
|
||||
bom = (len > 0 && buf[0] == 0xFEFF);
|
||||
s->colorize_func(buf + bom, len - bom, &colorize_state, 0);
|
||||
|
||||
/* XXX: if state is same as previous, minimize invalid region? */
|
||||
s->colorize_states[line_num + 1] = colorize_state;
|
||||
|
|
@ -3504,7 +3531,8 @@ int text_display(EditState *s, DisplayState *ds, int offset)
|
|||
/* currently, we cannot display these chars */
|
||||
display_printf(ds, offset0, offset, "\\U%08x", c);
|
||||
} else
|
||||
if (c >= 256 && s->qe_state->show_unicode == 1) {
|
||||
if (c >= 256 && (s->qe_state->show_unicode == 1 || c == 0xfeff)) {
|
||||
/* Display BOM as \uFEFF to make it explicit */
|
||||
display_printf(ds, offset0, offset, "\\u%04x", c);
|
||||
} else {
|
||||
display_char_bidir(ds, offset0, offset, embedding_level, c);
|
||||
|
|
|
|||
8
qe.h
8
qe.h
|
|
@ -506,10 +506,14 @@ struct QECharset {
|
|||
};
|
||||
|
||||
extern QECharset *first_charset;
|
||||
extern QECharset charset_utf8, charset_8859_1; /* predefined charsets */
|
||||
/* predefined charsets */
|
||||
extern QECharset charset_raw;
|
||||
extern QECharset charset_8859_1;
|
||||
extern QECharset charset_utf8;
|
||||
extern QECharset charset_vt100; /* used for the tty output */
|
||||
extern QECharset charset_ucs2le, charset_ucs2be;
|
||||
extern QECharset charset_ucs4le, charset_ucs4be;
|
||||
extern QECharset charset_mac_roman;
|
||||
|
||||
typedef enum EOLType {
|
||||
EOL_UNIX = 0,
|
||||
|
|
@ -1760,6 +1764,8 @@ void do_yank_pop(EditState *s);
|
|||
void do_exchange_point_and_mark(EditState *s);
|
||||
QECharset *read_charset(EditState *s, const char *charset_str,
|
||||
EOLType *eol_typep);
|
||||
void do_show_coding_system(EditState *s);
|
||||
void do_set_auto_coding(EditState *s, int verbose);
|
||||
void do_set_buffer_file_coding_system(EditState *s, const char *charset_str);
|
||||
void do_convert_buffer_file_coding_system(EditState *s,
|
||||
const char *charset_str);
|
||||
|
|
|
|||
|
|
@ -381,6 +381,8 @@ static CmdDef basic_commands[] = {
|
|||
CMD2( KEY_NONE, KEY_NONE,
|
||||
"set-mode", do_set_mode, ESs,
|
||||
"s{Set mode: }[mode]")
|
||||
CMD1( KEY_NONE, KEY_NONE,
|
||||
"set-auto-coding", do_set_auto_coding, 1)
|
||||
|
||||
/* tab & indent */
|
||||
CMD2( KEY_NONE, KEY_NONE,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue