ref: 7bcf6ac7b612306d197d063aaf6d971ca5dd2e3f
dir: /utf8.c/
/* Basic UTF-8 manipulation routines by Jeff Bezanson placed in the public domain Fall 2005 This code is designed to provide the utilities you need to manipulate UTF-8 as an internal string encoding. These functions do not perform the error checking normally needed when handling UTF-8 data, so if you happen to be from the Unicode Consortium you will want to flay me alive. I do this because error checking can be performed at the boundaries (I/O), with these routines reserved for higher performance on data known to be valid. A UTF-8 validation routine is included. */ #include "flisp.h" static const uint32_t offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; static const char trailingBytesForUTF8[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 }; // straight from musl int u8_iswprint(Rune c) { if(c < 0xff) return ((c+1) & 0x7f) >= 0x21; if(c < 0x2028 || c-0x202a < 0xd800-0x202a || c-0xe000 < 0xfff9-0xe000) return 1; return !(c-0xfffc > Runemax-0xfffc || (c&0xfffe) == 0xfffe); } /* returns length of next utf-8 sequence */ size_t u8_seqlen(const char *s) { return trailingBytesForUTF8[(unsigned int)(uint8_t)s[0]] + 1; } /* byte offset => charnum */ size_t u8_charnum(const char *s, size_t offset) { size_t charnum = 0, i = 0; while(i < offset){ if((s[i++] & 0x80) != 0 && !isutf(s[++i]) && !isutf(s[++i])) i++; charnum++; } return charnum; } size_t u8_strwidth(const char *s) { size_t i, w; Rune r; for(i = w = 0; s[i];){ i += chartorune(&r, s+i); w += wcwidth(r); } return w; } /* next character without NUL character terminator */ Rune u8_nextmemchar(const char *s, size_t *i) { Rune ch = 0; size_t sz = 0; do{ ch <<= 6; ch += (uint8_t)s[(*i)++]; sz++; }while(!isutf(s[*i])); return ch - offsetsFromUTF8[sz-1]; } int octal_digit(char c) { return (c >= '0' && c <= '7'); } int hex_digit(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } char read_escape_control_char(char c) { switch(c){ case 'n': return '\n'; case 't': return '\t'; case 'a': return '\a'; case 'b': return '\b'; case 'e': return 0x1b; case 'f': return '\f'; case 'r': return '\r'; case 'v': return '\v'; } return c; } static inline int buf_put2c(char *buf, const char *src) { buf[0] = src[0]; buf[1] = src[1]; buf[2] = '\0'; return 2; } int u8_escape_rune(char *buf, size_t sz, Rune ch) { assert(sz > 2); if(ch >= 0x20 && ch < 0x7f){ buf[0] = ch; buf[1] = '\0'; return 1; } if(ch > 0xffff) return snprintf(buf, sz, "\\U%.8x", ch); if(ch >= Runeself) return snprintf(buf, sz, "\\u%04x", ch); switch(ch){ case '\n': return buf_put2c(buf, "\\n"); case '\t': return buf_put2c(buf, "\\t"); case '\\': return buf_put2c(buf, "\\\\"); case '\a': return buf_put2c(buf, "\\a"); case '\b': return buf_put2c(buf, "\\b"); case 0x1b: return buf_put2c(buf, "\\e"); case '\f': return buf_put2c(buf, "\\f"); case '\r': return buf_put2c(buf, "\\r"); case '\v': return buf_put2c(buf, "\\v"); } return snprintf(buf, sz, "\\x%02x", ch); } size_t u8_escape(char *buf, size_t sz, const char *src, size_t *pi, size_t end, int escape_quotes, int ascii) { size_t i = *pi, i0; Rune ch; char *start = buf; char *blim = start + sz-11; assert(sz > 11); while(i < end && buf < blim){ // sz-11: leaves room for longest escape sequence if(escape_quotes && src[i] == '"'){ buf += buf_put2c(buf, "\\\""); i++; }else if(src[i] == '\\'){ buf += buf_put2c(buf, "\\\\"); i++; }else{ i0 = i; ch = u8_nextmemchar(src, &i); if(ascii || !u8_iswprint(ch)){ buf += u8_escape_rune(buf, sz - (buf-start), ch); }else{ i = i0; do{ *buf++ = src[i++]; }while(!isutf(src[i])); } } } *buf++ = '\0'; *pi = i; return (buf-start); } char * u8_memchr(const char *s, Rune ch, size_t sz, size_t *charn) { size_t i = 0, lasti = 0; Rune c; int csz; *charn = 0; while(i < sz){ c = csz = 0; do{ c <<= 6; c += (uint8_t)s[i++]; csz++; }while(i < sz && !isutf(s[i])); c -= offsetsFromUTF8[csz-1]; if(c == ch) return (char*)&s[lasti]; lasti = i; (*charn)++; } return nil; } /* based on the valid_utf8 routine from the PCRE library by Philip Hazel length is in bytes, since without knowing whether the string is valid it's hard to know how many characters there are! */ int u8_isvalid(const char *str, int length) { const uint8_t *p, *pend = (uint8_t*)str + length; uint8_t c; int ab; for(p = (uint8_t*)str; p < pend; p++){ c = *p; if(c < 128) continue; if((c & 0xc0) != 0xc0) return 0; ab = trailingBytesForUTF8[c]; if(length < ab) return 0; length -= ab; p++; /* Check top bits in the second byte */ if((*p & 0xc0) != 0x80) return 0; /* Check for overlong sequences for each different length */ switch(ab){ /* Check for xx00 000x */ case 1: if((c & 0x3e) == 0) return 0; continue; /* We know there aren't any more bytes to check */ /* Check for 1110 0000, xx0x xxxx */ case 2: if(c == 0xe0 && (*p & 0x20) == 0) return 0; break; /* Check for 1111 0000, xx00 xxxx */ case 3: if(c == 0xf0 && (*p & 0x30) == 0) return 0; break; /* Check for 1111 1000, xx00 0xxx */ case 4: if(c == 0xf8 && (*p & 0x38) == 0) return 0; break; /* Check for leading 0xfe or 0xff and then for 1111 1100, xx00 00xx */ case 5: if(c == 0xfe || c == 0xff || (c == 0xfc && (*p & 0x3c) == 0)) return 0; break; } /* Check for valid bytes after the 2nd, if any; all must start 10 */ while(--ab > 0) if((*(++p) & 0xc0) != 0x80) return 0; } return 1; } int u8_reverse(char *dest, char *src, size_t len) { size_t si, di; dest[di = len] = '\0'; for(si = 0; si < len;){ switch((uint8_t)src[si]>>4){ case 0xf: di -= 4; dest[di+0] = src[si++]; dest[di+1] = src[si++]; dest[di+2] = src[si++]; dest[di+3] = src[si++]; break; case 0xe: di -= 3; dest[di+0] = src[si++]; dest[di+1] = src[si++]; dest[di+2] = src[si++]; break; case 0xd: case 0xc: di -= 2; dest[di+0] = src[si++]; dest[di+1] = src[si++]; break; default: di--; dest[di+0] = src[si++]; break; } } return 0; }