Ruby 4.1.0dev (2026-11-17 revision 373d8c72a360f0d6c3be9cd13d1f0c27360e5d90)
encoding.c
1#include "prism/internal/encoding.h"
2
4#include "prism/internal/assert.h"
5#include "prism/internal/strncasecmp.h"
6
7typedef uint32_t pm_unicode_codepoint_t;
8
23#define UNICODE_CODEPOINTS_MAXIMUM 0x1F200
24
25static const uint8_t pm_unicode_root[498] = {
26 /* U+00000 */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
27 /* U+01000 */ 0x07, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x0A, 0x0B,
28 /* U+02000 */ 0x00, 0x0C, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00,
29 /* U+03000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
30 /* U+04000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
31 /* U+05000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
32 /* U+06000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
33 /* U+07000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
34 /* U+08000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
35 /* U+09000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
36 /* U+0A000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0F, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
37 /* U+0B000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
38 /* U+0C000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
39 /* U+0D000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
40 /* U+0E000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
41 /* U+0F000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11,
42 /* U+10000 */ 0x00, 0x00, 0x00, 0x00, 0x12, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x15, 0x00, 0x00,
43 /* U+11000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
44 /* U+12000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
45 /* U+13000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
46 /* U+14000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
47 /* U+15000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
48 /* U+16000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00,
49 /* U+17000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
50 /* U+18000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
51 /* U+19000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
52 /* U+1A000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
53 /* U+1B000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
54 /* U+1C000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
55 /* U+1D000 */ 0x00, 0x00, 0x00, 0x00, 0x18, 0x19, 0x1A, 0x1B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
56 /* U+1E000 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
57 /* U+1F000 */ 0x00, 0x1D,
58};
59
60static const uint8_t pm_unicode_middle[30][32] = {
61 [0x00] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
62 [0x01] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x01, 0x01, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x0A, 0x0A, 0x00, 0x00, 0x00, 0x00 },
63 [0x02] = { 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x0B, 0x0B, 0x0C, 0x02, 0x02, 0x02, 0x02, 0x02, 0x1E, 0x12, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x13, 0x26, 0x0B, 0x27, 0x02, 0x02, 0x12, 0x02 },
64 [0x03] = { 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x10, 0x28, 0x29, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
65 [0x04] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x15, 0x16, 0x2A, 0x08, 0x01, 0x2B, 0x0D, 0x00, 0x00, 0x00, 0x15, 0x2C, 0x02, 0x02, 0x02, 0x2D, 0x2E },
66 [0x05] = { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x0E, 0x0C, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x2F, 0x30, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02 },
67 [0x06] = { 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x08, 0x01, 0x01, 0x01, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
68 [0x07] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x31, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
69 [0x08] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x00 },
70 [0x09] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32, 0x01, 0x01, 0x01, 0x01, 0x01, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
71 [0x0A] = { 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x18, 0x16, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02 },
72 [0x0B] = { 0x00, 0x01, 0x00, 0x04, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x0B, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x05, 0x00, 0x05, 0x00, 0x0D, 0x00, 0x05, 0x00, 0x05 },
73 [0x0C] = { 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x0D, 0x0F, 0x11, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
74 [0x0D] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0F, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
75 [0x0E] = { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, 0x3B, 0x3C, 0x0F, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x10, 0x19, 0x1A, 0x00 },
76 [0x0F] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x18, 0x00, 0x00, 0x02, 0x02, 0x02, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
77 [0x10] = { 0x00, 0x00, 0x00, 0x00, 0x0C, 0x02, 0x0C, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x00, 0x3D, 0x02, 0x19, 0x14, 0x02, 0x02, 0x3E, 0x1B, 0x02, 0x3F, 0x40, 0x41, 0x42, 0x00, 0x00, 0x11, 0x00 },
78 [0x11] = { 0x00, 0x00, 0x00, 0x00, 0x08, 0x01, 0x01, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
79 [0x12] = { 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x0D, 0x00, 0x00, 0x00, 0x00, 0x00 },
80 [0x13] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x1C, 0x01, 0x1C, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
81 [0x14] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
82 [0x15] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x06, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
83 [0x16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
84 [0x17] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x06, 0x06, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
85 [0x18] = { 0x01, 0x01, 0x01, 0x03, 0x00, 0x00, 0x07, 0x01, 0x01, 0x04, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x03, 0x00, 0x00, 0x46, 0x47, 0x48, 0x04, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x03, 0x00, 0x00 },
86 [0x19] = { 0x13, 0x17, 0x49, 0x05, 0x00, 0x00, 0x00, 0x4A, 0x1B, 0x1D, 0x0E, 0x00, 0x00, 0x07, 0x01, 0x01, 0x04, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x03, 0x00, 0x00, 0x07, 0x01, 0x01, 0x04, 0x00, 0x00 },
87 [0x1A] = { 0x00, 0x01, 0x01, 0x01, 0x03, 0x00, 0x00, 0x07, 0x01, 0x01, 0x04, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x03, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x0E, 0x00, 0x00, 0x00, 0x1D, 0x01, 0x01, 0x09 },
88 [0x1B] = { 0x00, 0x00, 0x00, 0x07, 0x01, 0x01, 0x05, 0x00, 0x00, 0x00, 0x0F, 0x01, 0x01, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x0E, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
89 [0x1C] = { 0x01, 0x01, 0x01, 0x01, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
90 [0x1D] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x01, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
91};
92
99#define PM_UNICODE_TABLES_17_UPDATE PM_OPTIONS_VERSION_CRUBY_4_0
100
107#define PM_UNICODE_15_NOT_UPPERCASE_BIT (1 << 3)
108
109/* Not uppercase. */
110#define N 0
111
112/* An uppercase or titlecase letter. */
113#define U (PRISM_ENCODING_UPPERCASE_BIT)
114
115/* An uppercase or titlecase letter that is not one in Unicode 15.0.0. */
116#define V (PRISM_ENCODING_UPPERCASE_BIT | PM_UNICODE_15_NOT_UPPERCASE_BIT)
117
118static const uint8_t pm_unicode_leaf[75][8] = {
119 [0x00] = { N, N, N, N, N, N, N, N },
120 [0x01] = { U, U, U, U, U, U, U, U },
121 [0x02] = { U, N, U, N, U, N, U, N },
122 [0x03] = { U, U, N, N, N, N, N, N },
123 [0x04] = { U, U, U, U, U, U, N, N },
124 [0x05] = { U, U, U, U, U, N, N, N },
125 [0x06] = { V, V, V, V, V, V, V, V },
126 [0x07] = { N, N, N, N, U, U, U, U },
127 [0x08] = { N, U, U, U, U, U, U, U },
128 [0x09] = { U, U, U, N, N, N, N, N },
129 [0x0A] = { U, U, U, U, U, U, U, N },
130 [0x0B] = { N, U, N, U, N, U, N, U },
131 [0x0C] = { N, N, U, N, U, N, U, N },
132 [0x0D] = { U, U, U, U, N, N, N, N },
133 [0x0E] = { U, N, N, N, N, N, N, N },
134 [0x0F] = { N, N, N, N, N, N, U, U },
135 [0x10] = { U, N, U, N, N, N, N, N },
136 [0x11] = { N, N, N, N, N, U, N, N },
137 [0x12] = { N, U, U, N, U, N, U, U },
138 [0x13] = { N, N, N, N, U, U, N, U },
139 [0x14] = { U, N, U, N, N, N, U, N },
140 [0x15] = { N, N, N, N, N, N, N, U },
141 [0x16] = { N, N, N, N, N, N, U, N },
142 [0x17] = { U, U, U, N, N, U, U, U },
143 [0x18] = { U, N, U, N, U, N, N, N },
144 [0x19] = { N, N, N, U, N, U, N, N },
145 [0x1A] = { N, N, U, N, N, N, N, N },
146 [0x1B] = { U, U, U, U, U, N, U, N },
147 [0x1C] = { U, U, U, N, U, U, U, U },
148 [0x1D] = { N, N, U, U, U, U, U, U },
149 [0x1E] = { U, U, N, U, N, U, N, N },
150 [0x1F] = { N, U, U, U, N, N, U, U },
151 [0x20] = { U, U, N, U, U, N, U, U },
152 [0x21] = { U, N, N, N, U, U, N, U },
153 [0x22] = { U, N, U, N, U, N, U, U },
154 [0x23] = { N, U, N, N, U, N, U, U },
155 [0x24] = { N, U, U, U, N, U, N, U },
156 [0x25] = { U, N, N, N, U, N, N, N },
157 [0x26] = { U, N, U, U, N, U, N, U },
158 [0x27] = { N, U, N, U, N, N, U, N },
159 [0x28] = { N, N, U, U, N, U, U, N },
160 [0x29] = { N, U, N, U, U, U, U, N },
161 [0x2A] = { U, U, U, N, U, N, U, U },
162 [0x2B] = { U, U, N, U, U, U, U, U },
163 [0x2C] = { N, N, U, U, U, N, N, N },
164 [0x2D] = { N, N, N, N, U, N, N, U },
165 [0x2E] = { N, U, U, N, N, U, U, U },
166 [0x2F] = { U, U, N, U, N, U, N, U },
167 [0x30] = { N, U, N, U, N, U, N, N },
168 [0x31] = { U, U, U, U, U, U, N, U },
169 [0x32] = { N, V, N, N, N, N, N, N },
170 [0x33] = { N, N, U, N, N, N, N, U },
171 [0x34] = { N, N, N, U, U, U, N, N },
172 [0x35] = { U, U, U, N, N, U, N, N },
173 [0x36] = { N, U, U, U, U, U, N, N },
174 [0x37] = { N, N, N, N, U, N, U, N },
175 [0x38] = { U, N, U, U, U, U, N, N },
176 [0x39] = { N, N, N, U, N, N, N, N },
177 [0x3A] = { U, N, U, U, U, N, N, U },
178 [0x3B] = { N, U, N, U, N, U, U, U },
179 [0x3C] = { U, N, U, N, N, U, N, N },
180 [0x3D] = { N, U, N, U, N, U, U, N },
181 [0x3E] = { U, N, U, U, U, U, U, N },
182 [0x3F] = { U, N, U, N, U, U, U, U },
183 [0x40] = { N, U, N, V, V, N, V, N },
184 [0x41] = { U, N, V, N, V, N, U, N },
185 [0x42] = { U, N, V, N, V, N, N, N },
186 [0x43] = { U, U, U, N, U, U, N, N },
187 [0x44] = { V, V, V, V, V, V, N, N },
188 [0x45] = { V, N, N, N, N, N, N, N },
189 [0x46] = { N, N, N, N, U, N, U, U },
190 [0x47] = { N, N, U, N, N, U, U, N },
191 [0x48] = { N, U, U, U, U, N, U, U },
192 [0x49] = { U, U, U, U, U, N, U, U },
193 [0x4A] = { U, U, N, U, U, U, U, N },
194};
195
196#undef N
197#undef U
198#undef V
199
206const uint8_t pm_encoding_ascii_table[256] = {
207/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
208 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
209 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
210 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
211 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, /* 3x */
212 0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, /* 4x */
213 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, 0, 0, /* 5x */
214 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* 6x */
215 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, /* 7x */
216 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8x */
217 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9x */
218 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Ax */
219 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
220 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Cx */
221 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Dx */
222 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Ex */
223 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */
224};
225
229static uint8_t
230pm_unicode_codepoint_properties(pm_unicode_codepoint_t codepoint) {
231 if (codepoint >= UNICODE_CODEPOINTS_MAXIMUM) return 0;
232
233 uint8_t middle = pm_unicode_root[codepoint >> 8];
234 uint8_t leaf = pm_unicode_middle[middle][(codepoint >> 3) & 0x1F];
235 return pm_unicode_leaf[leaf][codepoint & 0x7];
236}
237
242static bool
243pm_unicode_uppercase_p(pm_unicode_codepoint_t codepoint, pm_options_version_t version) {
244 uint8_t properties = pm_unicode_codepoint_properties(codepoint);
245 bool uppercase = (properties & PRISM_ENCODING_UPPERCASE_BIT) != 0;
246
247 /* The tables say if it is an uppercase codepoint in the latest version of
248 * Unicode that Ruby uses. If we are parsing as an older version, then we
249 * need to exclude more recently added codepoint ranges. */
250 if (PRISM_UNLIKELY(version < PM_UNICODE_TABLES_17_UPDATE && (properties & PM_UNICODE_15_NOT_UPPERCASE_BIT))) {
251 uppercase = false;
252 }
253
254 return uppercase;
255}
256
280static const uint8_t pm_utf_8_dfa[] = {
281 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
282 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
283 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
284 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
285 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
286 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
287 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
288 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
289 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
290 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
291 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
292 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
293 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
294 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
295};
296
302static pm_unicode_codepoint_t
303pm_utf_8_codepoint(const uint8_t *b, ptrdiff_t n, size_t *width) {
304 PRISM_ASSERT(n >= 0);
305
306 size_t maximum = (n > 4) ? 4 : ((size_t) n);
307 uint32_t codepoint;
308 uint32_t state = 0;
309
310 for (size_t index = 0; index < maximum; index++) {
311 uint32_t byte = b[index];
312 uint32_t type = pm_utf_8_dfa[byte];
313
314 codepoint = (state != 0) ?
315 (byte & 0x3fu) | (codepoint << 6) :
316 (0xffu >> type) & (byte);
317
318 state = pm_utf_8_dfa[256 + (state * 16) + type];
319 if (state == 0) {
320 *width = index + 1;
321 return (pm_unicode_codepoint_t) codepoint;
322 }
323 }
324
325 *width = 0;
326 return 0;
327}
328
332size_t
333pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n) {
334 PRISM_ASSERT(n >= 0);
335
336 size_t maximum = (n > 4) ? 4 : ((size_t) n);
337 uint32_t state = 0;
338
339 for (size_t index = 0; index < maximum; index++) {
340 state = pm_utf_8_dfa[256 + (state * 16) + pm_utf_8_dfa[b[index]]];
341 if (state == 0) return index + 1;
342 }
343
344 return 0;
345}
346
351bool
352pm_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n, pm_options_version_t version) {
353 if (n == 0) {
354 return 0;
355 }
356
357 if (*b < 0x80) {
358 return pm_encoding_ascii_upper_p(*b);
359 }
360
361 size_t width;
362 pm_unicode_codepoint_t codepoint = pm_utf_8_codepoint(b, n, &width);
363
364 return pm_unicode_uppercase_p(codepoint, version);
365}
366
367#ifndef PRISM_ENCODING_EXCLUDE_FULL
368
373static pm_unicode_codepoint_t
374pm_cesu_8_codepoint(const uint8_t *b, size_t width) {
375 switch (width) {
376 case 1:
377 return (pm_unicode_codepoint_t) b[0];
378 case 2:
379 /* 110xxxxx 10xxxxxx */
380 return (pm_unicode_codepoint_t) (((b[0] & 0x1F) << 6) | (b[1] & 0x3F));
381 case 3:
382 /* 1110xxxx 10xxxxxx 10xxxxxx */
383 return (pm_unicode_codepoint_t) (((b[0] & 0xF) << 12) | ((b[1] & 0x3F) << 6) | (b[2] & 0x3F));
384 default:
385 /* 11101101 1010xxxx 10xxxxxx 11101101 1011xxxx 10xxxxxx */
386 return (pm_unicode_codepoint_t) (0x10000 + (((b[1] & 0xF) << 16) | ((b[2] & 0x3F) << 10) | ((b[4] & 0xF) << 6) | (b[5] & 0x3F)));
387 }
388}
389
390#endif
391
392#undef UNICODE_CODEPOINTS_MAXIMUM
393
394#ifndef PRISM_ENCODING_EXCLUDE_FULL
395
400static const uint8_t pm_encoding_iso_8859_1_uppercase[16] = {
401 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80 */
402 0xFF, 0xFF, 0x7F, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
403};
404
408static const uint8_t pm_encoding_iso_8859_2_uppercase[16] = {
409 0x00, 0x00, 0x00, 0x00, 0x6A, 0xDE, 0x00, 0x00, /* 0x80 */
410 0xFF, 0xFF, 0x7F, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
411};
412
416static const uint8_t pm_encoding_iso_8859_3_uppercase[16] = {
417 0x00, 0x00, 0x00, 0x00, 0x42, 0x9E, 0x00, 0x00, /* 0x80 */
418 0xF7, 0xFF, 0x7E, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
419};
420
424static const uint8_t pm_encoding_iso_8859_4_uppercase[16] = {
425 0x00, 0x00, 0x00, 0x00, 0x6A, 0x5E, 0x00, 0x20, /* 0x80 */
426 0xFF, 0xFF, 0x7F, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
427};
428
432static const uint8_t pm_encoding_iso_8859_5_uppercase[16] = {
433 0x00, 0x00, 0x00, 0x00, 0xFE, 0xDF, 0xFF, 0xFF, /* 0x80 */
434 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
435};
436
440static const uint8_t pm_encoding_iso_8859_7_uppercase[16] = {
441 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0xD7, /* 0x80 */
442 0xFE, 0xFF, 0xFB, 0x0F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
443};
444
448static const uint8_t pm_encoding_iso_8859_10_uppercase[16] = {
449 0x00, 0x00, 0x00, 0x00, 0x7E, 0xDF, 0x00, 0x00, /* 0x80 */
450 0xFF, 0xFF, 0xFF, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
451};
452
457static const uint8_t pm_encoding_iso_8859_13_uppercase[16] = {
458 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, /* 0x80 */
459 0xFF, 0xFF, 0x7F, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
460};
461
465static const uint8_t pm_encoding_iso_8859_14_uppercase[16] = {
466 0x00, 0x00, 0x00, 0x00, 0x52, 0x95, 0x95, 0x28, /* 0x80 */
467 0xFF, 0xFF, 0xFF, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
468};
469
473static const uint8_t pm_encoding_iso_8859_15_uppercase[16] = {
474 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x10, 0x50, /* 0x80 */
475 0xFF, 0xFF, 0x7F, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
476};
477
481static const uint8_t pm_encoding_iso_8859_16_uppercase[16] = {
482 0x00, 0x00, 0x00, 0x00, 0x5A, 0x94, 0x14, 0x50, /* 0x80 */
483 0xFF, 0xFF, 0xFF, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
484};
485
489static const uint8_t pm_encoding_koi8_r_uppercase[16] = {
490 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, /* 0x80 */
491 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, /* 0xC0 */
492};
493
497static const uint8_t pm_encoding_koi8_u_uppercase[16] = {
498 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0x20, /* 0x80 */
499 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, /* 0xC0 */
500};
501
505static const uint8_t pm_encoding_windows_1250_uppercase[16] = {
506 0x00, 0xF4, 0x00, 0x00, 0x28, 0x84, 0x00, 0x10, /* 0x80 */
507 0xFF, 0xFF, 0xFF, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
508};
509
513static const uint8_t pm_encoding_windows_1251_uppercase[16] = {
514 0x03, 0xF4, 0x00, 0x00, 0x2A, 0x85, 0x04, 0x20, /* 0x80 */
515 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
516};
517
521static const uint8_t pm_encoding_windows_1252_uppercase[16] = {
522 0x00, 0x54, 0x00, 0x80, 0x00, 0x80, 0x00, 0x00, /* 0x80 */
523 0xFF, 0xFF, 0xFF, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
524};
525
529static const uint8_t pm_encoding_windows_1253_uppercase[16] = {
530 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x40, 0xD7, /* 0x80 */
531 0xFE, 0xFF, 0xFB, 0x0F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
532};
533
537static const uint8_t pm_encoding_windows_1254_uppercase[16] = {
538 0x00, 0x14, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, /* 0x80 */
539 0xFF, 0xFF, 0x7F, 0x7F, 0x00, 0x00, 0x00, 0x00, /* 0xC0 */
540};
541
542#endif
543
544/*
545 * Each multibyte encoding other than UTF-8 is ASCII below 0x80, and its other
546 * characters take a list of forms. A form is the number of bytes in its
547 * characters, their PRISM_ENCODING_*_BIT properties, and the inclusive range of
548 * bytes at each position. Bytes that match a form are a character. Forms may
549 * overlap, in which case a character has the properties of every form that it
550 * matches. No character is the start of a longer one, and every form with
551 * properties lies within the forms without them, so that the first form that
552 * answers a question about a character answers it for all of them.
553 *
554 * Each list is a macro that passes its forms to X_1 through X_6 by the number
555 * of bytes in them, where X is the name of a family of macros below that turns
556 * each form into a check. The checks run in the order of the list.
557 */
558
564#define PM_ENCODING_BYTE_IN(byte_, start_, end_) \
565 ((uint8_t) ((byte_) - (start_)) <= (uint8_t) ((end_) - (start_)))
566
567/* Return the width of the character at b if it takes the given form. */
568#define PM_ENCODING_WIDTH_1(properties_, s0_, e0_) \
569 if (n >= 1 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_)) return 1;
570#define PM_ENCODING_WIDTH_2(properties_, s0_, e0_, s1_, e1_) \
571 if (n >= 2 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_) && PM_ENCODING_BYTE_IN(b[1], s1_, e1_)) return 2;
572#define PM_ENCODING_WIDTH_3(properties_, s0_, e0_, s1_, e1_, s2_, e2_) \
573 if (n >= 3 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_) && PM_ENCODING_BYTE_IN(b[1], s1_, e1_) && PM_ENCODING_BYTE_IN(b[2], s2_, e2_)) return 3;
574#define PM_ENCODING_WIDTH_4(properties_, s0_, e0_, s1_, e1_, s2_, e2_, s3_, e3_) \
575 if (n >= 4 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_) && PM_ENCODING_BYTE_IN(b[1], s1_, e1_) && PM_ENCODING_BYTE_IN(b[2], s2_, e2_) && PM_ENCODING_BYTE_IN(b[3], s3_, e3_)) return 4;
576#define PM_ENCODING_WIDTH_6(properties_, s0_, e0_, s1_, e1_, s2_, e2_, s3_, e3_, s4_, e4_, s5_, e5_) \
577 if (n >= 6 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_) && PM_ENCODING_BYTE_IN(b[1], s1_, e1_) && PM_ENCODING_BYTE_IN(b[2], s2_, e2_) && PM_ENCODING_BYTE_IN(b[3], s3_, e3_) && PM_ENCODING_BYTE_IN(b[4], s4_, e4_) && PM_ENCODING_BYTE_IN(b[5], s5_, e5_)) return 6;
578
579/*
580 * Return true if the character at b takes the given form and the form is
581 * uppercase. Forms that are not uppercase never answer, so their checks fold
582 * away.
583 */
584#define PM_ENCODING_UPPERCASE_1(properties_, s0_, e0_) \
585 if (((properties_) & PRISM_ENCODING_UPPERCASE_BIT) && n >= 1 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_)) return true;
586#define PM_ENCODING_UPPERCASE_2(properties_, s0_, e0_, s1_, e1_) \
587 if (((properties_) & PRISM_ENCODING_UPPERCASE_BIT) && n >= 2 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_) && PM_ENCODING_BYTE_IN(b[1], s1_, e1_)) return true;
588#define PM_ENCODING_UPPERCASE_3(properties_, s0_, e0_, s1_, e1_, s2_, e2_) \
589 if (((properties_) & PRISM_ENCODING_UPPERCASE_BIT) && n >= 3 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_) && PM_ENCODING_BYTE_IN(b[1], s1_, e1_) && PM_ENCODING_BYTE_IN(b[2], s2_, e2_)) return true;
590#define PM_ENCODING_UPPERCASE_4(properties_, s0_, e0_, s1_, e1_, s2_, e2_, s3_, e3_) \
591 if (((properties_) & PRISM_ENCODING_UPPERCASE_BIT) && n >= 4 && PM_ENCODING_BYTE_IN(b[0], s0_, e0_) && PM_ENCODING_BYTE_IN(b[1], s1_, e1_) && PM_ENCODING_BYTE_IN(b[2], s2_, e2_) && PM_ENCODING_BYTE_IN(b[3], s3_, e3_)) return true;
592
593/*
594 * The body of the function that returns the width of the character at b in an
595 * encoding whose characters beyond ASCII take the given forms, or 0 if the
596 * bytes do not form one.
597 */
598#define PM_ENCODING_FORMS_CHAR_WIDTH(forms_) \
599 if (n <= 0) return 0; \
600 if (b[0] < 0x80) return 1; \
601 forms_(PM_ENCODING_WIDTH) \
602 return 0;
603
604/*
605 * The body of the function that returns true if the character at b begins a
606 * constant name in an encoding whose characters beyond ASCII take the given
607 * forms.
608 */
609#define PM_ENCODING_FORMS_ISUPPER_CHAR(forms_) \
610 if (n <= 0) return false; \
611 if (b[0] < 0x80) return pm_encoding_ascii_upper_p(b[0]); \
612 forms_(PM_ENCODING_UPPERCASE) \
613 return false;
614
619#define PM_ENCODING_EUC_JP_FORMS(X) \
620 X##_2(0, 0x8E, 0x8E, 0xA1, 0xFE) \
621 X##_2(0, 0xA1, 0xFE, 0xA1, 0xFE) \
622 X##_3(0, 0x8F, 0x8F, 0xA1, 0xFE, 0xA1, 0xFE) \
623 X##_2(PRISM_ENCODING_UPPERCASE_BIT, 0xA3, 0xA3, 0xC1, 0xDA) \
624 X##_2(PRISM_ENCODING_UPPERCASE_BIT, 0xA6, 0xA6, 0xA1, 0xB8) \
625 X##_2(PRISM_ENCODING_UPPERCASE_BIT, 0xA7, 0xA7, 0xA1, 0xC1)
626
630static size_t
631pm_encoding_euc_jp_char_width(const uint8_t *b, ptrdiff_t n) {
632 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_EUC_JP_FORMS)
633}
634
638static bool
639pm_encoding_euc_jp_isupper_char(const uint8_t *b, ptrdiff_t n) {
640 PM_ENCODING_FORMS_ISUPPER_CHAR(PM_ENCODING_EUC_JP_FORMS)
641}
642
648#define PM_ENCODING_SHIFT_JIS_FORMS(X) \
649 X##_1(0, 0xA1, 0xDF) \
650 X##_2(0, 0x81, 0x9F, 0x40, 0x7E) \
651 X##_2(0, 0x81, 0x9F, 0x80, 0xFC) \
652 X##_2(0, 0xE0, 0xFC, 0x40, 0x7E) \
653 X##_2(0, 0xE0, 0xFC, 0x80, 0xFC) \
654 X##_2(PRISM_ENCODING_UPPERCASE_BIT, 0x82, 0x82, 0x60, 0x79) \
655 X##_2(PRISM_ENCODING_UPPERCASE_BIT, 0x83, 0x83, 0x9F, 0xB6) \
656 X##_2(PRISM_ENCODING_UPPERCASE_BIT, 0x84, 0x84, 0x40, 0x60)
657
661static size_t
662pm_encoding_shift_jis_char_width(const uint8_t *b, ptrdiff_t n) {
663 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_SHIFT_JIS_FORMS)
664}
665
669static bool
670pm_encoding_shift_jis_isupper_char(const uint8_t *b, ptrdiff_t n) {
671 PM_ENCODING_FORMS_ISUPPER_CHAR(PM_ENCODING_SHIFT_JIS_FORMS)
672}
673
674#ifndef PRISM_ENCODING_EXCLUDE_FULL
675
680#define PM_ENCODING_BIG5_FORMS(X) \
681 X##_2(0, 0xA1, 0xFE, 0x40, 0x7E) \
682 X##_2(0, 0xA1, 0xFE, 0xA1, 0xFE)
683
687static size_t
688pm_encoding_big5_char_width(const uint8_t *b, ptrdiff_t n) {
689 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_BIG5_FORMS)
690}
691
697#define PM_ENCODING_CESU_8_FORMS(X) \
698 X##_2(0, 0xC2, 0xDF, 0x80, 0xBF) \
699 X##_3(0, 0xE0, 0xE0, 0xA0, 0xBF, 0x80, 0xBF) \
700 X##_3(0, 0xE1, 0xEC, 0x80, 0xBF, 0x80, 0xBF) \
701 X##_3(0, 0xED, 0xED, 0x80, 0x9F, 0x80, 0xBF) \
702 X##_3(0, 0xEE, 0xEF, 0x80, 0xBF, 0x80, 0xBF) \
703 X##_6(0, 0xED, 0xED, 0xA0, 0xAF, 0x80, 0xBF, 0xED, 0xED, 0xB0, 0xBF, 0x80, 0xBF)
704
708static size_t
709pm_encoding_cesu_8_char_width(const uint8_t *b, ptrdiff_t n) {
710 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_CESU_8_FORMS)
711}
712
717#define PM_ENCODING_CP949_FORMS(X) \
718 X##_1(0, 0x80, 0x80) \
719 X##_2(0, 0x81, 0xFE, 0x41, 0x5A) \
720 X##_2(0, 0x81, 0xFE, 0x61, 0x7A) \
721 X##_2(0, 0x81, 0xFE, 0x81, 0xFE)
722
726static size_t
727pm_encoding_cp949_char_width(const uint8_t *b, ptrdiff_t n) {
728 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_CP949_FORMS)
729}
730
734#define PM_ENCODING_EMACS_MULE_FORMS(X) \
735 X##_2(0, 0x81, 0x8F, 0xA0, 0xFF) \
736 X##_3(0, 0x90, 0x99, 0xA0, 0xFF, 0xA0, 0xFF) \
737 X##_3(0, 0x9A, 0x9B, 0xE0, 0xEF, 0xA0, 0xFF) \
738 X##_4(0, 0x9C, 0x9C, 0xF0, 0xF4, 0xA0, 0xFF, 0xA0, 0xFF) \
739 X##_4(0, 0x9D, 0x9D, 0xF5, 0xFE, 0xA0, 0xFF, 0xA0, 0xFF)
740
744static size_t
745pm_encoding_emacs_mule_char_width(const uint8_t *b, ptrdiff_t n) {
746 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_EMACS_MULE_FORMS)
747}
748
752#define PM_ENCODING_EUC_KR_FORMS(X) \
753 X##_2(0, 0xA1, 0xFE, 0xA1, 0xFE)
754
758static size_t
759pm_encoding_euc_kr_char_width(const uint8_t *b, ptrdiff_t n) {
760 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_EUC_KR_FORMS)
761}
762
766#define PM_ENCODING_EUC_TW_FORMS(X) \
767 X##_2(0, 0xA1, 0xFE, 0xA1, 0xFE) \
768 X##_4(0, 0x8E, 0x8E, 0xA1, 0xB0, 0xA1, 0xFE, 0xA1, 0xFE)
769
773static size_t
774pm_encoding_euc_tw_char_width(const uint8_t *b, ptrdiff_t n) {
775 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_EUC_TW_FORMS)
776}
777
781#define PM_ENCODING_GB18030_FORMS(X) \
782 X##_2(0, 0x81, 0xFE, 0x40, 0x7E) \
783 X##_2(0, 0x81, 0xFE, 0x80, 0xFE) \
784 X##_4(0, 0x81, 0xFE, 0x30, 0x39, 0x81, 0xFE, 0x30, 0x39)
785
789static size_t
790pm_encoding_gb18030_char_width(const uint8_t *b, ptrdiff_t n) {
791 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_GB18030_FORMS)
792}
793
798#define PM_ENCODING_GBK_FORMS(X) \
799 X##_1(0, 0x80, 0x80) \
800 X##_2(0, 0xA1, 0xA9, 0xA1, 0xFE) /* GBK/1 */ \
801 X##_2(0, 0xB0, 0xF7, 0xA1, 0xFE) /* GBK/2 */ \
802 X##_2(0, 0x81, 0xA0, 0x40, 0x7E) /* GBK/3 */ \
803 X##_2(0, 0x81, 0xA0, 0x80, 0xFE) /* GBK/3 */ \
804 X##_2(0, 0xAA, 0xFE, 0x40, 0x7E) /* GBK/4 */ \
805 X##_2(0, 0xAA, 0xFE, 0x80, 0xA0) /* GBK/4 */ \
806 X##_2(0, 0xA8, 0xA9, 0x40, 0x7E) /* GBK/5 */ \
807 X##_2(0, 0xA8, 0xA9, 0x80, 0xA0) /* GBK/5 */ \
808 X##_2(0, 0xAA, 0xAF, 0xA1, 0xFE) /* user-defined 1 */ \
809 X##_2(0, 0xF8, 0xFE, 0xA1, 0xFE) /* user-defined 2 */ \
810 X##_2(0, 0xA1, 0xA7, 0x40, 0x7E) /* user-defined 3 */ \
811 X##_2(0, 0xA1, 0xA7, 0x80, 0xA0) /* user-defined 3 */
812
816static size_t
817pm_encoding_gbk_char_width(const uint8_t *b, ptrdiff_t n) {
818 PM_ENCODING_FORMS_CHAR_WIDTH(PM_ENCODING_GBK_FORMS)
819}
820
821#endif
822
827size_t
828pm_encoding_char_width(const pm_encoding_t *encoding, const uint8_t *b, ptrdiff_t n) {
829 switch (encoding->kind) {
830 case PM_ENCODING_KIND_UTF_8:
831 return pm_encoding_utf_8_char_width(b, n);
832 case PM_ENCODING_KIND_ASCII:
833 return ((n > 0) && (*b < 0x80)) ? 1 : 0;
834 case PM_ENCODING_KIND_SINGLE_BYTE:
835 return n > 0 ? 1 : 0;
836 case PM_ENCODING_KIND_EUC_JP:
837 return pm_encoding_euc_jp_char_width(b, n);
838 case PM_ENCODING_KIND_SHIFT_JIS:
839 return pm_encoding_shift_jis_char_width(b, n);
840#ifndef PRISM_ENCODING_EXCLUDE_FULL
841 case PM_ENCODING_KIND_BIG5:
842 return pm_encoding_big5_char_width(b, n);
843 case PM_ENCODING_KIND_CESU_8:
844 return pm_encoding_cesu_8_char_width(b, n);
845 case PM_ENCODING_KIND_CP949:
846 return pm_encoding_cp949_char_width(b, n);
847 case PM_ENCODING_KIND_EMACS_MULE:
848 return pm_encoding_emacs_mule_char_width(b, n);
849 case PM_ENCODING_KIND_EUC_KR:
850 return pm_encoding_euc_kr_char_width(b, n);
851 case PM_ENCODING_KIND_EUC_TW:
852 return pm_encoding_euc_tw_char_width(b, n);
853 case PM_ENCODING_KIND_GB18030:
854 return pm_encoding_gb18030_char_width(b, n);
855 case PM_ENCODING_KIND_GBK:
856 return pm_encoding_gbk_char_width(b, n);
857#endif
858 default:
859 PRISM_ASSERT(false && "unreachable");
860 return 0;
861 }
862}
863
869bool
870pm_encoding_isupper_char(const pm_encoding_t *encoding, const uint8_t *b, ptrdiff_t n, pm_options_version_t version) {
871 switch (encoding->kind) {
872 case PM_ENCODING_KIND_ASCII:
873 case PM_ENCODING_KIND_SINGLE_BYTE: {
874 if (n <= 0) return false;
875 if (*b < 0x80) return pm_encoding_ascii_upper_p(*b);
876
877 const uint8_t *uppercase = encoding->uppercase;
878 return uppercase != NULL && ((uppercase[(*b - 0x80) >> 3] >> (*b & 7)) & 1);
879 }
880 case PM_ENCODING_KIND_UTF_8:
881 return pm_encoding_utf_8_isupper_char(b, n, version);
882 case PM_ENCODING_KIND_EUC_JP:
883 return pm_encoding_euc_jp_isupper_char(b, n);
884 case PM_ENCODING_KIND_SHIFT_JIS:
885 return pm_encoding_shift_jis_isupper_char(b, n);
886#ifndef PRISM_ENCODING_EXCLUDE_FULL
887 case PM_ENCODING_KIND_CESU_8: {
888 size_t width = pm_encoding_cesu_8_char_width(b, n);
889 return width != 0 && pm_unicode_uppercase_p(pm_cesu_8_codepoint(b, width), version);
890 }
891 /* These encodings have no uppercase letters beyond ASCII. */
892 case PM_ENCODING_KIND_BIG5:
893 case PM_ENCODING_KIND_CP949:
894 case PM_ENCODING_KIND_EMACS_MULE:
895 case PM_ENCODING_KIND_EUC_KR:
896 case PM_ENCODING_KIND_EUC_TW:
897 case PM_ENCODING_KIND_GB18030:
898 case PM_ENCODING_KIND_GBK:
899 return n > 0 && pm_encoding_ascii_upper_p(*b);
900#endif
901 default:
902 PRISM_ASSERT(false && "unreachable");
903 return false;
904 }
905}
906
907#undef PM_ENCODING_FORMS_ISUPPER_CHAR
908#undef PM_ENCODING_FORMS_CHAR_WIDTH
909#undef PM_ENCODING_UPPERCASE_4
910#undef PM_ENCODING_UPPERCASE_3
911#undef PM_ENCODING_UPPERCASE_2
912#undef PM_ENCODING_UPPERCASE_1
913#undef PM_ENCODING_WIDTH_6
914#undef PM_ENCODING_WIDTH_4
915#undef PM_ENCODING_WIDTH_3
916#undef PM_ENCODING_WIDTH_2
917#undef PM_ENCODING_WIDTH_1
918#undef PM_ENCODING_BYTE_IN
919
923const pm_encoding_t pm_encodings[] = {
924 [PM_ENCODING_UTF_8] = {
925 .name = "UTF-8",
926 .kind = PM_ENCODING_KIND_UTF_8
927 },
928 [PM_ENCODING_US_ASCII] = {
929 .name = "US-ASCII",
930 .kind = PM_ENCODING_KIND_ASCII
931 },
932 [PM_ENCODING_ASCII_8BIT] = {
933 .name = "ASCII-8BIT",
934 .kind = PM_ENCODING_KIND_SINGLE_BYTE
935 },
936 [PM_ENCODING_EUC_JP] = {
937 .name = "EUC-JP",
938 .kind = PM_ENCODING_KIND_EUC_JP
939 },
940 [PM_ENCODING_WINDOWS_31J] = {
941 .name = "Windows-31J",
942 .kind = PM_ENCODING_KIND_SHIFT_JIS
943 },
944
945#ifndef PRISM_ENCODING_EXCLUDE_FULL
946 [PM_ENCODING_BIG5] = {
947 .name = "Big5",
948 .kind = PM_ENCODING_KIND_BIG5
949 },
950 [PM_ENCODING_BIG5_HKSCS] = {
951 .name = "Big5-HKSCS",
952 .kind = PM_ENCODING_KIND_BIG5
953 },
954 [PM_ENCODING_BIG5_UAO] = {
955 .name = "Big5-UAO",
956 .kind = PM_ENCODING_KIND_BIG5
957 },
958 [PM_ENCODING_CESU_8] = {
959 .name = "CESU-8",
960 .kind = PM_ENCODING_KIND_CESU_8
961 },
962 [PM_ENCODING_CP51932] = {
963 .name = "CP51932",
964 .kind = PM_ENCODING_KIND_EUC_JP
965 },
966 [PM_ENCODING_CP850] = {
967 .name = "CP850",
968 .kind = PM_ENCODING_KIND_SINGLE_BYTE
969 },
970 [PM_ENCODING_CP852] = {
971 .name = "CP852",
972 .kind = PM_ENCODING_KIND_SINGLE_BYTE
973 },
974 [PM_ENCODING_CP855] = {
975 .name = "CP855",
976 .kind = PM_ENCODING_KIND_SINGLE_BYTE
977 },
978 [PM_ENCODING_CP949] = {
979 .name = "CP949",
980 .kind = PM_ENCODING_KIND_CP949
981 },
982 [PM_ENCODING_CP950] = {
983 .name = "CP950",
984 .kind = PM_ENCODING_KIND_BIG5
985 },
986 [PM_ENCODING_CP951] = {
987 .name = "CP951",
988 .kind = PM_ENCODING_KIND_BIG5
989 },
990 [PM_ENCODING_EMACS_MULE] = {
991 .name = "Emacs-Mule",
992 .kind = PM_ENCODING_KIND_EMACS_MULE
993 },
994 [PM_ENCODING_EUC_JP_MS] = {
995 .name = "eucJP-ms",
996 .kind = PM_ENCODING_KIND_EUC_JP
997 },
998 [PM_ENCODING_EUC_JIS_2004] = {
999 .name = "EUC-JIS-2004",
1000 .kind = PM_ENCODING_KIND_EUC_JP
1001 },
1002 [PM_ENCODING_EUC_KR] = {
1003 .name = "EUC-KR",
1004 .kind = PM_ENCODING_KIND_EUC_KR
1005 },
1006 [PM_ENCODING_EUC_TW] = {
1007 .name = "EUC-TW",
1008 .kind = PM_ENCODING_KIND_EUC_TW
1009 },
1010 [PM_ENCODING_GB12345] = {
1011 .name = "GB12345",
1012 .kind = PM_ENCODING_KIND_EUC_KR
1013 },
1014 [PM_ENCODING_GB18030] = {
1015 .name = "GB18030",
1016 .kind = PM_ENCODING_KIND_GB18030
1017 },
1018 [PM_ENCODING_GB1988] = {
1019 .name = "GB1988",
1020 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1021 },
1022 [PM_ENCODING_GB2312] = {
1023 .name = "GB2312",
1024 .kind = PM_ENCODING_KIND_EUC_KR
1025 },
1026 [PM_ENCODING_GBK] = {
1027 .name = "GBK",
1028 .kind = PM_ENCODING_KIND_GBK
1029 },
1030 [PM_ENCODING_IBM437] = {
1031 .name = "IBM437",
1032 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1033 },
1034 [PM_ENCODING_IBM720] = {
1035 .name = "IBM720",
1036 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1037 },
1038 [PM_ENCODING_IBM737] = {
1039 .name = "IBM737",
1040 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1041 },
1042 [PM_ENCODING_IBM775] = {
1043 .name = "IBM775",
1044 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1045 },
1046 [PM_ENCODING_IBM852] = {
1047 .name = "IBM852",
1048 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1049 },
1050 [PM_ENCODING_IBM855] = {
1051 .name = "IBM855",
1052 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1053 },
1054 [PM_ENCODING_IBM857] = {
1055 .name = "IBM857",
1056 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1057 },
1058 [PM_ENCODING_IBM860] = {
1059 .name = "IBM860",
1060 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1061 },
1062 [PM_ENCODING_IBM861] = {
1063 .name = "IBM861",
1064 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1065 },
1066 [PM_ENCODING_IBM862] = {
1067 .name = "IBM862",
1068 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1069 },
1070 [PM_ENCODING_IBM863] = {
1071 .name = "IBM863",
1072 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1073 },
1074 [PM_ENCODING_IBM864] = {
1075 .name = "IBM864",
1076 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1077 },
1078 [PM_ENCODING_IBM865] = {
1079 .name = "IBM865",
1080 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1081 },
1082 [PM_ENCODING_IBM866] = {
1083 .name = "IBM866",
1084 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1085 },
1086 [PM_ENCODING_IBM869] = {
1087 .name = "IBM869",
1088 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1089 },
1090 [PM_ENCODING_ISO_8859_1] = {
1091 .name = "ISO-8859-1",
1092 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1093 .uppercase = pm_encoding_iso_8859_1_uppercase
1094 },
1095 [PM_ENCODING_ISO_8859_2] = {
1096 .name = "ISO-8859-2",
1097 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1098 .uppercase = pm_encoding_iso_8859_2_uppercase
1099 },
1100 [PM_ENCODING_ISO_8859_3] = {
1101 .name = "ISO-8859-3",
1102 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1103 .uppercase = pm_encoding_iso_8859_3_uppercase
1104 },
1105 [PM_ENCODING_ISO_8859_4] = {
1106 .name = "ISO-8859-4",
1107 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1108 .uppercase = pm_encoding_iso_8859_4_uppercase
1109 },
1110 [PM_ENCODING_ISO_8859_5] = {
1111 .name = "ISO-8859-5",
1112 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1113 .uppercase = pm_encoding_iso_8859_5_uppercase
1114 },
1115 [PM_ENCODING_ISO_8859_6] = {
1116 .name = "ISO-8859-6",
1117 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1118 },
1119 [PM_ENCODING_ISO_8859_7] = {
1120 .name = "ISO-8859-7",
1121 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1122 .uppercase = pm_encoding_iso_8859_7_uppercase
1123 },
1124 [PM_ENCODING_ISO_8859_8] = {
1125 .name = "ISO-8859-8",
1126 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1127 },
1128 [PM_ENCODING_ISO_8859_9] = {
1129 .name = "ISO-8859-9",
1130 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1131 .uppercase = pm_encoding_iso_8859_1_uppercase
1132 },
1133 [PM_ENCODING_ISO_8859_10] = {
1134 .name = "ISO-8859-10",
1135 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1136 .uppercase = pm_encoding_iso_8859_10_uppercase
1137 },
1138 [PM_ENCODING_ISO_8859_11] = {
1139 .name = "ISO-8859-11",
1140 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1141 },
1142 [PM_ENCODING_ISO_8859_13] = {
1143 .name = "ISO-8859-13",
1144 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1145 .uppercase = pm_encoding_iso_8859_13_uppercase
1146 },
1147 [PM_ENCODING_ISO_8859_14] = {
1148 .name = "ISO-8859-14",
1149 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1150 .uppercase = pm_encoding_iso_8859_14_uppercase
1151 },
1152 [PM_ENCODING_ISO_8859_15] = {
1153 .name = "ISO-8859-15",
1154 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1155 .uppercase = pm_encoding_iso_8859_15_uppercase
1156 },
1157 [PM_ENCODING_ISO_8859_16] = {
1158 .name = "ISO-8859-16",
1159 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1160 .uppercase = pm_encoding_iso_8859_16_uppercase
1161 },
1162 [PM_ENCODING_KOI8_R] = {
1163 .name = "KOI8-R",
1164 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1165 .uppercase = pm_encoding_koi8_r_uppercase
1166 },
1167 [PM_ENCODING_KOI8_U] = {
1168 .name = "KOI8-U",
1169 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1170 .uppercase = pm_encoding_koi8_u_uppercase
1171 },
1172 [PM_ENCODING_MAC_CENT_EURO] = {
1173 .name = "macCentEuro",
1174 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1175 },
1176 [PM_ENCODING_MAC_CROATIAN] = {
1177 .name = "macCroatian",
1178 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1179 },
1180 [PM_ENCODING_MAC_CYRILLIC] = {
1181 .name = "macCyrillic",
1182 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1183 },
1184 [PM_ENCODING_MAC_GREEK] = {
1185 .name = "macGreek",
1186 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1187 },
1188 [PM_ENCODING_MAC_ICELAND] = {
1189 .name = "macIceland",
1190 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1191 },
1192 [PM_ENCODING_MAC_JAPANESE] = {
1193 .name = "MacJapanese",
1194 .kind = PM_ENCODING_KIND_SHIFT_JIS
1195 },
1196 [PM_ENCODING_MAC_ROMAN] = {
1197 .name = "macRoman",
1198 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1199 },
1200 [PM_ENCODING_MAC_ROMANIA] = {
1201 .name = "macRomania",
1202 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1203 },
1204 [PM_ENCODING_MAC_THAI] = {
1205 .name = "macThai",
1206 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1207 },
1208 [PM_ENCODING_MAC_TURKISH] = {
1209 .name = "macTurkish",
1210 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1211 },
1212 [PM_ENCODING_MAC_UKRAINE] = {
1213 .name = "macUkraine",
1214 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1215 },
1216 [PM_ENCODING_SHIFT_JIS] = {
1217 .name = "Shift_JIS",
1218 .kind = PM_ENCODING_KIND_SHIFT_JIS
1219 },
1220 [PM_ENCODING_SJIS_DOCOMO] = {
1221 .name = "SJIS-DoCoMo",
1222 .kind = PM_ENCODING_KIND_SHIFT_JIS
1223 },
1224 [PM_ENCODING_SJIS_KDDI] = {
1225 .name = "SJIS-KDDI",
1226 .kind = PM_ENCODING_KIND_SHIFT_JIS
1227 },
1228 [PM_ENCODING_SJIS_SOFTBANK] = {
1229 .name = "SJIS-SoftBank",
1230 .kind = PM_ENCODING_KIND_SHIFT_JIS
1231 },
1232 [PM_ENCODING_STATELESS_ISO_2022_JP] = {
1233 .name = "stateless-ISO-2022-JP",
1234 .kind = PM_ENCODING_KIND_EMACS_MULE
1235 },
1236 [PM_ENCODING_STATELESS_ISO_2022_JP_KDDI] = {
1237 .name = "stateless-ISO-2022-JP-KDDI",
1238 .kind = PM_ENCODING_KIND_EMACS_MULE
1239 },
1240 [PM_ENCODING_TIS_620] = {
1241 .name = "TIS-620",
1242 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1243 },
1244 [PM_ENCODING_UTF8_MAC] = {
1245 .name = "UTF8-MAC",
1246 .kind = PM_ENCODING_KIND_UTF_8
1247 },
1248 [PM_ENCODING_UTF8_DOCOMO] = {
1249 .name = "UTF8-DoCoMo",
1250 .kind = PM_ENCODING_KIND_UTF_8
1251 },
1252 [PM_ENCODING_UTF8_KDDI] = {
1253 .name = "UTF8-KDDI",
1254 .kind = PM_ENCODING_KIND_UTF_8
1255 },
1256 [PM_ENCODING_UTF8_SOFTBANK] = {
1257 .name = "UTF8-SoftBank",
1258 .kind = PM_ENCODING_KIND_UTF_8
1259 },
1260 [PM_ENCODING_WINDOWS_1250] = {
1261 .name = "Windows-1250",
1262 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1263 .uppercase = pm_encoding_windows_1250_uppercase
1264 },
1265 [PM_ENCODING_WINDOWS_1251] = {
1266 .name = "Windows-1251",
1267 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1268 .uppercase = pm_encoding_windows_1251_uppercase
1269 },
1270 [PM_ENCODING_WINDOWS_1252] = {
1271 .name = "Windows-1252",
1272 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1273 .uppercase = pm_encoding_windows_1252_uppercase
1274 },
1275 [PM_ENCODING_WINDOWS_1253] = {
1276 .name = "Windows-1253",
1277 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1278 .uppercase = pm_encoding_windows_1253_uppercase
1279 },
1280 [PM_ENCODING_WINDOWS_1254] = {
1281 .name = "Windows-1254",
1282 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1283 .uppercase = pm_encoding_windows_1254_uppercase
1284 },
1285 [PM_ENCODING_WINDOWS_1255] = {
1286 .name = "Windows-1255",
1287 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1288 },
1289 [PM_ENCODING_WINDOWS_1256] = {
1290 .name = "Windows-1256",
1291 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1292 },
1293 [PM_ENCODING_WINDOWS_1257] = {
1294 .name = "Windows-1257",
1295 .kind = PM_ENCODING_KIND_SINGLE_BYTE,
1296 .uppercase = pm_encoding_iso_8859_13_uppercase
1297 },
1298 [PM_ENCODING_WINDOWS_1258] = {
1299 .name = "Windows-1258",
1300 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1301 },
1302 [PM_ENCODING_WINDOWS_874] = {
1303 .name = "Windows-874",
1304 .kind = PM_ENCODING_KIND_SINGLE_BYTE
1305 }
1306#endif
1307};
1308
1313const pm_encoding_t *
1314pm_encoding_find(const uint8_t *start, const uint8_t *end) {
1315 size_t width = (size_t) (end - start);
1316
1317 // First, we're going to check for UTF-8. This is the most common encoding.
1318 // UTF-8 can contain extra information at the end about the platform it is
1319 // encoded on, such as UTF-8-MAC or UTF-8-UNIX. We'll ignore those suffixes.
1320 if ((start + 5 <= end) && (pm_strncasecmp(start, (const uint8_t *) "UTF-8", 5) == 0)) {
1321#ifndef PRISM_ENCODING_EXCLUDE_FULL
1322 // We need to explicitly handle UTF-8-HFS, as that one needs to switch
1323 // over to being UTF8-MAC.
1324 if (width == 9 && (pm_strncasecmp(start + 5, (const uint8_t *) "-HFS", 4) == 0)) {
1325 return &pm_encodings[PM_ENCODING_UTF8_MAC];
1326 }
1327#endif
1328
1329 // Otherwise we'll return the default UTF-8 encoding.
1330 return PM_ENCODING_UTF_8_ENTRY;
1331 }
1332
1333 // Next, we're going to loop through each of the encodings that we handle
1334 // explicitly. If we found one that we understand, we'll use that value.
1335#define ENCODING1(name, encoding) if (width == sizeof(name) - 1 && pm_strncasecmp(start, (const uint8_t *) name, width) == 0) return &pm_encodings[encoding];
1336#define ENCODING2(name1, name2, encoding) ENCODING1(name1, encoding) ENCODING1(name2, encoding)
1337
1338 if (width >= 3) {
1339 switch (*start) {
1340 case 'A': case 'a':
1341 ENCODING1("ASCII", PM_ENCODING_US_ASCII);
1342 ENCODING1("ASCII-8BIT", PM_ENCODING_ASCII_8BIT);
1343 ENCODING1("ANSI_X3.4-1968", PM_ENCODING_US_ASCII);
1344 break;
1345 case 'B': case 'b':
1346 ENCODING1("BINARY", PM_ENCODING_ASCII_8BIT);
1347#ifndef PRISM_ENCODING_EXCLUDE_FULL
1348 ENCODING1("Big5", PM_ENCODING_BIG5);
1349 ENCODING2("Big5-HKSCS", "Big5-HKSCS:2008", PM_ENCODING_BIG5_HKSCS);
1350 ENCODING1("Big5-UAO", PM_ENCODING_BIG5_UAO);
1351#endif
1352 break;
1353 case 'C': case 'c':
1354 ENCODING1("CP65001", PM_ENCODING_UTF_8);
1355 ENCODING2("CP932", "csWindows31J", PM_ENCODING_WINDOWS_31J);
1356#ifndef PRISM_ENCODING_EXCLUDE_FULL
1357 ENCODING1("CESU-8", PM_ENCODING_CESU_8);
1358 ENCODING1("CP437", PM_ENCODING_IBM437);
1359 ENCODING1("CP720", PM_ENCODING_IBM720);
1360 ENCODING1("CP737", PM_ENCODING_IBM737);
1361 ENCODING1("CP775", PM_ENCODING_IBM775);
1362 ENCODING1("CP850", PM_ENCODING_CP850);
1363 ENCODING1("CP852", PM_ENCODING_CP852);
1364 ENCODING1("CP855", PM_ENCODING_CP855);
1365 ENCODING1("CP857", PM_ENCODING_IBM857);
1366 ENCODING1("CP860", PM_ENCODING_IBM860);
1367 ENCODING1("CP861", PM_ENCODING_IBM861);
1368 ENCODING1("CP862", PM_ENCODING_IBM862);
1369 ENCODING1("CP864", PM_ENCODING_IBM864);
1370 ENCODING1("CP865", PM_ENCODING_IBM865);
1371 ENCODING1("CP866", PM_ENCODING_IBM866);
1372 ENCODING1("CP869", PM_ENCODING_IBM869);
1373 ENCODING1("CP874", PM_ENCODING_WINDOWS_874);
1374 ENCODING1("CP878", PM_ENCODING_KOI8_R);
1375 ENCODING1("CP863", PM_ENCODING_IBM863);
1376 ENCODING1("CP936", PM_ENCODING_GBK);
1377 ENCODING1("CP949", PM_ENCODING_CP949);
1378 ENCODING1("CP950", PM_ENCODING_CP950);
1379 ENCODING1("CP951", PM_ENCODING_CP951);
1380 ENCODING1("CP1250", PM_ENCODING_WINDOWS_1250);
1381 ENCODING1("CP1251", PM_ENCODING_WINDOWS_1251);
1382 ENCODING1("CP1252", PM_ENCODING_WINDOWS_1252);
1383 ENCODING1("CP1253", PM_ENCODING_WINDOWS_1253);
1384 ENCODING1("CP1254", PM_ENCODING_WINDOWS_1254);
1385 ENCODING1("CP1255", PM_ENCODING_WINDOWS_1255);
1386 ENCODING1("CP1256", PM_ENCODING_WINDOWS_1256);
1387 ENCODING1("CP1257", PM_ENCODING_WINDOWS_1257);
1388 ENCODING1("CP1258", PM_ENCODING_WINDOWS_1258);
1389 ENCODING1("CP51932", PM_ENCODING_CP51932);
1390#endif
1391 break;
1392 case 'E': case 'e':
1393 ENCODING2("EUC-JP", "eucJP", PM_ENCODING_EUC_JP);
1394#ifndef PRISM_ENCODING_EXCLUDE_FULL
1395 ENCODING2("eucJP-ms", "euc-jp-ms", PM_ENCODING_EUC_JP_MS);
1396 ENCODING2("EUC-JIS-2004", "EUC-JISX0213", PM_ENCODING_EUC_JIS_2004);
1397 ENCODING2("EUC-KR", "eucKR", PM_ENCODING_EUC_KR);
1398 ENCODING2("EUC-CN", "eucCN", PM_ENCODING_GB2312);
1399 ENCODING2("EUC-TW", "eucTW", PM_ENCODING_EUC_TW);
1400 ENCODING1("Emacs-Mule", PM_ENCODING_EMACS_MULE);
1401#endif
1402 break;
1403 case 'G': case 'g':
1404#ifndef PRISM_ENCODING_EXCLUDE_FULL
1405 ENCODING1("GBK", PM_ENCODING_GBK);
1406 ENCODING1("GB12345", PM_ENCODING_GB12345);
1407 ENCODING1("GB18030", PM_ENCODING_GB18030);
1408 ENCODING1("GB1988", PM_ENCODING_GB1988);
1409 ENCODING1("GB2312", PM_ENCODING_GB2312);
1410#endif
1411 break;
1412 case 'I': case 'i':
1413#ifndef PRISM_ENCODING_EXCLUDE_FULL
1414 ENCODING1("IBM437", PM_ENCODING_IBM437);
1415 ENCODING1("IBM720", PM_ENCODING_IBM720);
1416 ENCODING1("IBM737", PM_ENCODING_IBM737);
1417 ENCODING1("IBM775", PM_ENCODING_IBM775);
1418 ENCODING1("IBM850", PM_ENCODING_CP850);
1419 ENCODING1("IBM852", PM_ENCODING_IBM852);
1420 ENCODING1("IBM855", PM_ENCODING_IBM855);
1421 ENCODING1("IBM857", PM_ENCODING_IBM857);
1422 ENCODING1("IBM860", PM_ENCODING_IBM860);
1423 ENCODING1("IBM861", PM_ENCODING_IBM861);
1424 ENCODING1("IBM862", PM_ENCODING_IBM862);
1425 ENCODING1("IBM863", PM_ENCODING_IBM863);
1426 ENCODING1("IBM864", PM_ENCODING_IBM864);
1427 ENCODING1("IBM865", PM_ENCODING_IBM865);
1428 ENCODING1("IBM866", PM_ENCODING_IBM866);
1429 ENCODING1("IBM869", PM_ENCODING_IBM869);
1430 ENCODING2("ISO-8859-1", "ISO8859-1", PM_ENCODING_ISO_8859_1);
1431 ENCODING2("ISO-8859-2", "ISO8859-2", PM_ENCODING_ISO_8859_2);
1432 ENCODING2("ISO-8859-3", "ISO8859-3", PM_ENCODING_ISO_8859_3);
1433 ENCODING2("ISO-8859-4", "ISO8859-4", PM_ENCODING_ISO_8859_4);
1434 ENCODING2("ISO-8859-5", "ISO8859-5", PM_ENCODING_ISO_8859_5);
1435 ENCODING2("ISO-8859-6", "ISO8859-6", PM_ENCODING_ISO_8859_6);
1436 ENCODING2("ISO-8859-7", "ISO8859-7", PM_ENCODING_ISO_8859_7);
1437 ENCODING2("ISO-8859-8", "ISO8859-8", PM_ENCODING_ISO_8859_8);
1438 ENCODING2("ISO-8859-9", "ISO8859-9", PM_ENCODING_ISO_8859_9);
1439 ENCODING2("ISO-8859-10", "ISO8859-10", PM_ENCODING_ISO_8859_10);
1440 ENCODING2("ISO-8859-11", "ISO8859-11", PM_ENCODING_ISO_8859_11);
1441 ENCODING2("ISO-8859-13", "ISO8859-13", PM_ENCODING_ISO_8859_13);
1442 ENCODING2("ISO-8859-14", "ISO8859-14", PM_ENCODING_ISO_8859_14);
1443 ENCODING2("ISO-8859-15", "ISO8859-15", PM_ENCODING_ISO_8859_15);
1444 ENCODING2("ISO-8859-16", "ISO8859-16", PM_ENCODING_ISO_8859_16);
1445#endif
1446 break;
1447 case 'K': case 'k':
1448#ifndef PRISM_ENCODING_EXCLUDE_FULL
1449 ENCODING1("KOI8-R", PM_ENCODING_KOI8_R);
1450 ENCODING1("KOI8-U", PM_ENCODING_KOI8_U);
1451#endif
1452 break;
1453 case 'M': case 'm':
1454#ifndef PRISM_ENCODING_EXCLUDE_FULL
1455 ENCODING1("macCentEuro", PM_ENCODING_MAC_CENT_EURO);
1456 ENCODING1("macCroatian", PM_ENCODING_MAC_CROATIAN);
1457 ENCODING1("macCyrillic", PM_ENCODING_MAC_CYRILLIC);
1458 ENCODING1("macGreek", PM_ENCODING_MAC_GREEK);
1459 ENCODING1("macIceland", PM_ENCODING_MAC_ICELAND);
1460 ENCODING1("MacJapanese", PM_ENCODING_MAC_JAPANESE);
1461 ENCODING1("MacJapan", PM_ENCODING_MAC_JAPANESE);
1462 ENCODING1("macRoman", PM_ENCODING_MAC_ROMAN);
1463 ENCODING1("macRomania", PM_ENCODING_MAC_ROMANIA);
1464 ENCODING1("macThai", PM_ENCODING_MAC_THAI);
1465 ENCODING1("macTurkish", PM_ENCODING_MAC_TURKISH);
1466 ENCODING1("macUkraine", PM_ENCODING_MAC_UKRAINE);
1467#endif
1468 break;
1469 case 'P': case 'p':
1470 ENCODING1("PCK", PM_ENCODING_WINDOWS_31J);
1471 break;
1472 case 'S': case 's':
1473 ENCODING1("SJIS", PM_ENCODING_WINDOWS_31J);
1474#ifndef PRISM_ENCODING_EXCLUDE_FULL
1475 ENCODING1("Shift_JIS", PM_ENCODING_SHIFT_JIS);
1476 ENCODING1("SJIS-DoCoMo", PM_ENCODING_SJIS_DOCOMO);
1477 ENCODING1("SJIS-KDDI", PM_ENCODING_SJIS_KDDI);
1478 ENCODING1("SJIS-SoftBank", PM_ENCODING_SJIS_SOFTBANK);
1479 ENCODING1("stateless-ISO-2022-JP", PM_ENCODING_STATELESS_ISO_2022_JP);
1480 ENCODING1("stateless-ISO-2022-JP-KDDI", PM_ENCODING_STATELESS_ISO_2022_JP_KDDI);
1481#endif
1482 break;
1483 case 'T': case 't':
1484#ifndef PRISM_ENCODING_EXCLUDE_FULL
1485 ENCODING1("TIS-620", PM_ENCODING_TIS_620);
1486#endif
1487 break;
1488 case 'U': case 'u':
1489 ENCODING1("US-ASCII", PM_ENCODING_US_ASCII);
1490#ifndef PRISM_ENCODING_EXCLUDE_FULL
1491 ENCODING2("UTF8-MAC", "UTF-8-HFS", PM_ENCODING_UTF8_MAC);
1492 ENCODING1("UTF8-DoCoMo", PM_ENCODING_UTF8_DOCOMO);
1493 ENCODING1("UTF8-KDDI", PM_ENCODING_UTF8_KDDI);
1494 ENCODING1("UTF8-SoftBank", PM_ENCODING_UTF8_SOFTBANK);
1495#endif
1496 break;
1497 case 'W': case 'w':
1498 ENCODING1("Windows-31J", PM_ENCODING_WINDOWS_31J);
1499#ifndef PRISM_ENCODING_EXCLUDE_FULL
1500 ENCODING1("Windows-874", PM_ENCODING_WINDOWS_874);
1501 ENCODING1("Windows-1250", PM_ENCODING_WINDOWS_1250);
1502 ENCODING1("Windows-1251", PM_ENCODING_WINDOWS_1251);
1503 ENCODING1("Windows-1252", PM_ENCODING_WINDOWS_1252);
1504 ENCODING1("Windows-1253", PM_ENCODING_WINDOWS_1253);
1505 ENCODING1("Windows-1254", PM_ENCODING_WINDOWS_1254);
1506 ENCODING1("Windows-1255", PM_ENCODING_WINDOWS_1255);
1507 ENCODING1("Windows-1256", PM_ENCODING_WINDOWS_1256);
1508 ENCODING1("Windows-1257", PM_ENCODING_WINDOWS_1257);
1509 ENCODING1("Windows-1258", PM_ENCODING_WINDOWS_1258);
1510#endif
1511 break;
1512 case '6':
1513 ENCODING1("646", PM_ENCODING_US_ASCII);
1514 break;
1515 }
1516 }
1517
1518#undef ENCODING2
1519#undef ENCODING1
1520
1521 // If we didn't match any encodings, return NULL.
1522 return NULL;
1523}
#define PRISM_UNLIKELY(x)
The compiler should predicate that this branch will not be taken.
Definition branch.h:17
VALUE type(ANYARGS)
ANYARGS-ed function type.