Ruby 4.1.0dev (2026-11-17 revision 3a19e2efcf133ee35daaeffc36633cd99a9f2c47)
encoding.h
1#ifndef PRISM_INTERNAL_ENCODING_H
2#define PRISM_INTERNAL_ENCODING_H
3
5#include "prism/excludes.h"
6#include "prism/internal/options.h"
7
8#include <stdbool.h>
9#include <stddef.h>
10#include <stdint.h>
11
12/*
13 * The ways in which an encoding answers questions about its characters. Every
14 * kind after UTF-8 is a multibyte encoding that is ASCII below 0x80 and whose
15 * other characters take the forms listed for it in encoding.c.
16 */
17typedef enum {
18 /* Every byte is a character. */
19 PM_ENCODING_KIND_SINGLE_BYTE,
20
21 /* Every byte below 0x80 is a character, and every other byte is invalid. */
22 PM_ENCODING_KIND_ASCII,
23
24 /* UTF-8, whose characters have the properties of their code points. */
25 PM_ENCODING_KIND_UTF_8,
26
27 /* EUC-JP and the encodings that share its forms. */
28 PM_ENCODING_KIND_EUC_JP,
29
30 /* Shift_JIS and the encodings that share its forms. */
31 PM_ENCODING_KIND_SHIFT_JIS,
32
33 /* Big5 and the encodings that share its forms. */
34 PM_ENCODING_KIND_BIG5,
35
36 /* CESU-8, whose characters have the properties of their code points. */
37 PM_ENCODING_KIND_CESU_8,
38
39 /* CP949. */
40 PM_ENCODING_KIND_CP949,
41
42 /* Emacs-Mule and the encodings that share its forms. */
43 PM_ENCODING_KIND_EMACS_MULE,
44
45 /* EUC-KR and the encodings that share its forms. */
46 PM_ENCODING_KIND_EUC_KR,
47
48 /* EUC-TW. */
49 PM_ENCODING_KIND_EUC_TW,
50
51 /* GB18030. */
52 PM_ENCODING_KIND_GB18030,
53
54 /* GBK. */
55 PM_ENCODING_KIND_GBK
56} pm_encoding_kind_t;
57
58/*
59 * An encoding that prism can parse source in. It holds what the answers to the
60 * questions about its characters come from, which pm_encoding_char_width and
61 * the functions after it read.
62 */
63typedef struct {
64 /*
65 * The name of the encoding. This should correspond to a value that can be
66 * passed to Encoding.find in Ruby.
67 */
68 const char *name;
69
70 /*
71 * In a single-byte encoding, the bytes from 0x80 up that begin a constant
72 * name, one bit per byte, or NULL if none do.
73 */
74 const uint8_t *uppercase;
75
76 /*
77 * How the encoding answers questions about its characters, as a
78 * pm_encoding_kind_t stored in a byte to keep the table of encodings small.
79 */
80 uint8_t kind;
82
83/*
84 * Return true if the encoding is a multibyte encoding. Every kind other than
85 * single-byte and ASCII has characters that span more than one byte.
86 */
87static PRISM_INLINE bool
88pm_encoding_multibyte_p(const pm_encoding_t *encoding) {
89 return encoding->kind != PM_ENCODING_KIND_SINGLE_BYTE && encoding->kind != PM_ENCODING_KIND_ASCII;
90}
91
92/* The bit of the ASCII table that marks a letter. */
93#define PRISM_ENCODING_ALPHABETIC_BIT (1 << 0)
94
95/* The bit of the ASCII table that marks a letter or a digit. */
96#define PRISM_ENCODING_ALPHANUMERIC_BIT (1 << 1)
97
98/*
99 * The bit that marks a character that begins a constant name, in the ASCII
100 * table, the unicode tables, and the forms of the multibyte encodings.
101 */
102#define PRISM_ENCODING_UPPERCASE_BIT (1 << 2)
103
104/*
105 * Return the number of bytes in the next character in the given encoding, or 0
106 * if the bytes do not form one.
107 */
108size_t pm_encoding_char_width(const pm_encoding_t *encoding, const uint8_t *b, ptrdiff_t n);
109
110/*
111 * Return true if the next character in the given encoding is uppercase, or
112 * titlecase, in the given version of Ruby. Such a character begins a constant
113 * name. The version only matters for an encoding whose characters are Unicode
114 * code points, since each version of Ruby uses a version of Unicode.
115 */
116bool pm_encoding_isupper_char(const pm_encoding_t *encoding, const uint8_t *b, ptrdiff_t n, pm_options_version_t version);
117
118/* Return the size of the next character in the UTF-8 encoding. */
119size_t pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
120
121/*
122 * Return true if the next character in the UTF-8 encoding is uppercase, or
123 * titlecase, in the given version of Ruby.
124 */
125bool pm_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n, pm_options_version_t version);
126
127/*
128 * This lookup table indicates whether a US-ASCII character is alphabetical,
129 * alphanumeric, or uppercase. Every encoding that prism supports is
130 * ASCII-compatible, so this table answers for the bytes below 0x80 in all of
131 * them.
132 */
133extern const uint8_t pm_encoding_ascii_table[256];
134
135/*
136 * Return true if the byte is an ASCII letter. A byte that is not ASCII is not
137 * one, in every encoding.
138 */
139static PRISM_INLINE bool
140pm_encoding_ascii_alpha_p(uint8_t byte) {
141 return pm_encoding_ascii_table[byte] & PRISM_ENCODING_ALPHABETIC_BIT;
142}
143
144/*
145 * Return true if the byte is an ASCII uppercase letter. A byte that is not
146 * ASCII is not one, in every encoding.
147 */
148static PRISM_INLINE bool
149pm_encoding_ascii_upper_p(uint8_t byte) {
150 return pm_encoding_ascii_table[byte] & PRISM_ENCODING_UPPERCASE_BIT;
151}
152
153/*
154 * Return true if the byte is an ASCII letter or digit. A byte that is not ASCII
155 * is neither, in every encoding.
156 */
157static PRISM_INLINE bool
158pm_encoding_ascii_alnum_p(uint8_t byte) {
159 return pm_encoding_ascii_table[byte] & PRISM_ENCODING_ALPHANUMERIC_BIT;
160}
161
162/* These are all of the encodings that prism supports. */
163typedef enum {
164 PM_ENCODING_UTF_8 = 0,
165 PM_ENCODING_US_ASCII,
166 PM_ENCODING_ASCII_8BIT,
167 PM_ENCODING_EUC_JP,
168 PM_ENCODING_WINDOWS_31J,
169
170/* We optionally support excluding the full set of encodings to only support the
171 * minimum necessary to process Ruby code without encoding comments. */
172#ifndef PRISM_ENCODING_EXCLUDE_FULL
173 PM_ENCODING_BIG5,
174 PM_ENCODING_BIG5_HKSCS,
175 PM_ENCODING_BIG5_UAO,
176 PM_ENCODING_CESU_8,
177 PM_ENCODING_CP51932,
178 PM_ENCODING_CP850,
179 PM_ENCODING_CP852,
180 PM_ENCODING_CP855,
181 PM_ENCODING_CP949,
182 PM_ENCODING_CP950,
183 PM_ENCODING_CP951,
184 PM_ENCODING_EMACS_MULE,
185 PM_ENCODING_EUC_JP_MS,
186 PM_ENCODING_EUC_JIS_2004,
187 PM_ENCODING_EUC_KR,
188 PM_ENCODING_EUC_TW,
189 PM_ENCODING_GB12345,
190 PM_ENCODING_GB18030,
191 PM_ENCODING_GB1988,
192 PM_ENCODING_GB2312,
193 PM_ENCODING_GBK,
194 PM_ENCODING_IBM437,
195 PM_ENCODING_IBM720,
196 PM_ENCODING_IBM737,
197 PM_ENCODING_IBM775,
198 PM_ENCODING_IBM852,
199 PM_ENCODING_IBM855,
200 PM_ENCODING_IBM857,
201 PM_ENCODING_IBM860,
202 PM_ENCODING_IBM861,
203 PM_ENCODING_IBM862,
204 PM_ENCODING_IBM863,
205 PM_ENCODING_IBM864,
206 PM_ENCODING_IBM865,
207 PM_ENCODING_IBM866,
208 PM_ENCODING_IBM869,
209 PM_ENCODING_ISO_8859_1,
210 PM_ENCODING_ISO_8859_2,
211 PM_ENCODING_ISO_8859_3,
212 PM_ENCODING_ISO_8859_4,
213 PM_ENCODING_ISO_8859_5,
214 PM_ENCODING_ISO_8859_6,
215 PM_ENCODING_ISO_8859_7,
216 PM_ENCODING_ISO_8859_8,
217 PM_ENCODING_ISO_8859_9,
218 PM_ENCODING_ISO_8859_10,
219 PM_ENCODING_ISO_8859_11,
220 PM_ENCODING_ISO_8859_13,
221 PM_ENCODING_ISO_8859_14,
222 PM_ENCODING_ISO_8859_15,
223 PM_ENCODING_ISO_8859_16,
224 PM_ENCODING_KOI8_R,
225 PM_ENCODING_KOI8_U,
226 PM_ENCODING_MAC_CENT_EURO,
227 PM_ENCODING_MAC_CROATIAN,
228 PM_ENCODING_MAC_CYRILLIC,
229 PM_ENCODING_MAC_GREEK,
230 PM_ENCODING_MAC_ICELAND,
231 PM_ENCODING_MAC_JAPANESE,
232 PM_ENCODING_MAC_ROMAN,
233 PM_ENCODING_MAC_ROMANIA,
234 PM_ENCODING_MAC_THAI,
235 PM_ENCODING_MAC_TURKISH,
236 PM_ENCODING_MAC_UKRAINE,
237 PM_ENCODING_SHIFT_JIS,
238 PM_ENCODING_SJIS_DOCOMO,
239 PM_ENCODING_SJIS_KDDI,
240 PM_ENCODING_SJIS_SOFTBANK,
241 PM_ENCODING_STATELESS_ISO_2022_JP,
242 PM_ENCODING_STATELESS_ISO_2022_JP_KDDI,
243 PM_ENCODING_TIS_620,
244 PM_ENCODING_UTF8_MAC,
245 PM_ENCODING_UTF8_DOCOMO,
246 PM_ENCODING_UTF8_KDDI,
247 PM_ENCODING_UTF8_SOFTBANK,
248 PM_ENCODING_WINDOWS_1250,
249 PM_ENCODING_WINDOWS_1251,
250 PM_ENCODING_WINDOWS_1252,
251 PM_ENCODING_WINDOWS_1253,
252 PM_ENCODING_WINDOWS_1254,
253 PM_ENCODING_WINDOWS_1255,
254 PM_ENCODING_WINDOWS_1256,
255 PM_ENCODING_WINDOWS_1257,
256 PM_ENCODING_WINDOWS_1258,
257 PM_ENCODING_WINDOWS_874,
258#endif
259
260 PM_ENCODING_MAXIMUM
261} pm_encoding_type_t;
262
263/* This is the table of all of the encodings that prism supports. */
264extern const pm_encoding_t pm_encodings[PM_ENCODING_MAXIMUM];
265
266/*
267 * This is the default UTF-8 encoding. We need a reference to it to quickly
268 * create parsers.
269 */
270#define PM_ENCODING_UTF_8_ENTRY (&pm_encodings[PM_ENCODING_UTF_8])
271
272/*
273 * This is the US-ASCII encoding. We need a reference to it to be able to
274 * compare against it when a string is being created because it could possibly
275 * need to fall back to ASCII-8BIT.
276 */
277#define PM_ENCODING_US_ASCII_ENTRY (&pm_encodings[PM_ENCODING_US_ASCII])
278
279/*
280 * This is the ASCII-8BIT encoding. We need a reference to it so that pm_strpbrk
281 * can compare against it because invalid multibyte characters are not a thing
282 * in this encoding. It is also needed for handling Regexp encoding flags.
283 */
284#define PM_ENCODING_ASCII_8BIT_ENTRY (&pm_encodings[PM_ENCODING_ASCII_8BIT])
285
286/*
287 * This is the EUC-JP encoding. We need a reference to it to quickly process
288 * regular expression modifiers.
289 */
290#define PM_ENCODING_EUC_JP_ENTRY (&pm_encodings[PM_ENCODING_EUC_JP])
291
292/*
293 * This is the Windows-31J encoding. We need a reference to it to quickly
294 * process regular expression modifiers.
295 */
296#define PM_ENCODING_WINDOWS_31J_ENTRY (&pm_encodings[PM_ENCODING_WINDOWS_31J])
297
298/*
299 * Parse the given name of an encoding and return a pointer to the corresponding
300 * encoding struct if one can be found, otherwise return NULL.
301 */
302const pm_encoding_t * pm_encoding_find(const uint8_t *start, const uint8_t *end);
303
304#endif
A header file that defines macros to exclude certain features of the prism library.
#define PRISM_INLINE
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
Definition inline.h:12
C99 shim for <stdbool.h>.