1#include "prism/internal/regexp.h"
5#include "prism/internal/assert.h"
6#include "prism/internal/buffer.h"
7#include "prism/internal/char.h"
8#include "prism/internal/diagnostic.h"
9#include "prism/internal/encoding.h"
10#include "prism/internal/memchr.h"
11#include "prism/internal/parser.h"
12#include "prism/internal/stringy.h"
13#include "prism/internal/strncasecmp.h"
18#define PM_REGEXP_PARSE_DEPTH_MAX 4096
135pm_regexp_parse_error(
pm_regexp_parser_t *regexp_parser,
const uint8_t *start,
const uint8_t *end,
const char *message) {
138 pm_diagnostic_list_append_format(&parser->metadata_arena, &parser->error_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), PM_ERR_REGEXP_PARSE_ERROR, message);
155 pm_parser_err_stack_exhausted(parser, (uint32_t) (regexp_parser->
start - parser->start), (uint32_t) (regexp_parser->
end - regexp_parser->
start));
157 pm_parser_err_stack_exhausted(parser, (uint32_t) (parser->current.start - parser->start), (uint32_t) (parser->current.end - parser->current.start));
166#define pm_regexp_parse_error_format(regexp_parser_, err_start_, err_end_, diag_id, ...) \
168 if ((regexp_parser_)->report_errors) { \
169 pm_parser_t *parser_ = (regexp_parser_)->parser; \
170 pm_diagnostic_list_append_format(&parser_->metadata_arena, &parser_->error_list, (uint32_t) ((err_start_) - parser_->start), (uint32_t) ((err_end_) - (err_start_)), diag_id, __VA_ARGS__); \
179pm_regexp_parser_named_capture(
pm_regexp_parser_t *parser,
const uint8_t *start,
const uint8_t *end) {
181 pm_string_shared_init(&
string, start, end);
198 if (!pm_regexp_char_is_eof(parser) && *parser->
cursor == value) {
210 if (!pm_regexp_char_is_eof(parser) && *parser->
cursor == value) {
222 if (pm_regexp_char_is_eof(parser)) {
261 pm_regexp_hex_group_boundary(parser);
269pm_regexp_hex_digit_value(uint8_t
byte) {
270 if (
byte >=
'0' &&
byte <=
'9')
return byte -
'0';
271 if (
byte >=
'a' &&
byte <=
'f')
return byte -
'a' + 10;
272 if (
byte >=
'A' &&
byte <=
'F')
return byte -
'A' + 10;
310 const uint8_t *savepoint = parser->
cursor;
313 PM_REGEXP_RANGE_QUANTIFIER_STATE_START,
314 PM_REGEXP_RANGE_QUANTIFIER_STATE_MINIMUM,
315 PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM,
316 PM_REGEXP_RANGE_QUANTIFIER_STATE_COMMA
317 } state = PM_REGEXP_RANGE_QUANTIFIER_STATE_START;
321 parser->
cursor = savepoint;
326 case PM_REGEXP_RANGE_QUANTIFIER_STATE_START:
327 switch (*parser->
cursor) {
328 case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
330 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_MINIMUM;
334 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_COMMA;
337 parser->
cursor = savepoint;
341 case PM_REGEXP_RANGE_QUANTIFIER_STATE_MINIMUM:
342 switch (*parser->
cursor) {
343 case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
348 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM;
354 parser->
cursor = savepoint;
358 case PM_REGEXP_RANGE_QUANTIFIER_STATE_COMMA:
359 switch (*parser->
cursor) {
360 case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
362 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM;
365 parser->
cursor = savepoint;
369 case PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM:
370 switch (*parser->
cursor) {
371 case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
378 parser->
cursor = savepoint;
398 while (!pm_regexp_char_is_eof(parser)) {
399 switch (*parser->
cursor) {
407 if (!pm_regexp_parse_range_quantifier(parser))
return false;
424 if (!pm_regexp_char_expect(parser,
':')) {
428 pm_regexp_char_accept(parser,
'^');
431 pm_regexp_char_find(parser,
':') &&
432 pm_regexp_char_expect(parser,
']') &&
433 pm_regexp_char_expect(parser,
']')
449 PM_REGEXP_PROPERTY_POSIX,
450 PM_REGEXP_PROPERTY_SCRIPT,
451 PM_REGEXP_PROPERTY_UNICODE
452} pm_regexp_property_type_t;
458static pm_regexp_property_type_t
459pm_regexp_classify_property(
const uint8_t *name,
size_t length) {
461 if (length > 0 && name[0] ==
'^') {
466#define PM_REGEXP_CASECMP(str_) (pm_strncasecmp(name, (const uint8_t *) (str_), length) == 0)
470 if (PM_REGEXP_CASECMP(
"Han"))
return PM_REGEXP_PROPERTY_SCRIPT;
473 if (PM_REGEXP_CASECMP(
"Word"))
return PM_REGEXP_PROPERTY_POSIX;
477 switch (name[0] | 0x20) {
479 if (PM_REGEXP_CASECMP(
"Alnum"))
return PM_REGEXP_PROPERTY_POSIX;
480 if (PM_REGEXP_CASECMP(
"Alpha"))
return PM_REGEXP_PROPERTY_POSIX;
481 if (PM_REGEXP_CASECMP(
"ASCII"))
return PM_REGEXP_PROPERTY_POSIX;
484 if (PM_REGEXP_CASECMP(
"Blank"))
return PM_REGEXP_PROPERTY_POSIX;
487 if (PM_REGEXP_CASECMP(
"Cntrl"))
return PM_REGEXP_PROPERTY_POSIX;
490 if (PM_REGEXP_CASECMP(
"Digit"))
return PM_REGEXP_PROPERTY_POSIX;
493 if (PM_REGEXP_CASECMP(
"Graph"))
return PM_REGEXP_PROPERTY_POSIX;
494 if (PM_REGEXP_CASECMP(
"Greek"))
return PM_REGEXP_PROPERTY_SCRIPT;
497 if (PM_REGEXP_CASECMP(
"Lower"))
return PM_REGEXP_PROPERTY_POSIX;
498 if (PM_REGEXP_CASECMP(
"Latin"))
return PM_REGEXP_PROPERTY_SCRIPT;
501 if (PM_REGEXP_CASECMP(
"Print"))
return PM_REGEXP_PROPERTY_POSIX;
502 if (PM_REGEXP_CASECMP(
"Punct"))
return PM_REGEXP_PROPERTY_POSIX;
505 if (PM_REGEXP_CASECMP(
"Space"))
return PM_REGEXP_PROPERTY_POSIX;
508 if (PM_REGEXP_CASECMP(
"Upper"))
return PM_REGEXP_PROPERTY_POSIX;
513 if (PM_REGEXP_CASECMP(
"XDigit"))
return PM_REGEXP_PROPERTY_POSIX;
516 if (PM_REGEXP_CASECMP(
"Hiragana"))
return PM_REGEXP_PROPERTY_SCRIPT;
517 if (PM_REGEXP_CASECMP(
"Katakana"))
return PM_REGEXP_PROPERTY_SCRIPT;
518 if (PM_REGEXP_CASECMP(
"Cyrillic"))
return PM_REGEXP_PROPERTY_SCRIPT;
522#undef PM_REGEXP_CASECMP
525 return PM_REGEXP_PROPERTY_UNICODE;
539 PRISM_ASSERT(*parser->
cursor ==
'p' || *parser->
cursor ==
'P');
542 const uint8_t *name_start = parser->
cursor + 2;
543 const uint8_t *search = name_start;
545 while (search < parser->end && *search !=
'}') search++;
547 if (search < parser->end) {
548 size_t name_length = (size_t) (search - name_start);
551 pm_regexp_property_type_t
type = pm_regexp_classify_property(name_start, name_length);
554 if (type >= PM_REGEXP_PROPERTY_SCRIPT && parser->
property_name == NULL) {
560 if (type == PM_REGEXP_PROPERTY_UNICODE) {
568 parser->
cursor = search + 1;
586 const uint8_t *escape_start = parser->
cursor - 2;
588 if (pm_regexp_char_is_eof(parser)) {
589 pm_regexp_parse_error(parser, escape_start, parser->
cursor,
"invalid Unicode escape");
593 if (*parser->
cursor ==
'{') {
597 while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->
cursor)) {
601 bool has_codepoint =
false;
603 while (!pm_regexp_char_is_eof(parser) && *parser->
cursor !=
'}') {
606 size_t hex_count = 0;
609 while (!pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->
cursor)) >= 0) {
610 value = (value << 4) | (uint32_t) digit;
615 if (hex_count == 0) {
617 while (!pm_regexp_char_is_eof(parser) && *parser->
cursor !=
'}') {
621 const uint8_t *escape_end = parser->
cursor;
622 if (!pm_regexp_char_is_eof(parser)) {
627 pm_regexp_parse_error_format(parser, escape_start, escape_end, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (
int) (escape_end - escape_start), (
const char *) escape_start);
632 pm_regexp_parse_error(parser, escape_start, parser->
cursor,
"invalid Unicode range");
640 pm_regexp_hex_group_boundary(parser);
644 if (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF)) {
648 has_codepoint =
true;
651 while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->
cursor)) {
656 if (pm_regexp_char_is_eof(parser)) {
657 pm_regexp_parse_error(parser, escape_start, parser->
cursor,
"unterminated Unicode escape");
659 if (!has_codepoint) {
660 pm_regexp_parse_error_format(parser, escape_start, parser->
cursor + 1, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (
int) (parser->
cursor + 1 - escape_start), (
const char *) escape_start);
667 size_t hex_count = 0;
670 while (hex_count < 4 && !pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->
cursor)) >= 0) {
671 value = (value << 4) | (uint32_t) digit;
677 pm_regexp_parse_error(parser, escape_start, parser->
cursor,
"invalid Unicode escape");
678 }
else if (value >= 0x80) {
682 pm_regexp_hex_group_boundary(parser);
686 if (hex_count == 4 && (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF))) {
704 if (!pm_regexp_char_is_eof(parser)) {
705 int digit = pm_regexp_hex_digit_value(*parser->
cursor);
710 if (!pm_regexp_char_is_eof(parser)) {
711 digit = pm_regexp_hex_digit_value(*parser->
cursor);
713 value = (value << 4) | digit;
721 pm_regexp_track_hex_escape(parser, (uint8_t) value);
736 if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) {
737 pm_regexp_parse_error(parser, parser->
start, parser->
end,
"parse depth limit over");
741 if (pm_parser_stack_exhausted(parser->
parser)) {
742 pm_regexp_parse_stack_exhausted(parser);
746 if (pm_regexp_char_is_eof(parser))
return true;
748 switch (*parser->
cursor) {
751 pm_regexp_parse_unicode_escape(parser);
755 pm_regexp_parse_property_escape(parser);
759 pm_regexp_parse_hex_escape(parser);
765 if (!pm_regexp_char_is_eof(parser)) {
766 if (*parser->
cursor ==
'\\') {
770 if (!pm_regexp_parse_backslash_escape(parser, (uint16_t) (depth + 1)))
return false;
775 pm_regexp_track_hex_escape(parser, 0x80);
785 if (!pm_regexp_char_is_eof(parser)) {
786 if (*parser->
cursor ==
'\\') {
788 if (!pm_regexp_parse_backslash_escape(parser, (uint16_t) (depth + 1)))
return false;
800 if (!pm_regexp_char_is_eof(parser)) {
801 if (*parser->
cursor ==
'\\') {
803 if (!pm_regexp_parse_backslash_escape(parser, (uint16_t) (depth + 1)))
return false;
810 pm_regexp_hex_group_boundary(parser);
825 uint8_t
byte = *cursor;
827 size_t width = pm_encoding_char_width(parser->
encoding, cursor, (ptrdiff_t) (parser->
end - cursor));
829 parser->
cursor += width - 1;
830 }
else if (width == 0) {
832 pm_regexp_parse_error_format(parser, cursor, cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->
encoding->name);
843 pm_regexp_char_accept(parser,
'^');
845 while (!pm_regexp_char_is_eof(parser) && *parser->
cursor !=
']') {
846 switch (*parser->
cursor++) {
848 if (!pm_regexp_parse_lbracket(parser, (uint16_t) (depth + 1)))
return false;
851 if (!pm_regexp_parse_backslash_escape(parser, depth))
return false;
857 if (*(parser->
cursor - 1) >= 0x80) {
860 pm_regexp_parse_invalid_multibyte(parser, parser->
cursor - 1);
865 return pm_regexp_char_expect(parser,
']');
873 if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) {
874 pm_regexp_parse_error(parser, parser->
start, parser->
end,
"parse depth limit over");
878 if (pm_parser_stack_exhausted(parser->
parser)) {
879 pm_regexp_parse_stack_exhausted(parser);
885 pm_regexp_parse_error(parser, parser->
cursor - 1, parser->
cursor,
"empty char-class");
889 const uint8_t *reset = parser->
cursor;
893 if (pm_regexp_parse_posix_class(parser))
return true;
898 return pm_regexp_parse_character_set(parser, depth);
911 PM_REGEXP_OPTION_STATE_INVALID,
912 PM_REGEXP_OPTION_STATE_TOGGLEABLE,
913 PM_REGEXP_OPTION_STATE_ADDABLE,
914 PM_REGEXP_OPTION_STATE_ADDED,
915 PM_REGEXP_OPTION_STATE_REMOVED
916} pm_regexp_option_state_t;
922#define PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM 'a'
925#define PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM 'x'
928#define PRISM_REGEXP_OPTION_STATE_SLOTS (PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM + 1)
935 uint8_t
values[PRISM_REGEXP_OPTION_STATE_SLOTS];
943 memset(options, PM_REGEXP_OPTION_STATE_INVALID,
sizeof(uint8_t) * PRISM_REGEXP_OPTION_STATE_SLOTS);
944 options->
values[
'i' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_TOGGLEABLE;
945 options->
values[
'm' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_TOGGLEABLE;
946 options->
values[
'x' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_TOGGLEABLE;
947 options->
values[
'd' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_ADDABLE;
948 options->
values[
'a' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_ADDABLE;
949 options->
values[
'u' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_ADDABLE;
958 if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
959 key = (uint8_t) (key - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM);
961 switch (options->
values[key]) {
962 case PM_REGEXP_OPTION_STATE_INVALID:
963 case PM_REGEXP_OPTION_STATE_REMOVED:
965 case PM_REGEXP_OPTION_STATE_TOGGLEABLE:
966 case PM_REGEXP_OPTION_STATE_ADDABLE:
967 options->
values[key] = PM_REGEXP_OPTION_STATE_ADDED;
969 case PM_REGEXP_OPTION_STATE_ADDED:
983 if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
984 key = (uint8_t) (key - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM);
986 switch (options->
values[key]) {
987 case PM_REGEXP_OPTION_STATE_INVALID:
988 case PM_REGEXP_OPTION_STATE_ADDABLE:
990 case PM_REGEXP_OPTION_STATE_TOGGLEABLE:
991 case PM_REGEXP_OPTION_STATE_ADDED:
992 case PM_REGEXP_OPTION_STATE_REMOVED:
993 options->
values[key] = PM_REGEXP_OPTION_STATE_REMOVED;
1006 if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
1007 key = (uint8_t) (key - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM);
1008 return options->
values[key];
1042 if (pm_parser_stack_exhausted(parser->
parser)) {
1043 pm_regexp_parse_stack_exhausted(parser);
1047 const uint8_t *group_start = parser->
cursor;
1050 pm_regexp_options_init(&options);
1053 if (pm_regexp_char_accept(parser,
'?')) {
1054 if (pm_regexp_char_is_eof(parser)) {
1055 pm_regexp_parse_error(parser, group_start, parser->
cursor,
"end pattern in group");
1059 switch (*parser->
cursor) {
1062 if (pm_regexp_char_is_eof(parser)) {
1063 pm_regexp_parse_error(parser, group_start, parser->
cursor,
"end pattern in group");
1068 bool escaped =
false;
1074 if (!escaped && *parser->
cursor ==
')') {
1079 size_t width = pm_encoding_char_width(parser->
encoding, parser->
cursor, (ptrdiff_t) (parser->
end - parser->
cursor));
1081 if (*parser->
cursor >= 0x80) {
1083 pm_regexp_parse_error_format(parser, parser->
cursor, parser->
cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->
encoding->name);
1090 escaped = (width == 1) && (*parser->
cursor ==
'\\');
1099 bool found = pm_regexp_char_find(parser,
')');
1101 while (found && (parser->
start <= parser->
cursor - 2) && (*(parser->
cursor - 2) ==
'\\')) {
1102 found = pm_regexp_char_find(parser,
')');
1117 if (pm_regexp_char_is_eof(parser)) {
1118 pm_regexp_parse_error(parser, group_start, parser->
cursor,
"end pattern with unmatched parenthesis");
1122 switch (*parser->
cursor) {
1128 const uint8_t *start = parser->
cursor;
1129 if (!pm_regexp_char_find(parser,
'>')) {
1133 if (parser->
cursor - start == 1) {
1134 pm_regexp_parse_error(parser, start, parser->
cursor,
"group name is empty");
1138 pm_regexp_parser_named_capture(parser, start, parser->
cursor - 1);
1146 const uint8_t *start = ++parser->
cursor;
1147 if (!pm_regexp_char_find(parser,
'\'')) {
1152 pm_regexp_parser_named_capture(parser, start, parser->
cursor - 1);
1158 if (!pm_regexp_char_find(parser,
')')) {
1162 case 'i':
case 'm':
case 'x':
case 'd':
case 'a':
case 'u':
1163 while (!pm_regexp_char_is_eof(parser) && *parser->
cursor !=
'-' && *parser->
cursor !=
':' && *parser->
cursor !=
')') {
1164 if (!pm_regexp_options_add(&options, *parser->
cursor)) {
1170 if (pm_regexp_char_is_eof(parser)) {
1177 if (*parser->
cursor ==
')') {
1178 if (pm_regexp_options_state(&options,
'x') == PM_REGEXP_OPTION_STATE_ADDED) {
1187 if (*parser->
cursor !=
'-')
break;
1192 while (!pm_regexp_char_is_eof(parser) && *parser->
cursor !=
':' && *parser->
cursor !=
')') {
1193 if (!pm_regexp_options_remove(&options, *parser->
cursor)) {
1199 if (pm_regexp_char_is_eof(parser)) {
1206 if (*parser->
cursor ==
')') {
1207 switch (pm_regexp_options_state(&options,
'x')) {
1208 case PM_REGEXP_OPTION_STATE_ADDED:
1211 case PM_REGEXP_OPTION_STATE_REMOVED:
1223 pm_regexp_parse_error(parser, parser->
cursor - 1, parser->
cursor,
"undefined group option");
1229 switch (pm_regexp_options_state(&options,
'x')) {
1230 case PM_REGEXP_OPTION_STATE_ADDED:
1233 case PM_REGEXP_OPTION_STATE_REMOVED:
1239 while (!pm_regexp_char_is_eof(parser) && *parser->
cursor !=
')') {
1240 if (!pm_regexp_parse_expression(parser, (uint16_t) (depth + 1))) {
1244 pm_regexp_char_accept(parser,
'|');
1249 if (pm_regexp_char_expect(parser,
')'))
return true;
1251 pm_regexp_parse_error(parser, group_start, parser->
cursor,
"end pattern with unmatched parenthesis");
1269 switch (*parser->
cursor) {
1273 return pm_regexp_parse_quantifier(parser);
1276 return pm_regexp_parse_backslash_escape(parser, depth) && pm_regexp_parse_quantifier(parser);
1279 return pm_regexp_parse_group(parser, depth) && pm_regexp_parse_quantifier(parser);
1282 return pm_regexp_parse_lbracket(parser, depth) && pm_regexp_parse_quantifier(parser);
1287 pm_regexp_parse_error(parser, parser->
cursor - 1, parser->
cursor,
"target of repeat operator is not specified");
1291 pm_regexp_parse_error(parser, parser->
cursor - 1, parser->
cursor,
"unmatched close parenthesis");
1295 if (!pm_regexp_char_find(parser,
'\n')) parser->
cursor = parser->
end;
1302 width = pm_encoding_utf_8_char_width(parser->
cursor, (ptrdiff_t) (parser->
end - parser->
cursor));
1309 if (pm_encoding_multibyte_p(parser->
encoding)) {
1313 pm_regexp_parse_error_format(parser, parser->
cursor, parser->
cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->
encoding->name);
1320 return pm_regexp_parse_quantifier(parser);
1326 for (
size_t i = 0; i < width; i++) {
1331 return pm_regexp_parse_quantifier(parser);
1342 if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) {
1343 pm_regexp_parse_error(parser, parser->
start, parser->
end,
"parse depth limit over");
1347 if (!pm_regexp_parse_item(parser, depth)) {
1351 while (!pm_regexp_char_is_eof(parser) && *parser->
cursor !=
')' && *parser->
cursor !=
'|') {
1352 if (!pm_regexp_parse_item(parser, depth)) {
1369 if (pm_regexp_char_is_eof(parser))
return true;
1370 if (!pm_regexp_parse_expression(parser, 0))
return false;
1371 }
while (pm_regexp_char_accept(parser,
'|'));
1373 return pm_regexp_char_is_eof(parser);
1386 const uint8_t *data = (
const uint8_t *) pm_buffer_value(buffer);
1387 size_t len = pm_buffer_length(buffer);
1391 size_t group_start = i;
1392 while (i <
len && data[i] != 0x00) i++;
1394 for (
size_t j = group_start; j < i; ) {
1395 size_t width = pm_encoding_char_width(encoding, data + j, (ptrdiff_t) (i - j));
1396 if (width == 0)
return false;
1411pm_regexp_format_for_error(
pm_buffer_t *buffer,
const pm_encoding_t *encoding,
const uint8_t *source,
size_t length) {
1414 if (encoding == PM_ENCODING_UTF_8_ENTRY) {
1415 pm_buffer_append_string(buffer, (
const char *) source, length);
1419 while (index < length) {
1420 if (source[index] < 0x80) {
1421 pm_buffer_append_byte(buffer, source[index]);
1423 }
else if (pm_encoding_multibyte_p(encoding)) {
1424 size_t width = pm_encoding_char_width(encoding, source + index, (ptrdiff_t) (length - index));
1427 pm_buffer_append_string(buffer,
"\\x{", 3);
1428 for (
size_t i = 0; i < width; i++) {
1429 pm_buffer_append_format(buffer,
"%02X", source[index + i]);
1431 pm_buffer_append_byte(buffer,
'}');
1434 pm_buffer_append_format(buffer,
"\\x%02X", source[index]);
1438 pm_buffer_append_format(buffer,
"\\x%02X", source[index]);
1447#define PM_REGEXP_ENCODING_ERROR(parser, diag_id, ...) \
1448 pm_diagnostic_list_append_format( \
1449 &(parser)->parser->metadata_arena, \
1450 &(parser)->parser->error_list, \
1451 (uint32_t) ((parser)->node_start - (parser)->parser->start), \
1452 (uint32_t) ((parser)->node_end - (parser)->node_start), \
1453 diag_id, __VA_ARGS__)
1474static pm_node_flags_t
1475pm_regexp_validate_encoding_modifier(
pm_regexp_parser_t *parser,
bool ascii_only, pm_node_flags_t flags,
char modifier,
const pm_encoding_t *modifier_encoding,
const char *source_start,
int source_length) {
1484 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
1486 source_length, source_start);
1488 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
1490 source_length, source_start);
1492 return modifier ==
'n' ? PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING : flags;
1495 if (parser->
encoding == PM_ENCODING_US_ASCII_ENTRY) {
1497 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->
encoding->name);
1499 }
else if (parser->
encoding != modifier_encoding) {
1500 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->
encoding->name);
1502 if (modifier ==
'n' && !ascii_only) {
1504 pm_regexp_format_for_error(&formatted, parser->
encoding, (
const uint8_t *) source_start, (
size_t) source_length);
1505 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_NON_ESCAPED_MBC, (
int) formatted.length, (
const char *) formatted.value);
1506 pm_buffer_cleanup(&formatted);
1515 if (modifier ==
'n') {
1517 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
1519 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
1522 if (!pm_regexp_validate_hex_escapes(modifier_encoding, &parser->
hex_escape_buffer)) {
1523 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1532 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
1533 }
else if (parser->
encoding != PM_ENCODING_UTF_8_ENTRY) {
1534 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
1538 if (modifier !=
'n' && !pm_regexp_validate_hex_escapes(modifier_encoding, &parser->
hex_escape_buffer)) {
1539 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1561static pm_node_flags_t
1562pm_regexp_validate_encoding(
pm_regexp_parser_t *parser,
bool ascii_only, pm_node_flags_t flags,
const char *source_start,
int source_length) {
1571 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, source_length, source_start);
1576 if (flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) {
1577 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags,
'n', PM_ENCODING_ASCII_8BIT_ENTRY, source_start, source_length);
1579 if (flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) {
1580 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags,
'u', PM_ENCODING_UTF_8_ENTRY, source_start, source_length);
1582 if (flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) {
1583 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags,
'e', PM_ENCODING_EUC_JP_ENTRY, source_start, source_length);
1585 if (flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) {
1586 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags,
's', PM_ENCODING_WINDOWS_31J_ENTRY, source_start, source_length);
1592 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->
encoding->name);
1600 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
1602 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING;
1609 if (parser->
encoding != PM_ENCODING_US_ASCII_ENTRY &&
1610 parser->
encoding != PM_ENCODING_ASCII_8BIT_ENTRY &&
1612 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1614 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
1616 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
1624 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1627 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
1628 }
else if (parser->
encoding == PM_ENCODING_US_ASCII_ENTRY) {
1629 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_BINARY_ENCODING;
1632 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1648pm_regexp_parse(
pm_parser_t *parser, pm_regular_expression_node_t *node, pm_regexp_name_callback_t name_callback,
void *name_data) {
1651 bool extended_mode = PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EXTENDED);
1652 pm_node_flags_t flags = PM_NODE_FLAGS(node);
1662 .end = source + size,
1663 .extended_mode = extended_mode,
1664 .encoding_changed = parser->encoding_changed,
1665 .encoding = parser->encoding,
1666 .name_callback = NULL,
1668 .report_errors =
true,
1669 .node_start = node_start,
1670 .node_end = node_end,
1671 .has_unicode_escape =
false,
1672 .has_hex_escape =
false,
1673 .last_escape_was_unicode =
false,
1674 .explicit_encoding = NULL,
1675 .has_property_escape =
false,
1676 .has_unicode_property_escape =
false,
1677 .property_name = NULL,
1678 .property_name_length = 0,
1679 .unicode_property_name = NULL,
1680 .unicode_property_name_length = 0,
1681 .non_ascii_literal_count = 0,
1682 .invalid_unicode_range =
false,
1683 .hex_escape_buffer = { 0 },
1684 .hex_group_active =
false,
1685 .has_invalid_multibyte =
false,
1688 pm_regexp_parse_pattern(®exp_parser);
1697 const char *error_source = (
const char *) pm_string_source(&node->
unescaped);
1698 int error_source_length = (int) pm_string_length(&node->
unescaped);
1699 pm_node_flags_t encoding_flags = pm_regexp_validate_encoding(®exp_parser, ascii_only, flags, error_source, error_source_length);
1703 if (name_callback != NULL) {
1704 pm_regexp_parse_named_captures(parser, pm_string_source(&node->
unescaped), pm_string_length(&node->
unescaped), extended_mode, name_callback, name_data);
1707 return encoding_flags;
1721pm_regexp_parse_named_captures(
pm_parser_t *parser,
const uint8_t *source,
size_t size,
bool extended_mode, pm_regexp_name_callback_t name_callback,
void *name_data) {
1726 .end = source + size,
1727 .extended_mode = extended_mode,
1728 .encoding_changed = parser->encoding_changed,
1729 .encoding = parser->encoding,
1730 .name_callback = name_callback,
1731 .name_data = name_data,
1732 .report_errors =
false,
1733 .has_unicode_escape =
false,
1734 .has_hex_escape =
false,
1735 .last_escape_was_unicode =
false,
1736 .explicit_encoding = NULL,
1737 .has_property_escape =
false,
1738 .has_unicode_property_escape =
false,
1739 .property_name = NULL,
1740 .property_name_length = 0,
1741 .unicode_property_name = NULL,
1742 .unicode_property_name_length = 0,
1743 .non_ascii_literal_count = 0,
1744 .invalid_unicode_range =
false,
1745 .hex_escape_buffer = { 0 },
1746 .hex_group_active =
false,
1747 .has_invalid_multibyte =
false,
1750 pm_regexp_parse_pattern(®exp_parser);
#define PRISM_FALLTHROUGH
We use -Wimplicit-fallthrough to guard potentially unintended fall-through between cases of a switch.
int len
Length of the buffer.
#define PRISM_INLINE
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
VALUE type(ANYARGS)
ANYARGS-ed function type.
uint32_t start
The offset of the location from the start of the source.
uint32_t length
The length of the location.
pm_location_t location
This is the location of the node in the source.
This is the set of options that are configurable on the regular expression.
uint8_t values[PRISM_REGEXP_OPTION_STATE_SLOTS]
The current state of each option.
This is the parser that is going to handle parsing regular expressions.
const uint8_t * cursor
A pointer to the current position in the source.
bool has_unicode_escape
Whether a \u{...} escape with value >= 0x80 was seen.
size_t unicode_property_name_length
Length of the first Unicode-only property name found.
bool has_hex_escape
Whether a \xNN escape (or \M-x, etc.) with value >= 0x80 was seen.
const uint8_t * start
A pointer to the start of the source that we are parsing.
const uint8_t * end
A pointer to the end of the source that we are parsing.
void * name_data
The data to pass to the name callback.
bool hex_group_active
Whether we are accumulating consecutive hex escape bytes.
pm_buffer_t hex_escape_buffer
Buffer of hex escape byte values >= 0x80, separated by 0x00 sentinels.
bool extended_mode
Whether or not the regular expression currently being parsed is in extended mode, wherein whitespace ...
pm_parser_t * parser
The parser that is currently being used.
size_t property_name_length
Length of the first non-POSIX property name found.
bool invalid_unicode_range
Whether a \u escape with invalid range (surrogate or > 0x10FFFF) was seen.
const uint8_t * property_name
Pointer to the first non-POSIX property name (for /n error messages).
const pm_encoding_t * encoding
The encoding of the source.
const pm_encoding_t * explicit_encoding
The explicit encoding determined by escape sequences.
const uint8_t * node_end
The end of the regexp node (for error locations).
uint32_t non_ascii_literal_count
Count of non-ASCII literal bytes (not from escapes).
const uint8_t * node_start
The start of the regexp node (for error locations).
bool report_errors
Whether syntax errors are reported.
const uint8_t * unicode_property_name
Pointer to the first Unicode-only property name (for /e, /s error messages).
pm_regexp_name_callback_t name_callback
The callback to call when a named capture group is found.
bool encoding_changed
Whether the encoding has changed from the default.
bool has_unicode_property_escape
Whether a Unicode-only property escape was found (not POSIX or script).
bool has_invalid_multibyte
Whether an invalid multibyte character was found during parsing.
bool last_escape_was_unicode
Tracks whether the last encoding-setting escape was \u (true) or \x (false).
bool has_property_escape
Whether any \p{...} or \P{...} property escape was found.
pm_node_t base
The embedded base node.
pm_string_t unescaped
RegularExpressionNode::unescaped.
pm_location_t content_loc
RegularExpressionNode::content_loc.
A view of a sequence of bytes.