Ruby 4.1.0dev (2026-11-17 revision 373d8c72a360f0d6c3be9cd13d1f0c27360e5d90)
regexp.c
1#include "prism/internal/regexp.h"
2
5#include "prism/internal/assert.h"
6#include "prism/internal/buffer.h"
7#include "prism/internal/char.h"
8#include "prism/internal/diagnostic.h"
9#include "prism/internal/encoding.h"
10#include "prism/internal/memchr.h"
11#include "prism/internal/parser.h"
12#include "prism/internal/stringy.h"
13#include "prism/internal/strncasecmp.h"
14
15#include <string.h>
16
18#define PM_REGEXP_PARSE_DEPTH_MAX 4096
19
23typedef struct {
26
28 const uint8_t *start;
29
31 const uint8_t *cursor;
32
34 const uint8_t *end;
35
38
40 pm_regexp_name_callback_t name_callback;
41
43 void *name_data;
44
46 const uint8_t *node_start;
47
49 const uint8_t *node_end;
50
57
64 const uint8_t *property_name;
65
68
73 const uint8_t *unicode_property_name;
74
77
80
83
89
92
100
103
106
112
115
118
121
124
128
134static PRISM_INLINE void
135pm_regexp_parse_error(pm_regexp_parser_t *regexp_parser, const uint8_t *start, const uint8_t *end, const char *message) {
136 if (regexp_parser->report_errors) {
137 pm_parser_t *parser = regexp_parser->parser;
138 pm_diagnostic_list_append_format(&parser->metadata_arena, &parser->error_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), PM_ERR_REGEXP_PARSE_ERROR, message);
139 }
140}
141
150static void
151pm_regexp_parse_stack_exhausted(pm_regexp_parser_t *regexp_parser) {
152 pm_parser_t *parser = regexp_parser->parser;
153
154 if (regexp_parser->report_errors) {
155 pm_parser_err_stack_exhausted(parser, (uint32_t) (regexp_parser->start - parser->start), (uint32_t) (regexp_parser->end - regexp_parser->start));
156 } else {
157 pm_parser_err_stack_exhausted(parser, (uint32_t) (parser->current.start - parser->start), (uint32_t) (parser->current.end - parser->current.start));
158 }
159}
160
166#define pm_regexp_parse_error_format(regexp_parser_, err_start_, err_end_, diag_id, ...) \
167 do { \
168 if ((regexp_parser_)->report_errors) { \
169 pm_parser_t *parser_ = (regexp_parser_)->parser; \
170 pm_diagnostic_list_append_format(&parser_->metadata_arena, &parser_->error_list, (uint32_t) ((err_start_) - parser_->start), (uint32_t) ((err_end_) - (err_start_)), diag_id, __VA_ARGS__); \
171 } \
172 } while (0)
173
178static void
179pm_regexp_parser_named_capture(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end) {
180 pm_string_t string;
181 pm_string_shared_init(&string, start, end);
182 parser->name_callback(parser->parser, &string, parser->name_data);
183}
184
188static PRISM_INLINE bool
189pm_regexp_char_is_eof(pm_regexp_parser_t *parser) {
190 return parser->cursor >= parser->end;
191}
192
196static PRISM_INLINE bool
197pm_regexp_char_accept(pm_regexp_parser_t *parser, uint8_t value) {
198 if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
199 parser->cursor++;
200 return true;
201 }
202 return false;
203}
204
208static PRISM_INLINE bool
209pm_regexp_char_expect(pm_regexp_parser_t *parser, uint8_t value) {
210 if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
211 parser->cursor++;
212 return true;
213 }
214 return false;
215}
216
220static bool
221pm_regexp_char_find(pm_regexp_parser_t *parser, uint8_t value) {
222 if (pm_regexp_char_is_eof(parser)) {
223 return false;
224 }
225
226 const uint8_t *end = (const uint8_t *) pm_memchr(parser->cursor, value, (size_t) (parser->end - parser->cursor), parser->encoding_changed, parser->encoding);
227 if (end == NULL) {
228 return false;
229 }
230
231 parser->cursor = end + 1;
232 return true;
233}
234
240static PRISM_INLINE void
241pm_regexp_hex_group_boundary(pm_regexp_parser_t *parser) {
242 if (parser->hex_group_active) {
243 pm_buffer_append_byte(&parser->hex_escape_buffer, 0x00);
244 parser->hex_group_active = false;
245 }
246}
247
251static PRISM_INLINE void
252pm_regexp_track_hex_escape(pm_regexp_parser_t *parser, uint8_t byte) {
253 if (byte >= 0x80) {
254 pm_buffer_append_byte(&parser->hex_escape_buffer, byte);
255 parser->hex_group_active = true;
256 parser->has_hex_escape = true;
257
258 parser->explicit_encoding = parser->encoding;
259 parser->last_escape_was_unicode = false;
260 } else {
261 pm_regexp_hex_group_boundary(parser);
262 }
263}
264
268static PRISM_INLINE int
269pm_regexp_hex_digit_value(uint8_t byte) {
270 if (byte >= '0' && byte <= '9') return byte - '0';
271 if (byte >= 'a' && byte <= 'f') return byte - 'a' + 10;
272 if (byte >= 'A' && byte <= 'F') return byte - 'A' + 10;
273 return -1;
274}
275
308static bool
309pm_regexp_parse_range_quantifier(pm_regexp_parser_t *parser) {
310 const uint8_t *savepoint = parser->cursor;
311
312 enum {
313 PM_REGEXP_RANGE_QUANTIFIER_STATE_START,
314 PM_REGEXP_RANGE_QUANTIFIER_STATE_MINIMUM,
315 PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM,
316 PM_REGEXP_RANGE_QUANTIFIER_STATE_COMMA
317 } state = PM_REGEXP_RANGE_QUANTIFIER_STATE_START;
318
319 while (1) {
320 if (parser->cursor >= parser->end) {
321 parser->cursor = savepoint;
322 return true;
323 }
324
325 switch (state) {
326 case PM_REGEXP_RANGE_QUANTIFIER_STATE_START:
327 switch (*parser->cursor) {
328 case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
329 parser->cursor++;
330 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_MINIMUM;
331 break;
332 case ',':
333 parser->cursor++;
334 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_COMMA;
335 break;
336 default:
337 parser->cursor = savepoint;
338 return true;
339 }
340 break;
341 case PM_REGEXP_RANGE_QUANTIFIER_STATE_MINIMUM:
342 switch (*parser->cursor) {
343 case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
344 parser->cursor++;
345 break;
346 case ',':
347 parser->cursor++;
348 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM;
349 break;
350 case '}':
351 parser->cursor++;
352 return true;
353 default:
354 parser->cursor = savepoint;
355 return true;
356 }
357 break;
358 case PM_REGEXP_RANGE_QUANTIFIER_STATE_COMMA:
359 switch (*parser->cursor) {
360 case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
361 parser->cursor++;
362 state = PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM;
363 break;
364 default:
365 parser->cursor = savepoint;
366 return true;
367 }
368 break;
369 case PM_REGEXP_RANGE_QUANTIFIER_STATE_MAXIMUM:
370 switch (*parser->cursor) {
371 case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
372 parser->cursor++;
373 break;
374 case '}':
375 parser->cursor++;
376 return true;
377 default:
378 parser->cursor = savepoint;
379 return true;
380 }
381 break;
382 }
383 }
384
385 return true;
386}
387
396static bool
397pm_regexp_parse_quantifier(pm_regexp_parser_t *parser) {
398 while (!pm_regexp_char_is_eof(parser)) {
399 switch (*parser->cursor) {
400 case '*':
401 case '+':
402 case '?':
403 parser->cursor++;
404 break;
405 case '{':
406 parser->cursor++;
407 if (!pm_regexp_parse_range_quantifier(parser)) return false;
408 break;
409 default:
410 // In this case there is no quantifier.
411 return true;
412 }
413 }
414
415 return true;
416}
417
422static bool
423pm_regexp_parse_posix_class(pm_regexp_parser_t *parser) {
424 if (!pm_regexp_char_expect(parser, ':')) {
425 return false;
426 }
427
428 pm_regexp_char_accept(parser, '^');
429
430 return (
431 pm_regexp_char_find(parser, ':') &&
432 pm_regexp_char_expect(parser, ']') &&
433 pm_regexp_char_expect(parser, ']')
434 );
435}
436
448typedef enum {
449 PM_REGEXP_PROPERTY_POSIX,
450 PM_REGEXP_PROPERTY_SCRIPT,
451 PM_REGEXP_PROPERTY_UNICODE
452} pm_regexp_property_type_t;
453
458static pm_regexp_property_type_t
459pm_regexp_classify_property(const uint8_t *name, size_t length) {
460 // Skip leading '^' for negated properties like \p{^Hiragana}.
461 if (length > 0 && name[0] == '^') {
462 name++;
463 length--;
464 }
465
466#define PM_REGEXP_CASECMP(str_) (pm_strncasecmp(name, (const uint8_t *) (str_), length) == 0)
467
468 switch (length) {
469 case 3:
470 if (PM_REGEXP_CASECMP("Han")) return PM_REGEXP_PROPERTY_SCRIPT;
471 break;
472 case 4:
473 if (PM_REGEXP_CASECMP("Word")) return PM_REGEXP_PROPERTY_POSIX;
474 break;
475 case 5:
476 /* Most properties are length 5, so dispatch on first character. */
477 switch (name[0] | 0x20) {
478 case 'a':
479 if (PM_REGEXP_CASECMP("Alnum")) return PM_REGEXP_PROPERTY_POSIX;
480 if (PM_REGEXP_CASECMP("Alpha")) return PM_REGEXP_PROPERTY_POSIX;
481 if (PM_REGEXP_CASECMP("ASCII")) return PM_REGEXP_PROPERTY_POSIX;
482 break;
483 case 'b':
484 if (PM_REGEXP_CASECMP("Blank")) return PM_REGEXP_PROPERTY_POSIX;
485 break;
486 case 'c':
487 if (PM_REGEXP_CASECMP("Cntrl")) return PM_REGEXP_PROPERTY_POSIX;
488 break;
489 case 'd':
490 if (PM_REGEXP_CASECMP("Digit")) return PM_REGEXP_PROPERTY_POSIX;
491 break;
492 case 'g':
493 if (PM_REGEXP_CASECMP("Graph")) return PM_REGEXP_PROPERTY_POSIX;
494 if (PM_REGEXP_CASECMP("Greek")) return PM_REGEXP_PROPERTY_SCRIPT;
495 break;
496 case 'l':
497 if (PM_REGEXP_CASECMP("Lower")) return PM_REGEXP_PROPERTY_POSIX;
498 if (PM_REGEXP_CASECMP("Latin")) return PM_REGEXP_PROPERTY_SCRIPT;
499 break;
500 case 'p':
501 if (PM_REGEXP_CASECMP("Print")) return PM_REGEXP_PROPERTY_POSIX;
502 if (PM_REGEXP_CASECMP("Punct")) return PM_REGEXP_PROPERTY_POSIX;
503 break;
504 case 's':
505 if (PM_REGEXP_CASECMP("Space")) return PM_REGEXP_PROPERTY_POSIX;
506 break;
507 case 'u':
508 if (PM_REGEXP_CASECMP("Upper")) return PM_REGEXP_PROPERTY_POSIX;
509 break;
510 }
511 break;
512 case 6:
513 if (PM_REGEXP_CASECMP("XDigit")) return PM_REGEXP_PROPERTY_POSIX;
514 break;
515 case 8:
516 if (PM_REGEXP_CASECMP("Hiragana")) return PM_REGEXP_PROPERTY_SCRIPT;
517 if (PM_REGEXP_CASECMP("Katakana")) return PM_REGEXP_PROPERTY_SCRIPT;
518 if (PM_REGEXP_CASECMP("Cyrillic")) return PM_REGEXP_PROPERTY_SCRIPT;
519 break;
520 }
521
522#undef PM_REGEXP_CASECMP
523
524 // Everything else is Unicode-only (general categories, other scripts, etc.).
525 return PM_REGEXP_PROPERTY_UNICODE;
526}
527
537static bool
538pm_regexp_parse_property_escape(pm_regexp_parser_t *parser) {
539 PRISM_ASSERT(*parser->cursor == 'p' || *parser->cursor == 'P');
540
541 if (parser->cursor + 1 < parser->end && parser->cursor[1] == '{') {
542 const uint8_t *name_start = parser->cursor + 2;
543 const uint8_t *search = name_start;
544
545 while (search < parser->end && *search != '}') search++;
546
547 if (search < parser->end) {
548 size_t name_length = (size_t) (search - name_start);
549 parser->has_property_escape = true;
550
551 pm_regexp_property_type_t type = pm_regexp_classify_property(name_start, name_length);
552
553 // Track the first non-POSIX property name (for /n error messages).
554 if (type >= PM_REGEXP_PROPERTY_SCRIPT && parser->property_name == NULL) {
555 parser->property_name = name_start;
556 parser->property_name_length = name_length;
557 }
558
559 // Track the first Unicode-only property name (for /e, /s error messages).
560 if (type == PM_REGEXP_PROPERTY_UNICODE) {
561 parser->has_unicode_property_escape = true;
562 if (parser->unicode_property_name == NULL) {
563 parser->unicode_property_name = name_start;
564 parser->unicode_property_name_length = name_length;
565 }
566 }
567
568 parser->cursor = search + 1; // skip past '}'
569 return true;
570 }
571 }
572
573 // Not a property escape, just skip the single character after '\'.
574 parser->cursor++;
575 return false;
576}
577
584static void
585pm_regexp_parse_unicode_escape(pm_regexp_parser_t *parser) {
586 const uint8_t *escape_start = parser->cursor - 2; // points to '\'
587
588 if (pm_regexp_char_is_eof(parser)) {
589 pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode escape");
590 return;
591 }
592
593 if (*parser->cursor == '{') {
594 parser->cursor++; // skip '{'
595
596 // Skip leading whitespace.
597 while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->cursor)) {
598 parser->cursor++;
599 }
600
601 bool has_codepoint = false;
602
603 while (!pm_regexp_char_is_eof(parser) && *parser->cursor != '}') {
604 // Parse the hex digits to compute the codepoint value.
605 uint32_t value = 0;
606 size_t hex_count = 0;
607
608 int digit;
609 while (!pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->cursor)) >= 0) {
610 value = (value << 4) | (uint32_t) digit;
611 hex_count++;
612 parser->cursor++;
613 }
614
615 if (hex_count == 0) {
616 // Skip to '}' or end of regexp to find the full extent.
617 while (!pm_regexp_char_is_eof(parser) && *parser->cursor != '}') {
618 parser->cursor++;
619 }
620
621 const uint8_t *escape_end = parser->cursor;
622 if (!pm_regexp_char_is_eof(parser)) {
623 escape_end++;
624 parser->cursor++; // skip '}'
625 }
626
627 pm_regexp_parse_error_format(parser, escape_start, escape_end, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (escape_end - escape_start), (const char *) escape_start);
628 return;
629 }
630
631 if (hex_count > 6) {
632 pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode range");
633 }
634
635 // Track encoding state for this codepoint.
636 if (value >= 0x80) {
637 parser->has_unicode_escape = true;
638 parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY;
639 parser->last_escape_was_unicode = true;
640 pm_regexp_hex_group_boundary(parser);
641 }
642
643 // Check for invalid Unicode range (surrogates or > 0x10FFFF).
644 if (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF)) {
645 parser->invalid_unicode_range = true;
646 }
647
648 has_codepoint = true;
649
650 // Skip whitespace between codepoints.
651 while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->cursor)) {
652 parser->cursor++;
653 }
654 }
655
656 if (pm_regexp_char_is_eof(parser)) {
657 pm_regexp_parse_error(parser, escape_start, parser->cursor, "unterminated Unicode escape");
658 } else {
659 if (!has_codepoint) {
660 pm_regexp_parse_error_format(parser, escape_start, parser->cursor + 1, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->cursor + 1 - escape_start), (const char *) escape_start);
661 }
662 parser->cursor++; // skip '}'
663 }
664 } else {
665 // \uNNNN form — need exactly 4 hex digits.
666 uint32_t value = 0;
667 size_t hex_count = 0;
668
669 int digit;
670 while (hex_count < 4 && !pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->cursor)) >= 0) {
671 value = (value << 4) | (uint32_t) digit;
672 hex_count++;
673 parser->cursor++;
674 }
675
676 if (hex_count < 4) {
677 pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode escape");
678 } else if (value >= 0x80) {
679 parser->has_unicode_escape = true;
680 parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY;
681 parser->last_escape_was_unicode = true;
682 pm_regexp_hex_group_boundary(parser);
683 }
684
685 // Check for invalid Unicode range.
686 if (hex_count == 4 && (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF))) {
687 parser->invalid_unicode_range = true;
688 }
689 }
690}
691
692// Forward declaration because character sets can be nested.
693static bool
694pm_regexp_parse_lbracket(pm_regexp_parser_t *parser, uint16_t depth);
695
700static int
701pm_regexp_parse_hex_escape(pm_regexp_parser_t *parser) {
702 int value = -1;
703
704 if (!pm_regexp_char_is_eof(parser)) {
705 int digit = pm_regexp_hex_digit_value(*parser->cursor);
706 if (digit >= 0) {
707 value = digit;
708 parser->cursor++;
709
710 if (!pm_regexp_char_is_eof(parser)) {
711 digit = pm_regexp_hex_digit_value(*parser->cursor);
712 if (digit >= 0) {
713 value = (value << 4) | digit;
714 parser->cursor++;
715 }
716 }
717 }
718 }
719
720 if (value >= 0) {
721 pm_regexp_track_hex_escape(parser, (uint8_t) value);
722 }
723
724 return value;
725}
726
734static bool
735pm_regexp_parse_backslash_escape(pm_regexp_parser_t *parser, uint16_t depth) {
736 if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) {
737 pm_regexp_parse_error(parser, parser->start, parser->end, "parse depth limit over");
738 return false;
739 }
740
741 if (pm_parser_stack_exhausted(parser->parser)) {
742 pm_regexp_parse_stack_exhausted(parser);
743 return false;
744 }
745
746 if (pm_regexp_char_is_eof(parser)) return true;
747
748 switch (*parser->cursor) {
749 case 'u':
750 parser->cursor++; // skip 'u'
751 pm_regexp_parse_unicode_escape(parser);
752 break;
753 case 'p':
754 case 'P':
755 pm_regexp_parse_property_escape(parser);
756 break;
757 case 'x':
758 parser->cursor++; // skip 'x'
759 pm_regexp_parse_hex_escape(parser);
760 break;
761 case 'M':
762 // \M-x produces (x | 0x80), always >= 0x80
763 if (parser->cursor + 2 < parser->end && parser->cursor[1] == '-') {
764 parser->cursor += 2; // skip 'M-'
765 if (!pm_regexp_char_is_eof(parser)) {
766 if (*parser->cursor == '\\') {
767 parser->cursor++;
768 // \M-\C-x or \M-\cx — the resulting byte is always >= 0x80
769 // We just need to track it as a hex escape >= 0x80.
770 if (!pm_regexp_parse_backslash_escape(parser, (uint16_t) (depth + 1))) return false;
771 } else {
772 parser->cursor++;
773 }
774 // \M-x always produces a byte >= 0x80
775 pm_regexp_track_hex_escape(parser, 0x80);
776 }
777 } else {
778 parser->cursor++;
779 }
780 break;
781 case 'C':
782 // \C-x produces (x & 0x1F)
783 if (parser->cursor + 2 < parser->end && parser->cursor[1] == '-') {
784 parser->cursor += 2; // skip 'C-'
785 if (!pm_regexp_char_is_eof(parser)) {
786 if (*parser->cursor == '\\') {
787 parser->cursor++;
788 if (!pm_regexp_parse_backslash_escape(parser, (uint16_t) (depth + 1))) return false;
789 } else {
790 parser->cursor++;
791 }
792 }
793 } else {
794 parser->cursor++;
795 }
796 break;
797 case 'c':
798 // \cx produces (x & 0x1F)
799 parser->cursor++; // skip 'c'
800 if (!pm_regexp_char_is_eof(parser)) {
801 if (*parser->cursor == '\\') {
802 parser->cursor++;
803 if (!pm_regexp_parse_backslash_escape(parser, (uint16_t) (depth + 1))) return false;
804 } else {
805 parser->cursor++;
806 }
807 }
808 break;
809 default:
810 pm_regexp_hex_group_boundary(parser);
811 parser->cursor++;
812 break;
813 }
814
815 return true;
816}
817
823static void
824pm_regexp_parse_invalid_multibyte(pm_regexp_parser_t *parser, const uint8_t *cursor) {
825 uint8_t byte = *cursor;
826 if (byte >= 0x80 && parser->encoding_changed && pm_encoding_multibyte_p(parser->encoding)) {
827 size_t width = pm_encoding_char_width(parser->encoding, cursor, (ptrdiff_t) (parser->end - cursor));
828 if (width > 1) {
829 parser->cursor += width - 1;
830 } else if (width == 0) {
831 parser->has_invalid_multibyte = true;
832 pm_regexp_parse_error_format(parser, cursor, cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
833 }
834 }
835}
836
841static bool
842pm_regexp_parse_character_set(pm_regexp_parser_t *parser, uint16_t depth) {
843 pm_regexp_char_accept(parser, '^');
844
845 while (!pm_regexp_char_is_eof(parser) && *parser->cursor != ']') {
846 switch (*parser->cursor++) {
847 case '[':
848 if (!pm_regexp_parse_lbracket(parser, (uint16_t) (depth + 1))) return false;
849 break;
850 case '\\':
851 if (!pm_regexp_parse_backslash_escape(parser, depth)) return false;
852 break;
853 default:
854 // We've already advanced the cursor by one byte. If the byte
855 // was >= 0x80 in a multibyte encoding, we may need to consume
856 // additional continuation bytes and validate the character.
857 if (*(parser->cursor - 1) >= 0x80) {
858 parser->non_ascii_literal_count++;
859 }
860 pm_regexp_parse_invalid_multibyte(parser, parser->cursor - 1);
861 break;
862 }
863 }
864
865 return pm_regexp_char_expect(parser, ']');
866}
867
871static bool
872pm_regexp_parse_lbracket(pm_regexp_parser_t *parser, uint16_t depth) {
873 if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) {
874 pm_regexp_parse_error(parser, parser->start, parser->end, "parse depth limit over");
875 return false;
876 }
877
878 if (pm_parser_stack_exhausted(parser->parser)) {
879 pm_regexp_parse_stack_exhausted(parser);
880 return false;
881 }
882
883 if ((parser->cursor < parser->end) && parser->cursor[0] == ']') {
884 parser->cursor++;
885 pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "empty char-class");
886 return true;
887 }
888
889 const uint8_t *reset = parser->cursor;
890
891 if ((parser->cursor + 2 < parser->end) && parser->cursor[0] == '[' && parser->cursor[1] == ':') {
892 parser->cursor++;
893 if (pm_regexp_parse_posix_class(parser)) return true;
894
895 parser->cursor = reset;
896 }
897
898 return pm_regexp_parse_character_set(parser, depth);
899}
900
901// Forward declaration here since parsing groups needs to go back up the grammar
902// to parse expressions within them.
903static bool
904pm_regexp_parse_expression(pm_regexp_parser_t *parser, uint16_t depth);
905
910typedef enum {
911 PM_REGEXP_OPTION_STATE_INVALID,
912 PM_REGEXP_OPTION_STATE_TOGGLEABLE,
913 PM_REGEXP_OPTION_STATE_ADDABLE,
914 PM_REGEXP_OPTION_STATE_ADDED,
915 PM_REGEXP_OPTION_STATE_REMOVED
916} pm_regexp_option_state_t;
917
918// These are the options that are configurable on the regular expression (or
919// from within a group).
920
922#define PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM 'a'
923
925#define PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM 'x'
926
928#define PRISM_REGEXP_OPTION_STATE_SLOTS (PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM + 1)
929
933typedef struct {
935 uint8_t values[PRISM_REGEXP_OPTION_STATE_SLOTS];
937
941static void
942pm_regexp_options_init(pm_regexp_options_t *options) {
943 memset(options, PM_REGEXP_OPTION_STATE_INVALID, sizeof(uint8_t) * PRISM_REGEXP_OPTION_STATE_SLOTS);
944 options->values['i' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_TOGGLEABLE;
945 options->values['m' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_TOGGLEABLE;
946 options->values['x' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_TOGGLEABLE;
947 options->values['d' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_ADDABLE;
948 options->values['a' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_ADDABLE;
949 options->values['u' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_ADDABLE;
950}
951
956static bool
957pm_regexp_options_add(pm_regexp_options_t *options, uint8_t key) {
958 if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
959 key = (uint8_t) (key - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM);
960
961 switch (options->values[key]) {
962 case PM_REGEXP_OPTION_STATE_INVALID:
963 case PM_REGEXP_OPTION_STATE_REMOVED:
964 return false;
965 case PM_REGEXP_OPTION_STATE_TOGGLEABLE:
966 case PM_REGEXP_OPTION_STATE_ADDABLE:
967 options->values[key] = PM_REGEXP_OPTION_STATE_ADDED;
968 return true;
969 case PM_REGEXP_OPTION_STATE_ADDED:
970 return true;
971 }
972 }
973
974 return false;
975}
976
981static bool
982pm_regexp_options_remove(pm_regexp_options_t *options, uint8_t key) {
983 if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
984 key = (uint8_t) (key - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM);
985
986 switch (options->values[key]) {
987 case PM_REGEXP_OPTION_STATE_INVALID:
988 case PM_REGEXP_OPTION_STATE_ADDABLE:
989 return false;
990 case PM_REGEXP_OPTION_STATE_TOGGLEABLE:
991 case PM_REGEXP_OPTION_STATE_ADDED:
992 case PM_REGEXP_OPTION_STATE_REMOVED:
993 options->values[key] = PM_REGEXP_OPTION_STATE_REMOVED;
994 return true;
995 }
996 }
997
998 return false;
999}
1000
1004static uint8_t
1005pm_regexp_options_state(pm_regexp_options_t *options, uint8_t key) {
1006 if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
1007 key = (uint8_t) (key - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM);
1008 return options->values[key];
1009 }
1010
1011 return false;
1012}
1013
1035static bool
1036pm_regexp_parse_group(pm_regexp_parser_t *parser, uint16_t depth) {
1037 /* Every level of nested groups passes through here, so this bounds the
1038 * group recursion by the stack budget. The check lives here rather than
1039 * beside the depth check in pm_regexp_parse_expression because there it
1040 * stops that function from being inlined, which adds a frame to every
1041 * level of nesting. */
1042 if (pm_parser_stack_exhausted(parser->parser)) {
1043 pm_regexp_parse_stack_exhausted(parser);
1044 return false;
1045 }
1046
1047 const uint8_t *group_start = parser->cursor;
1048
1049 pm_regexp_options_t options;
1050 pm_regexp_options_init(&options);
1051
1052 // First, parse any options for the group.
1053 if (pm_regexp_char_accept(parser, '?')) {
1054 if (pm_regexp_char_is_eof(parser)) {
1055 pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern in group");
1056 return false;
1057 }
1058
1059 switch (*parser->cursor) {
1060 case '#': { // inline comments
1061 parser->cursor++;
1062 if (pm_regexp_char_is_eof(parser)) {
1063 pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern in group");
1064 return false;
1065 }
1066
1067 if (parser->encoding_changed && pm_encoding_multibyte_p(parser->encoding)) {
1068 bool escaped = false;
1069
1070 // Here we're going to take a slow path and iterate through
1071 // each multibyte character to find the close paren. We do
1072 // this because \ can be a trailing byte in some encodings.
1073 while (parser->cursor < parser->end) {
1074 if (!escaped && *parser->cursor == ')') {
1075 parser->cursor++;
1076 return true;
1077 }
1078
1079 size_t width = pm_encoding_char_width(parser->encoding, parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
1080 if (width == 0) {
1081 if (*parser->cursor >= 0x80) {
1082 parser->has_invalid_multibyte = true;
1083 pm_regexp_parse_error_format(parser, parser->cursor, parser->cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1084 parser->cursor++;
1085 continue;
1086 }
1087 return false;
1088 }
1089
1090 escaped = (width == 1) && (*parser->cursor == '\\');
1091 parser->cursor += width;
1092 }
1093
1094 return false;
1095 } else {
1096 // Here we can take the fast path and use memchr to find the
1097 // next ) because we are safe checking backward for \ since
1098 // it cannot be a trailing character.
1099 bool found = pm_regexp_char_find(parser, ')');
1100
1101 while (found && (parser->start <= parser->cursor - 2) && (*(parser->cursor - 2) == '\\')) {
1102 found = pm_regexp_char_find(parser, ')');
1103 }
1104
1105 return found;
1106 }
1107 }
1108 case ':': // non-capturing group
1109 case '=': // positive lookahead
1110 case '!': // negative lookahead
1111 case '>': // atomic group
1112 case '~': // absence operator
1113 parser->cursor++;
1114 break;
1115 case '<':
1116 parser->cursor++;
1117 if (pm_regexp_char_is_eof(parser)) {
1118 pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern with unmatched parenthesis");
1119 return false;
1120 }
1121
1122 switch (*parser->cursor) {
1123 case '=': // positive lookbehind
1124 case '!': // negative lookbehind
1125 parser->cursor++;
1126 break;
1127 default: { // named capture group
1128 const uint8_t *start = parser->cursor;
1129 if (!pm_regexp_char_find(parser, '>')) {
1130 return false;
1131 }
1132
1133 if (parser->cursor - start == 1) {
1134 pm_regexp_parse_error(parser, start, parser->cursor, "group name is empty");
1135 }
1136
1137 if (parser->name_callback != NULL) {
1138 pm_regexp_parser_named_capture(parser, start, parser->cursor - 1);
1139 }
1140
1141 break;
1142 }
1143 }
1144 break;
1145 case '\'': { // named capture group
1146 const uint8_t *start = ++parser->cursor;
1147 if (!pm_regexp_char_find(parser, '\'')) {
1148 return false;
1149 }
1150
1151 if (parser->name_callback != NULL) {
1152 pm_regexp_parser_named_capture(parser, start, parser->cursor - 1);
1153 }
1154
1155 break;
1156 }
1157 case '(': // conditional expression
1158 if (!pm_regexp_char_find(parser, ')')) {
1159 return false;
1160 }
1161 break;
1162 case 'i': case 'm': case 'x': case 'd': case 'a': case 'u': // options
1163 while (!pm_regexp_char_is_eof(parser) && *parser->cursor != '-' && *parser->cursor != ':' && *parser->cursor != ')') {
1164 if (!pm_regexp_options_add(&options, *parser->cursor)) {
1165 return false;
1166 }
1167 parser->cursor++;
1168 }
1169
1170 if (pm_regexp_char_is_eof(parser)) {
1171 return false;
1172 }
1173
1174 // If we are at the end of the group of options and there is no
1175 // subexpression, then we are going to be setting the options
1176 // for the parent group. In this case we are safe to return now.
1177 if (*parser->cursor == ')') {
1178 if (pm_regexp_options_state(&options, 'x') == PM_REGEXP_OPTION_STATE_ADDED) {
1179 parser->extended_mode = true;
1180 }
1181
1182 parser->cursor++;
1183 return true;
1184 }
1185
1186 // If we hit a -, then we're done parsing options.
1187 if (*parser->cursor != '-') break;
1188
1190 case '-':
1191 parser->cursor++;
1192 while (!pm_regexp_char_is_eof(parser) && *parser->cursor != ':' && *parser->cursor != ')') {
1193 if (!pm_regexp_options_remove(&options, *parser->cursor)) {
1194 return false;
1195 }
1196 parser->cursor++;
1197 }
1198
1199 if (pm_regexp_char_is_eof(parser)) {
1200 return false;
1201 }
1202
1203 // If we are at the end of the group of options and there is no
1204 // subexpression, then we are going to be setting the options
1205 // for the parent group. In this case we are safe to return now.
1206 if (*parser->cursor == ')') {
1207 switch (pm_regexp_options_state(&options, 'x')) {
1208 case PM_REGEXP_OPTION_STATE_ADDED:
1209 parser->extended_mode = true;
1210 break;
1211 case PM_REGEXP_OPTION_STATE_REMOVED:
1212 parser->extended_mode = false;
1213 break;
1214 }
1215
1216 parser->cursor++;
1217 return true;
1218 }
1219
1220 break;
1221 default:
1222 parser->cursor++;
1223 pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "undefined group option");
1224 break;
1225 }
1226 }
1227
1228 bool extended_mode = parser->extended_mode;
1229 switch (pm_regexp_options_state(&options, 'x')) {
1230 case PM_REGEXP_OPTION_STATE_ADDED:
1231 parser->extended_mode = true;
1232 break;
1233 case PM_REGEXP_OPTION_STATE_REMOVED:
1234 parser->extended_mode = false;
1235 break;
1236 }
1237
1238 // Now, parse the expressions within this group.
1239 while (!pm_regexp_char_is_eof(parser) && *parser->cursor != ')') {
1240 if (!pm_regexp_parse_expression(parser, (uint16_t) (depth + 1))) {
1241 parser->extended_mode = extended_mode;
1242 return false;
1243 }
1244 pm_regexp_char_accept(parser, '|');
1245 }
1246
1247 // Finally, make sure we have a closing parenthesis.
1248 parser->extended_mode = extended_mode;
1249 if (pm_regexp_char_expect(parser, ')')) return true;
1250
1251 pm_regexp_parse_error(parser, group_start, parser->cursor, "end pattern with unmatched parenthesis");
1252 return false;
1253}
1254
1267static bool
1268pm_regexp_parse_item(pm_regexp_parser_t *parser, uint16_t depth) {
1269 switch (*parser->cursor) {
1270 case '^':
1271 case '$':
1272 parser->cursor++;
1273 return pm_regexp_parse_quantifier(parser);
1274 case '\\':
1275 parser->cursor++;
1276 return pm_regexp_parse_backslash_escape(parser, depth) && pm_regexp_parse_quantifier(parser);
1277 case '(':
1278 parser->cursor++;
1279 return pm_regexp_parse_group(parser, depth) && pm_regexp_parse_quantifier(parser);
1280 case '[':
1281 parser->cursor++;
1282 return pm_regexp_parse_lbracket(parser, depth) && pm_regexp_parse_quantifier(parser);
1283 case '*':
1284 case '?':
1285 case '+':
1286 parser->cursor++;
1287 pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "target of repeat operator is not specified");
1288 return true;
1289 case ')':
1290 parser->cursor++;
1291 pm_regexp_parse_error(parser, parser->cursor - 1, parser->cursor, "unmatched close parenthesis");
1292 return true;
1293 case '#':
1294 if (parser->extended_mode) {
1295 if (!pm_regexp_char_find(parser, '\n')) parser->cursor = parser->end;
1296 return true;
1297 }
1299 default: {
1300 size_t width;
1301 if (!parser->encoding_changed) {
1302 width = pm_encoding_utf_8_char_width(parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
1303 } else {
1304 width = pm_encoding_char_width(parser->encoding, parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
1305 }
1306
1307 if (width == 0) {
1308 if (*parser->cursor >= 0x80 && parser->encoding_changed) {
1309 if (pm_encoding_multibyte_p(parser->encoding)) {
1310 // Invalid multibyte character in a multibyte encoding.
1311 // Emit the error at the byte location immediately.
1312 parser->has_invalid_multibyte = true;
1313 pm_regexp_parse_error_format(parser, parser->cursor, parser->cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1314 } else {
1315 // Non-ASCII byte in a single-byte encoding (e.g.,
1316 // US-ASCII). Count it for later error reporting.
1317 parser->non_ascii_literal_count++;
1318 }
1319 parser->cursor++;
1320 return pm_regexp_parse_quantifier(parser);
1321 }
1322 return false;
1323 }
1324
1325 // Count non-ASCII literal bytes.
1326 for (size_t i = 0; i < width; i++) {
1327 if (parser->cursor[i] >= 0x80) parser->non_ascii_literal_count++;
1328 }
1329
1330 parser->cursor += width;
1331 return pm_regexp_parse_quantifier(parser);
1332 }
1333 }
1334}
1335
1340static bool
1341pm_regexp_parse_expression(pm_regexp_parser_t *parser, uint16_t depth) {
1342 if (depth >= PM_REGEXP_PARSE_DEPTH_MAX) {
1343 pm_regexp_parse_error(parser, parser->start, parser->end, "parse depth limit over");
1344 return false;
1345 }
1346
1347 if (!pm_regexp_parse_item(parser, depth)) {
1348 return false;
1349 }
1350
1351 while (!pm_regexp_char_is_eof(parser) && *parser->cursor != ')' && *parser->cursor != '|') {
1352 if (!pm_regexp_parse_item(parser, depth)) {
1353 return false;
1354 }
1355 }
1356
1357 return true;
1358}
1359
1366static bool
1367pm_regexp_parse_pattern(pm_regexp_parser_t *parser) {
1368 do {
1369 if (pm_regexp_char_is_eof(parser)) return true;
1370 if (!pm_regexp_parse_expression(parser, 0)) return false;
1371 } while (pm_regexp_char_accept(parser, '|'));
1372
1373 return pm_regexp_char_is_eof(parser);
1374}
1375
1376// ---------------------------------------------------------------------------
1377// Encoding validation
1378// ---------------------------------------------------------------------------
1379
1384static bool
1385pm_regexp_validate_hex_escapes(const pm_encoding_t *encoding, const pm_buffer_t *buffer) {
1386 const uint8_t *data = (const uint8_t *) pm_buffer_value(buffer);
1387 size_t len = pm_buffer_length(buffer);
1388 size_t i = 0;
1389
1390 while (i < len) {
1391 size_t group_start = i;
1392 while (i < len && data[i] != 0x00) i++;
1393
1394 for (size_t j = group_start; j < i; ) {
1395 size_t width = pm_encoding_char_width(encoding, data + j, (ptrdiff_t) (i - j));
1396 if (width == 0) return false;
1397 j += width;
1398 }
1399
1400 if (i < len) i++; // skip sentinel
1401 }
1402
1403 return true;
1404}
1405
1410static void
1411pm_regexp_format_for_error(pm_buffer_t *buffer, const pm_encoding_t *encoding, const uint8_t *source, size_t length) {
1412 size_t index = 0;
1413
1414 if (encoding == PM_ENCODING_UTF_8_ENTRY) {
1415 pm_buffer_append_string(buffer, (const char *) source, length);
1416 return;
1417 }
1418
1419 while (index < length) {
1420 if (source[index] < 0x80) {
1421 pm_buffer_append_byte(buffer, source[index]);
1422 index++;
1423 } else if (pm_encoding_multibyte_p(encoding)) {
1424 size_t width = pm_encoding_char_width(encoding, source + index, (ptrdiff_t) (length - index));
1425
1426 if (width > 1) {
1427 pm_buffer_append_string(buffer, "\\x{", 3);
1428 for (size_t i = 0; i < width; i++) {
1429 pm_buffer_append_format(buffer, "%02X", source[index + i]);
1430 }
1431 pm_buffer_append_byte(buffer, '}');
1432 index += width;
1433 } else {
1434 pm_buffer_append_format(buffer, "\\x%02X", source[index]);
1435 index++;
1436 }
1437 } else {
1438 pm_buffer_append_format(buffer, "\\x%02X", source[index]);
1439 index++;
1440 }
1441 }
1442}
1443
1447#define PM_REGEXP_ENCODING_ERROR(parser, diag_id, ...) \
1448 pm_diagnostic_list_append_format( \
1449 &(parser)->parser->metadata_arena, \
1450 &(parser)->parser->error_list, \
1451 (uint32_t) ((parser)->node_start - (parser)->parser->start), \
1452 (uint32_t) ((parser)->node_end - (parser)->node_start), \
1453 diag_id, __VA_ARGS__)
1454
1474static pm_node_flags_t
1475pm_regexp_validate_encoding_modifier(pm_regexp_parser_t *parser, bool ascii_only, pm_node_flags_t flags, char modifier, const pm_encoding_t *modifier_encoding, const char *source_start, int source_length) {
1476
1477 if (parser->explicit_encoding == NULL) {
1478 if (ascii_only) {
1479 // Check property escapes against the modifier's encoding tier.
1480 // /n (ASCII-8BIT): only POSIX properties are valid.
1481 // /e, /s: POSIX and script properties are valid.
1482 // /u: all properties are valid.
1483 if (modifier == 'n' && parser->property_name != NULL) {
1484 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
1485 (int) parser->property_name_length, (const char *) parser->property_name,
1486 source_length, source_start);
1487 } else if (modifier != 'u' && parser->has_unicode_property_escape) {
1488 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
1489 (int) parser->unicode_property_name_length, (const char *) parser->unicode_property_name,
1490 source_length, source_start);
1491 }
1492 return modifier == 'n' ? PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING : flags;
1493 }
1494
1495 if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
1496 for (uint32_t i = 0; i < parser->non_ascii_literal_count; i++) {
1497 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1498 }
1499 } else if (parser->encoding != modifier_encoding) {
1500 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->encoding->name);
1501
1502 if (modifier == 'n' && !ascii_only) {
1503 pm_buffer_t formatted = { 0 };
1504 pm_regexp_format_for_error(&formatted, parser->encoding, (const uint8_t *) source_start, (size_t) source_length);
1505 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_NON_ESCAPED_MBC, (int) formatted.length, (const char *) formatted.value);
1506 pm_buffer_cleanup(&formatted);
1507 }
1508 }
1509
1510 return flags;
1511 }
1512
1513 // Mixed unicode + hex escapes.
1514 if (parser->has_unicode_escape && parser->has_hex_escape) {
1515 if (modifier == 'n') {
1516 if (parser->last_escape_was_unicode) {
1517 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
1518 } else {
1519 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
1520 }
1521 } else {
1522 if (!pm_regexp_validate_hex_escapes(modifier_encoding, &parser->hex_escape_buffer)) {
1523 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1524 }
1525 }
1526
1527 return flags;
1528 }
1529
1530 if (modifier != 'u' && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
1531 if (parser->last_escape_was_unicode) {
1532 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
1533 } else if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
1534 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
1535 }
1536 }
1537
1538 if (modifier != 'n' && !pm_regexp_validate_hex_escapes(modifier_encoding, &parser->hex_escape_buffer)) {
1539 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1540 }
1541
1542 return flags;
1543}
1544
1561static pm_node_flags_t
1562pm_regexp_validate_encoding(pm_regexp_parser_t *parser, bool ascii_only, pm_node_flags_t flags, const char *source_start, int source_length) {
1563
1564 // Invalid multibyte characters suppress further validation.
1565 // Errors were already emitted at the byte locations during parsing.
1566 if (parser->has_invalid_multibyte) {
1567 return flags;
1568 }
1569
1570 if (parser->invalid_unicode_range) {
1571 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, source_length, source_start);
1572 return flags;
1573 }
1574
1575 // Check modifier flags first.
1576 if (flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) {
1577 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'n', PM_ENCODING_ASCII_8BIT_ENTRY, source_start, source_length);
1578 }
1579 if (flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) {
1580 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'u', PM_ENCODING_UTF_8_ENTRY, source_start, source_length);
1581 }
1582 if (flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) {
1583 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'e', PM_ENCODING_EUC_JP_ENTRY, source_start, source_length);
1584 }
1585 if (flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) {
1586 return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 's', PM_ENCODING_WINDOWS_31J_ENTRY, source_start, source_length);
1587 }
1588
1589 // No modifier — check for non-ASCII literals in US-ASCII encoding.
1590 if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY && parser->explicit_encoding == NULL && !ascii_only) {
1591 for (uint32_t i = 0; i < parser->non_ascii_literal_count; i++) {
1592 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1593 }
1594 }
1595
1596 // ASCII-only regexps get downgraded to US-ASCII, unless property escapes
1597 // force UTF-8.
1598 if (ascii_only) {
1599 if (parser->has_property_escape) {
1600 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
1601 }
1602 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING;
1603 }
1604
1605 // Check explicit encoding from escape sequences.
1606 if (parser->explicit_encoding != NULL) {
1607 // Mixed unicode + hex escapes without modifier.
1608 if (parser->has_unicode_escape && parser->has_hex_escape && parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
1609 if (parser->encoding != PM_ENCODING_US_ASCII_ENTRY &&
1610 parser->encoding != PM_ENCODING_ASCII_8BIT_ENTRY &&
1611 !pm_regexp_validate_hex_escapes(parser->encoding, &parser->hex_escape_buffer)) {
1612 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1613 } else if (parser->last_escape_was_unicode) {
1614 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
1615 } else {
1616 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
1617 }
1618
1619 return 0;
1620 }
1621
1622 if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
1623 if (!pm_regexp_validate_hex_escapes(parser->explicit_encoding, &parser->hex_escape_buffer)) {
1624 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1625 }
1626
1627 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
1628 } else if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
1629 return PM_REGULAR_EXPRESSION_FLAGS_FORCED_BINARY_ENCODING;
1630 } else {
1631 if (!pm_regexp_validate_hex_escapes(parser->explicit_encoding, &parser->hex_escape_buffer)) {
1632 PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1633 }
1634 }
1635 }
1636
1637 return 0;
1638}
1639
1647pm_node_flags_t
1648pm_regexp_parse(pm_parser_t *parser, pm_regular_expression_node_t *node, pm_regexp_name_callback_t name_callback, void *name_data) {
1649 const uint8_t *source = parser->start + node->content_loc.start;
1650 size_t size = node->content_loc.length;
1651 bool extended_mode = PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EXTENDED);
1652 pm_node_flags_t flags = PM_NODE_FLAGS(node);
1653
1654 const uint8_t *node_start = parser->start + node->base.location.start;
1655 const uint8_t *node_end = parser->start + node->base.location.start + node->base.location.length;
1656
1657 // First pass: walk raw source for encoding validation (no name extraction).
1658 pm_regexp_parser_t regexp_parser = {
1659 .parser = parser,
1660 .start = source,
1661 .cursor = source,
1662 .end = source + size,
1663 .extended_mode = extended_mode,
1664 .encoding_changed = parser->encoding_changed,
1665 .encoding = parser->encoding,
1666 .name_callback = NULL,
1667 .name_data = NULL,
1668 .report_errors = true,
1669 .node_start = node_start,
1670 .node_end = node_end,
1671 .has_unicode_escape = false,
1672 .has_hex_escape = false,
1673 .last_escape_was_unicode = false,
1674 .explicit_encoding = NULL,
1675 .has_property_escape = false,
1676 .has_unicode_property_escape = false,
1677 .property_name = NULL,
1678 .property_name_length = 0,
1679 .unicode_property_name = NULL,
1680 .unicode_property_name_length = 0,
1681 .non_ascii_literal_count = 0,
1682 .invalid_unicode_range = false,
1683 .hex_escape_buffer = { 0 },
1684 .hex_group_active = false,
1685 .has_invalid_multibyte = false,
1686 };
1687
1688 pm_regexp_parse_pattern(&regexp_parser);
1689
1690 // Compute ascii_only from the regexp parser's tracked state. We cannot
1691 // use node->unescaped for this because regexp unescaped content preserves
1692 // escape text (e.g., \x80 is 4 ASCII chars), not the binary values.
1693 bool ascii_only = !regexp_parser.has_hex_escape && !regexp_parser.has_unicode_escape && regexp_parser.non_ascii_literal_count == 0;
1694 // Use the unescaped content for error messages to match CRuby's format,
1695 // where Ruby escapes like \M-\C-? are resolved to bytes but regexp escapes
1696 // like \u{80} are preserved as text.
1697 const char *error_source = (const char *) pm_string_source(&node->unescaped);
1698 int error_source_length = (int) pm_string_length(&node->unescaped);
1699 pm_node_flags_t encoding_flags = pm_regexp_validate_encoding(&regexp_parser, ascii_only, flags, error_source, error_source_length);
1700 pm_buffer_cleanup(&regexp_parser.hex_escape_buffer);
1701
1702 // Second pass: walk unescaped content for named capture extraction.
1703 if (name_callback != NULL) {
1704 pm_regexp_parse_named_captures(parser, pm_string_source(&node->unescaped), pm_string_length(&node->unescaped), extended_mode, name_callback, name_data);
1705 }
1706
1707 return encoding_flags;
1708}
1709
1720void
1721pm_regexp_parse_named_captures(pm_parser_t *parser, const uint8_t *source, size_t size, bool extended_mode, pm_regexp_name_callback_t name_callback, void *name_data) {
1722 pm_regexp_parser_t regexp_parser = {
1723 .parser = parser,
1724 .start = source,
1725 .cursor = source,
1726 .end = source + size,
1727 .extended_mode = extended_mode,
1728 .encoding_changed = parser->encoding_changed,
1729 .encoding = parser->encoding,
1730 .name_callback = name_callback,
1731 .name_data = name_data,
1732 .report_errors = false,
1733 .has_unicode_escape = false,
1734 .has_hex_escape = false,
1735 .last_escape_was_unicode = false,
1736 .explicit_encoding = NULL,
1737 .has_property_escape = false,
1738 .has_unicode_property_escape = false,
1739 .property_name = NULL,
1740 .property_name_length = 0,
1741 .unicode_property_name = NULL,
1742 .unicode_property_name_length = 0,
1743 .non_ascii_literal_count = 0,
1744 .invalid_unicode_range = false,
1745 .hex_escape_buffer = { 0 },
1746 .hex_group_active = false,
1747 .has_invalid_multibyte = false,
1748 };
1749
1750 pm_regexp_parse_pattern(&regexp_parser);
1751 pm_buffer_cleanup(&regexp_parser.hex_escape_buffer);
1752}
#define PRISM_FALLTHROUGH
We use -Wimplicit-fallthrough to guard potentially unintended fall-through between cases of a switch.
Definition fallthrough.h:15
int len
Length of the buffer.
Definition io.h:8
#define PRISM_INLINE
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
Definition inline.h:12
VALUE type(ANYARGS)
ANYARGS-ed function type.
uint32_t start
The offset of the location from the start of the source.
Definition ast.h:581
uint32_t length
The length of the location.
Definition ast.h:584
pm_location_t location
This is the location of the node in the source.
Definition ast.h:1116
This is the set of options that are configurable on the regular expression.
Definition regexp.c:933
uint8_t values[PRISM_REGEXP_OPTION_STATE_SLOTS]
The current state of each option.
Definition regexp.c:935
This is the parser that is going to handle parsing regular expressions.
Definition regexp.c:23
const uint8_t * cursor
A pointer to the current position in the source.
Definition regexp.c:31
bool has_unicode_escape
Whether a \u{...} escape with value >= 0x80 was seen.
Definition regexp.c:102
size_t unicode_property_name_length
Length of the first Unicode-only property name found.
Definition regexp.c:76
bool has_hex_escape
Whether a \xNN escape (or \M-x, etc.) with value >= 0x80 was seen.
Definition regexp.c:105
const uint8_t * start
A pointer to the start of the source that we are parsing.
Definition regexp.c:28
const uint8_t * end
A pointer to the end of the source that we are parsing.
Definition regexp.c:34
void * name_data
The data to pass to the name callback.
Definition regexp.c:43
bool hex_group_active
Whether we are accumulating consecutive hex escape bytes.
Definition regexp.c:123
pm_buffer_t hex_escape_buffer
Buffer of hex escape byte values >= 0x80, separated by 0x00 sentinels.
Definition regexp.c:79
bool extended_mode
Whether or not the regular expression currently being parsed is in extended mode, wherein whitespace ...
Definition regexp.c:88
pm_parser_t * parser
The parser that is currently being used.
Definition regexp.c:25
size_t property_name_length
Length of the first non-POSIX property name found.
Definition regexp.c:67
bool invalid_unicode_range
Whether a \u escape with invalid range (surrogate or > 0x10FFFF) was seen.
Definition regexp.c:120
const uint8_t * property_name
Pointer to the first non-POSIX property name (for /n error messages).
Definition regexp.c:64
const pm_encoding_t * encoding
The encoding of the source.
Definition regexp.c:37
const pm_encoding_t * explicit_encoding
The explicit encoding determined by escape sequences.
Definition regexp.c:56
const uint8_t * node_end
The end of the regexp node (for error locations).
Definition regexp.c:49
uint32_t non_ascii_literal_count
Count of non-ASCII literal bytes (not from escapes).
Definition regexp.c:82
const uint8_t * node_start
The start of the regexp node (for error locations).
Definition regexp.c:46
bool report_errors
Whether syntax errors are reported.
Definition regexp.c:99
const uint8_t * unicode_property_name
Pointer to the first Unicode-only property name (for /e, /s error messages).
Definition regexp.c:73
pm_regexp_name_callback_t name_callback
The callback to call when a named capture group is found.
Definition regexp.c:40
bool encoding_changed
Whether the encoding has changed from the default.
Definition regexp.c:91
bool has_unicode_property_escape
Whether a Unicode-only property escape was found (not POSIX or script).
Definition regexp.c:117
bool has_invalid_multibyte
Whether an invalid multibyte character was found during parsing.
Definition regexp.c:126
bool last_escape_was_unicode
Tracks whether the last encoding-setting escape was \u (true) or \x (false).
Definition regexp.c:111
bool has_property_escape
Whether any \p{...} or \P{...} property escape was found.
Definition regexp.c:114
pm_node_t base
The embedded base node.
Definition ast.h:7006
pm_string_t unescaped
RegularExpressionNode::unescaped.
Definition ast.h:7026
pm_location_t content_loc
RegularExpressionNode::content_loc.
Definition ast.h:7016
A view of a sequence of bytes.
Definition stringy.h:23