Ruby 4.1.0dev (2026-11-19 revision b409062b86ed964c0779df64675ef171a6c93a76)
prism.c
5
6#include "prism/internal/allocator.h"
7#include "prism/internal/arena.h"
8#include "prism/internal/assert.h"
9#include "prism/internal/bit.h"
10#include "prism/internal/buffer.h"
11#include "prism/internal/char.h"
12#include "prism/internal/comments.h"
13#include "prism/internal/constant_pool.h"
14#include "prism/internal/diagnostic.h"
15#include "prism/internal/double.h"
16#include "prism/internal/encoding.h"
17#include "prism/internal/integer.h"
18#include "prism/internal/line_offset_list.h"
19#include "prism/internal/list.h"
20#include "prism/internal/magic_comments.h"
21#include "prism/internal/memchr.h"
22#include "prism/internal/node.h"
23#include "prism/internal/oom.h"
24#include "prism/internal/options.h"
25#include "prism/internal/parser.h"
26#include "prism/internal/regexp.h"
27#include "prism/internal/serialize.h"
28#include "prism/internal/source.h"
29#include "prism/internal/static_literals.h"
30#include "prism/internal/stringy.h"
31#include "prism/internal/strncasecmp.h"
32#include "prism/internal/strpbrk.h"
33#include "prism/internal/tokens.h"
34
35#include "prism/excludes.h"
36#include "prism/serialize.h"
37#include "prism/stream.h"
38#include "prism/version.h"
39
40#include <limits.h>
41#include <stdio.h>
42#include <stdlib.h>
43
49#ifndef PRISM_DEPTH_MAXIMUM
50 #define PRISM_DEPTH_MAXIMUM 10000
51#endif
52
57#define PM_CONCATENATE(left, right) left ## right
58
64#if defined(_Static_assert)
65# define PM_STATIC_ASSERT(line, condition, message) _Static_assert(condition, message)
66#else
67# define PM_STATIC_ASSERT(line, condition, message) typedef char PM_CONCATENATE(static_assert_, line)[(condition) ? 1 : -1]
68#endif
69
73const char *
75 return PRISM_VERSION;
76}
77
82#define PM_TAB_WHITESPACE_SIZE 8
83
84// Macros for min/max.
85#define MIN(a,b) (((a)<(b))?(a):(b))
86#define MAX(a,b) (((a)>(b))?(a):(b))
87
88/******************************************************************************/
89/* Helpful AST-related macros */
90/******************************************************************************/
91
92#define U32(value_) ((uint32_t) (value_))
93
94#define FL PM_NODE_FLAGS
95#define UP PM_NODE_UPCAST
96
97#define PM_TOKEN_START(parser_, token_) U32((token_)->start - (parser_)->start)
98#define PM_TOKEN_END(parser_, token_) U32((token_)->end - (parser_)->start)
99#define PM_TOKEN_LENGTH(token_) U32((token_)->end - (token_)->start)
100#define PM_TOKENS_LENGTH(left_, right_) U32((right_)->end - (left_)->start)
101
102#define PM_NODE_START(node_) (UP(node_)->location.start)
103#define PM_NODE_LENGTH(node_) (UP(node_)->location.length)
104#define PM_NODE_END(node_) (UP(node_)->location.start + UP(node_)->location.length)
105#define PM_NODES_LENGTH(left_, right_) (PM_NODE_END(right_) - PM_NODE_START(left_))
106
107#define PM_TOKEN_NODE_LENGTH(parser_, token_, node_) (PM_NODE_END(node_) - PM_TOKEN_START(parser_, token_))
108#define PM_NODE_TOKEN_LENGTH(parser_, node_, token_) (PM_TOKEN_END(parser_, token_) - PM_NODE_START(node_))
109
110#define PM_NODE_START_SET_NODE(left_, right_) (PM_NODE_START(left_) = PM_NODE_START(right_))
111#define PM_NODE_START_SET_TOKEN(parser_, node_, token_) (PM_NODE_START(node_) = PM_TOKEN_START(parser_, token_))
112#define PM_NODE_LENGTH_SET_NODE(left_, right_) (PM_NODE_LENGTH(left_) = PM_NODE_END(right_) - PM_NODE_START(left_))
113#define PM_NODE_LENGTH_SET_TOKEN(parser_, node_, token_) (PM_NODE_LENGTH(node_) = PM_TOKEN_END(parser_, token_) - PM_NODE_START(node_))
114#define PM_NODE_LENGTH_SET_LOCATION(node_, location_) (PM_NODE_LENGTH(node_) = PM_LOCATION_END(location_) - PM_NODE_START(node_))
115
124pm_location_init(uint32_t start, uint32_t length) {
125 pm_location_t location = { .start = start, .length = length };
126 return location;
127}
128
129#define PM_LOCATION_INIT(start_, length_) pm_location_init((start_), (length_))
130#define PM_LOCATION_INIT_BOUNDS(start_, end_) PM_LOCATION_INIT((start_), U32((end_) - (start_)))
131#define PM_LOCATION_INIT_UNSET PM_LOCATION_INIT(0, 0)
132#define PM_LOCATION_INIT_TOKEN(parser_, token_) PM_LOCATION_INIT(PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_))
133#define PM_LOCATION_INIT_NODE(node_) UP(node_)->location
134
135#define PM_LOCATION_INIT_TOKENS(parser_, left_, right_) PM_LOCATION_INIT(PM_TOKEN_START(parser_, left_), PM_TOKENS_LENGTH(left_, right_))
136#define PM_LOCATION_INIT_NODES(left_, right_) PM_LOCATION_INIT(PM_NODE_START(left_), PM_NODES_LENGTH(left_, right_))
137#define PM_LOCATION_INIT_TOKEN_NODE(parser_, token_, node_) PM_LOCATION_INIT(PM_TOKEN_START(parser_, token_), PM_TOKEN_NODE_LENGTH(parser_, token_, node_))
138#define PM_LOCATION_INIT_NODE_TOKEN(parser_, node_, token_) PM_LOCATION_INIT(PM_NODE_START(node_), PM_NODE_TOKEN_LENGTH(parser_, node_, token_))
139
140#define TOK2LOC(parser_, token_) PM_LOCATION_INIT_TOKEN(parser_, token_)
141#define NTOK2LOC(parser_, token_) ((token_) == NULL ? PM_LOCATION_INIT_UNSET : TOK2LOC(parser_, token_))
142#define NTOK2PTR(token_) ((token_).start == NULL ? NULL : &(token_))
143
144/******************************************************************************/
145/* Lex mode manipulations */
146/******************************************************************************/
147
152static PRISM_INLINE uint8_t
153lex_mode_incrementor(const uint8_t start) {
154 switch (start) {
155 case '(':
156 case '[':
157 case '{':
158 case '<':
159 return start;
160 default:
161 return '\0';
162 }
163}
164
169static PRISM_INLINE uint8_t
170lex_mode_terminator(const uint8_t start) {
171 switch (start) {
172 case '(':
173 return ')';
174 case '[':
175 return ']';
176 case '{':
177 return '}';
178 case '<':
179 return '>';
180 default:
181 return start;
182 }
183}
184
189static void
190lex_mode_push(pm_parser_t *parser, pm_lex_mode_t lex_mode) {
191 pm_lex_mode_t *next = parser->lex_modes.free;
192
193 if (next != NULL) {
194 parser->lex_modes.free = next->prev;
195 } else {
196 next = PM_ARENA_ALLOC(&parser->scratch_arena, pm_lex_mode_t);
197 }
198
199 lex_mode.prev = parser->lex_modes.current;
200 *next = lex_mode;
201 parser->lex_modes.current = next;
202}
203
207static PRISM_INLINE void
208lex_mode_push_list(pm_parser_t *parser, bool interpolation, uint8_t delimiter) {
209 uint8_t incrementor = lex_mode_incrementor(delimiter);
210 uint8_t terminator = lex_mode_terminator(delimiter);
211
212 pm_lex_mode_t lex_mode = {
213 .mode = PM_LEX_LIST,
214 .as.list = {
215 .nesting = 0,
216 .interpolation = interpolation,
217 .started = false,
218 .separated = false,
219 .incrementor = incrementor,
220 .terminator = terminator
221 }
222 };
223
224 // These are the places where we need to split up the content of the list.
225 // We'll use strpbrk to find the first of these characters.
226 uint8_t *breakpoints = lex_mode.as.list.breakpoints;
227 memset(breakpoints, 0, PM_STRPBRK_CACHE_SIZE);
228 memcpy(breakpoints, "\\ \t\f\r\v\n", sizeof("\\ \t\f\r\v\n") - 1);
229 size_t index = 7;
230
231 // Now we'll add the terminator to the list of breakpoints. If the
232 // terminator is not already a NULL byte, add it to the list.
233 if (terminator != '\0') {
234 breakpoints[index++] = terminator;
235 }
236
237 // If interpolation is allowed, then we're going to check for the #
238 // character. Otherwise we'll only look for escapes and the terminator.
239 if (interpolation) {
240 breakpoints[index++] = '#';
241 }
242
243 // If there is an incrementor, then we'll check for that as well.
244 if (incrementor != '\0') {
245 breakpoints[index++] = incrementor;
246 }
247
248 parser->explicit_encoding = NULL;
249 lex_mode_push(parser, lex_mode);
250}
251
257static PRISM_INLINE void
258lex_mode_push_list_eof(pm_parser_t *parser) {
259 lex_mode_push_list(parser, false, '\0');
260}
261
265static PRISM_INLINE void
266lex_mode_push_regexp(pm_parser_t *parser, uint8_t incrementor, uint8_t terminator) {
267 pm_lex_mode_t lex_mode = {
268 .mode = PM_LEX_REGEXP,
269 .as.regexp = {
270 .nesting = 0,
271 .incrementor = incrementor,
272 .terminator = terminator
273 }
274 };
275
276 // These are the places where we need to split up the content of the
277 // regular expression. We'll use strpbrk to find the first of these
278 // characters.
279 uint8_t *breakpoints = lex_mode.as.regexp.breakpoints;
280 memset(breakpoints, 0, PM_STRPBRK_CACHE_SIZE);
281 memcpy(breakpoints, "\r\n\\#", sizeof("\r\n\\#") - 1);
282 size_t index = 4;
283
284 // First we'll add the terminator.
285 if (terminator != '\0') {
286 breakpoints[index++] = terminator;
287 }
288
289 // Next, if there is an incrementor, then we'll check for that as well.
290 if (incrementor != '\0') {
291 breakpoints[index++] = incrementor;
292 }
293
294 parser->explicit_encoding = NULL;
295 lex_mode_push(parser, lex_mode);
296}
297
301static PRISM_INLINE void
302lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed, uint8_t incrementor, uint8_t terminator) {
303 pm_lex_mode_t lex_mode = {
304 .mode = PM_LEX_STRING,
305 .as.string = {
306 .nesting = 0,
307 .interpolation = interpolation,
308 .label_allowed = label_allowed,
309 .incrementor = incrementor,
310 .terminator = terminator
311 }
312 };
313
314 // These are the places where we need to split up the content of the
315 // string. We'll use strpbrk to find the first of these characters.
316 uint8_t *breakpoints = lex_mode.as.string.breakpoints;
317 memset(breakpoints, 0, PM_STRPBRK_CACHE_SIZE);
318 memcpy(breakpoints, "\r\n\\", sizeof("\r\n\\") - 1);
319 size_t index = 3;
320
321 // Now add in the terminator. If the terminator is not already a NULL byte,
322 // then we'll add it.
323 if (terminator != '\0') {
324 breakpoints[index++] = terminator;
325 }
326
327 // If interpolation is allowed, then we're going to check for the #
328 // character. Otherwise we'll only look for escapes and the terminator.
329 if (interpolation) {
330 breakpoints[index++] = '#';
331 }
332
333 // If we have an incrementor, then we'll add that in as a breakpoint as
334 // well.
335 if (incrementor != '\0') {
336 breakpoints[index++] = incrementor;
337 }
338
339 parser->explicit_encoding = NULL;
340 lex_mode_push(parser, lex_mode);
341}
342
348static PRISM_INLINE void
349lex_mode_push_string_eof(pm_parser_t *parser) {
350 lex_mode_push_string(parser, false, false, '\0', '\0');
351}
352
358static void
359lex_mode_pop(pm_parser_t *parser) {
360 pm_lex_mode_t *popped = parser->lex_modes.current;
361
362 if (popped == &parser->lex_modes.base) {
363 popped->mode = PM_LEX_DEFAULT;
364 } else {
365 parser->lex_modes.current = popped->prev;
366 popped->prev = parser->lex_modes.free;
367 parser->lex_modes.free = popped;
368 }
369}
370
374static PRISM_INLINE bool
375lex_state_p(const pm_parser_t *parser, pm_lex_state_t state) {
376 return parser->lex_state & state;
377}
378
379typedef enum {
380 PM_IGNORED_NEWLINE_NONE = 0,
381 PM_IGNORED_NEWLINE_ALL,
382 PM_IGNORED_NEWLINE_PATTERN
383} pm_ignored_newline_type_t;
384
385static PRISM_INLINE pm_ignored_newline_type_t
386lex_state_ignored_p(pm_parser_t *parser) {
387 bool ignored = lex_state_p(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_CLASS | PM_LEX_STATE_FNAME | PM_LEX_STATE_DOT) && !lex_state_p(parser, PM_LEX_STATE_LABELED);
388
389 if (ignored) {
390 return PM_IGNORED_NEWLINE_ALL;
391 } else if ((parser->lex_state & ~((unsigned int) PM_LEX_STATE_LABEL)) == (PM_LEX_STATE_ARG | PM_LEX_STATE_LABELED)) {
392 return PM_IGNORED_NEWLINE_PATTERN;
393 } else {
394 return PM_IGNORED_NEWLINE_NONE;
395 }
396}
397
398static PRISM_INLINE bool
399lex_state_beg_p(pm_parser_t *parser) {
400 return lex_state_p(parser, PM_LEX_STATE_BEG_ANY) || ((parser->lex_state & (PM_LEX_STATE_ARG | PM_LEX_STATE_LABELED)) == (PM_LEX_STATE_ARG | PM_LEX_STATE_LABELED));
401}
402
403static PRISM_INLINE bool
404lex_state_arg_p(pm_parser_t *parser) {
405 return lex_state_p(parser, PM_LEX_STATE_ARG_ANY);
406}
407
408static PRISM_INLINE bool
409lex_state_spcarg_p(pm_parser_t *parser, bool space_seen) {
410 if (parser->current.end >= parser->end) {
411 return false;
412 }
413 return lex_state_arg_p(parser) && space_seen && !pm_char_is_whitespace(*parser->current.end);
414}
415
416static PRISM_INLINE bool
417lex_state_end_p(pm_parser_t *parser) {
418 return lex_state_p(parser, PM_LEX_STATE_END_ANY);
419}
420
424static PRISM_INLINE bool
425lex_state_operator_p(pm_parser_t *parser) {
426 return lex_state_p(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_DOT);
427}
428
433static PRISM_INLINE void
434lex_state_set(pm_parser_t *parser, pm_lex_state_t state) {
435 parser->lex_state = state;
436}
437
438#ifndef PM_DEBUG_LOGGING
443#define PM_DEBUG_LOGGING 0
444#endif
445
446#if PM_DEBUG_LOGGING
447PRISM_UNUSED static void
448debug_state(pm_parser_t *parser) {
449 fprintf(stderr, "STATE: ");
450 bool first = true;
451
452 if (parser->lex_state == PM_LEX_STATE_NONE) {
453 fprintf(stderr, "NONE\n");
454 return;
455 }
456
457#define CHECK_STATE(state) \
458 if (parser->lex_state & state) { \
459 if (!first) fprintf(stderr, "|"); \
460 fprintf(stderr, "%s", #state); \
461 first = false; \
462 }
463
464 CHECK_STATE(PM_LEX_STATE_BEG)
465 CHECK_STATE(PM_LEX_STATE_END)
466 CHECK_STATE(PM_LEX_STATE_ENDARG)
467 CHECK_STATE(PM_LEX_STATE_ENDFN)
468 CHECK_STATE(PM_LEX_STATE_ARG)
469 CHECK_STATE(PM_LEX_STATE_CMDARG)
470 CHECK_STATE(PM_LEX_STATE_MID)
471 CHECK_STATE(PM_LEX_STATE_FNAME)
472 CHECK_STATE(PM_LEX_STATE_DOT)
473 CHECK_STATE(PM_LEX_STATE_CLASS)
474 CHECK_STATE(PM_LEX_STATE_LABEL)
475 CHECK_STATE(PM_LEX_STATE_LABELED)
476 CHECK_STATE(PM_LEX_STATE_FITEM)
477
478#undef CHECK_STATE
479
480 fprintf(stderr, "\n");
481}
482
483static void
484debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * caller_name, int line_number) {
485 fprintf(stderr, "Caller: %s:%d\nPrevious: ", caller_name, line_number);
486 debug_state(parser);
487 lex_state_set(parser, state);
488 fprintf(stderr, "Now: ");
489 debug_state(parser);
490 fprintf(stderr, "\n");
491}
492
493#define lex_state_set(parser, state) debug_lex_state_set(parser, state, __func__, __LINE__)
494#endif
495
496/******************************************************************************/
497/* Command-line macro helpers */
498/******************************************************************************/
499
501#define PM_PARSER_COMMAND_LINE_OPTION(parser, option) ((parser)->command_line & (option))
502
504#define PM_PARSER_COMMAND_LINE_OPTION_A(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_A)
505
507#define PM_PARSER_COMMAND_LINE_OPTION_E(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_E)
508
510#define PM_PARSER_COMMAND_LINE_OPTION_L(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_L)
511
513#define PM_PARSER_COMMAND_LINE_OPTION_N(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_N)
514
516#define PM_PARSER_COMMAND_LINE_OPTION_P(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_P)
517
519#define PM_PARSER_COMMAND_LINE_OPTION_X(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_X)
520
521/******************************************************************************/
522/* Diagnostic-related functions */
523/******************************************************************************/
524
528static PRISM_INLINE void
529pm_parser_err(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) {
530 pm_diagnostic_list_append(&parser->metadata_arena, &parser->error_list, start, length, diag_id);
531}
532
537static PRISM_INLINE void
538pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) {
539 pm_parser_err(parser, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), diag_id);
540}
541
546static PRISM_INLINE void
547pm_parser_err_current(pm_parser_t *parser, pm_diagnostic_id_t diag_id) {
548 pm_parser_err_token(parser, &parser->current, diag_id);
549}
550
554void
555pm_parser_err_stack_exhausted(pm_parser_t *parser, uint32_t start, uint32_t length) {
556 if (parser->stack_exhausted_reported) return;
557 parser->stack_exhausted_reported = true;
558 pm_parser_err(parser, start, length, PM_ERR_STACK_EXHAUSTED);
559}
560
565static PRISM_INLINE void
566pm_parser_err_previous(pm_parser_t *parser, pm_diagnostic_id_t diag_id) {
567 pm_parser_err_token(parser, &parser->previous, diag_id);
568}
569
574static PRISM_INLINE void
575pm_parser_err_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) {
576 pm_parser_err(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), diag_id);
577}
578
582#define PM_PARSER_ERR_FORMAT(parser_, start_, length_, diag_id_, ...) \
583 pm_diagnostic_list_append_format(&(parser_)->metadata_arena, &(parser_)->error_list, start_, length_, diag_id_, __VA_ARGS__)
584
589#define PM_PARSER_ERR_NODE_FORMAT(parser_, node_, diag_id_, ...) \
590 PM_PARSER_ERR_FORMAT(parser_, PM_NODE_START(node_), PM_NODE_LENGTH(node_), diag_id_, __VA_ARGS__)
591
596#define PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser_, node_, diag_id_) \
597 PM_PARSER_ERR_NODE_FORMAT(parser_, node_, diag_id_, (int) PM_NODE_LENGTH(node_), (const char *) (parser_->start + PM_NODE_START(node_)))
598
603#define PM_PARSER_ERR_TOKEN_FORMAT(parser_, token_, diag_id, ...) \
604 PM_PARSER_ERR_FORMAT(parser_, PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_), diag_id, __VA_ARGS__)
605
610#define PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser_, token_, diag_id_) \
611 PM_PARSER_ERR_TOKEN_FORMAT(parser_, token_, diag_id_, (int) PM_TOKEN_LENGTH(token_), (const char *) (token_)->start)
612
616static PRISM_INLINE void
617pm_parser_warn(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) {
618 pm_diagnostic_list_append(&parser->metadata_arena, &parser->warning_list, start, length, diag_id);
619}
620
625static PRISM_INLINE void
626pm_parser_warn_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) {
627 pm_parser_warn(parser, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), diag_id);
628}
629
634static PRISM_INLINE void
635pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) {
636 pm_parser_warn(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), diag_id);
637}
638
643#define PM_PARSER_WARN_FORMAT(parser_, start_, length_, diag_id_, ...) \
644 pm_diagnostic_list_append_format(&(parser_)->metadata_arena, &(parser_)->warning_list, start_, length_, diag_id_, __VA_ARGS__)
645
650#define PM_PARSER_WARN_TOKEN_FORMAT(parser_, token_, diag_id_, ...) \
651 PM_PARSER_WARN_FORMAT(parser_, PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_), diag_id_, __VA_ARGS__)
652
657#define PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser_, token_, diag_id_) \
658 PM_PARSER_WARN_TOKEN_FORMAT(parser_, token_, diag_id_, (int) PM_TOKEN_LENGTH(token_), (const char *) (token_)->start)
659
664#define PM_PARSER_WARN_NODE_FORMAT(parser_, node_, diag_id_, ...) \
665 PM_PARSER_WARN_FORMAT(parser_, PM_NODE_START(node_), PM_NODE_LENGTH(node_), diag_id_, __VA_ARGS__)
666
672static void
673pm_parser_err_heredoc_term(pm_parser_t *parser, const uint8_t *ident_start, size_t ident_length) {
674 PM_PARSER_ERR_FORMAT(
675 parser,
676 U32(ident_start - parser->start),
677 U32(ident_length),
678 PM_ERR_HEREDOC_TERM,
679 (int) ident_length,
680 (const char *) ident_start
681 );
682}
683
684/******************************************************************************/
685/* Scope-related functions */
686/******************************************************************************/
687
691static void
692pm_parser_scope_push(pm_parser_t *parser, bool closed) {
693 pm_scope_t *previous = parser->current_scope;
694 pm_scope_t *scope = parser->free_scopes;
695
696 if (scope != NULL) {
697 parser->free_scopes = scope->previous;
698 } else {
699 scope = PM_ARENA_ALLOC(&parser->scratch_arena, pm_scope_t);
700 }
701
702 uint32_t locals_start = parser->locals_size;
703
704 *scope = (pm_scope_t) {
705 .previous = previous,
706 .locals_start = locals_start,
707 .level = previous == NULL ? 1 : previous->level + 1,
708 .floor = (closed || previous == NULL) ? locals_start : previous->floor,
709 .parameters = PM_SCOPE_PARAMETERS_NONE,
710 .implicit_parameters = { 0 },
711 .shareable_constant = previous == NULL ? PM_SCOPE_SHAREABLE_CONSTANT_NONE : previous->shareable_constant,
712 .closed = closed
713 };
714
715 parser->current_scope = scope;
716}
717
722static bool
723pm_parser_scope_toplevel_p(pm_parser_t *parser) {
724 pm_scope_t *scope = parser->current_scope;
725
726 do {
727 if (scope->previous == NULL) return true;
728 if (scope->closed) return false;
729 } while ((scope = scope->previous) != NULL);
730
731 PRISM_ASSERT(false && "unreachable");
732 return true;
733}
734
738static pm_scope_t *
739pm_parser_scope_find(pm_parser_t *parser, uint32_t depth) {
740 pm_scope_t *scope = parser->current_scope;
741
742 while (depth-- > 0) {
743 PRISM_ASSERT(scope != NULL);
744 scope = scope->previous;
745 }
746
747 return scope;
748}
749
750typedef enum {
751 PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS,
752 PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT,
753 PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL
754} pm_scope_forwarding_param_check_result_t;
755
756static pm_scope_forwarding_param_check_result_t
757pm_parser_scope_forwarding_param_check(pm_parser_t *parser, const uint8_t mask) {
758 pm_scope_t *scope = parser->current_scope;
759 bool conflict = false;
760
761 while (scope != NULL) {
762 if (scope->parameters & mask) {
763 if (scope->closed) {
764 if (conflict) {
765 return PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT;
766 } else {
767 return PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS;
768 }
769 }
770
771 conflict = true;
772 }
773
774 if (scope->closed) break;
775 scope = scope->previous;
776 }
777
778 return PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL;
779}
780
781static void
782pm_parser_scope_forwarding_block_check(pm_parser_t *parser, const pm_token_t * token) {
783 switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_BLOCK)) {
784 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS:
785 // Pass.
786 break;
787 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT:
788 pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_CONFLICT_AMPERSAND);
789 break;
790 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL:
791 pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND);
792 break;
793 }
794}
795
796static void
797pm_parser_scope_forwarding_positionals_check(pm_parser_t *parser, const pm_token_t * token) {
798 switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS)) {
799 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS:
800 // Pass.
801 break;
802 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT:
803 pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_CONFLICT_STAR);
804 break;
805 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL:
806 pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_STAR);
807 break;
808 }
809}
810
811static void
812pm_parser_scope_forwarding_all_check(pm_parser_t *parser, const pm_token_t *token) {
813 switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_ALL)) {
814 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS:
815 // Pass.
816 break;
817 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT:
818 // This shouldn't happen, because ... is not allowed in the
819 // declaration of blocks. If we get here, we assume we already have
820 // an error for this.
821 break;
822 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL:
823 pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES);
824 break;
825 }
826}
827
828static void
829pm_parser_scope_forwarding_keywords_check(pm_parser_t *parser, const pm_token_t * token) {
830 switch (pm_parser_scope_forwarding_param_check(parser, PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS)) {
831 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_PASS:
832 // Pass.
833 break;
834 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_CONFLICT:
835 pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_CONFLICT_STAR_STAR);
836 break;
837 case PM_SCOPE_FORWARDING_PARAM_CHECK_RESULT_FAIL:
838 pm_parser_err_token(parser, token, PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR);
839 break;
840 }
841}
842
846static PRISM_INLINE pm_shareable_constant_value_t
847pm_parser_scope_shareable_constant_get(pm_parser_t *parser) {
848 return parser->current_scope->shareable_constant;
849}
850
855static void
856pm_parser_scope_shareable_constant_set(pm_parser_t *parser, pm_shareable_constant_value_t shareable_constant) {
857 pm_scope_t *scope = parser->current_scope;
858
859 do {
860 scope->shareable_constant = shareable_constant;
861 } while (!scope->closed && (scope = scope->previous) != NULL);
862}
863
864/******************************************************************************/
865/* Local variable-related functions */
866/******************************************************************************/
867
872static PRISM_INLINE uint32_t
873pm_parser_binding(const pm_parser_t *parser, pm_constant_id_t name) {
874 if (name >= parser->bindings_capacity) return 0;
875 return parser->bindings[name];
876}
877
884static int
885pm_parser_local_depth_constant_id(pm_parser_t *parser, pm_constant_id_t constant_id) {
886 uint32_t index = pm_parser_binding(parser, constant_id);
887 const pm_scope_t *scope = parser->current_scope;
888
889 /* Every local at or above the floor belongs to a scope that the current
890 * scope can see into. The floor is never 0, so this also rejects names
891 * that are unbound. */
892 if (index < scope->floor) return -1;
893 return (int) (scope->level - parser->locals[index].level);
894}
895
905static void
906pm_parser_bindings_grow(pm_parser_t *parser) {
907 uint32_t capacity = parser->constant_pool.capacity;
908 uint32_t *bindings = PM_ARENA_ZALLOC_ARRAY(&parser->scratch_arena, uint32_t, capacity);
909
910 if (parser->bindings_capacity > 0) {
911 memcpy(bindings, parser->bindings, parser->bindings_capacity * sizeof(uint32_t));
912 }
913
914 parser->bindings = bindings;
915 parser->bindings_capacity = capacity;
916}
917
923static void
924pm_parser_locals_grow(pm_parser_t *parser) {
925 uint32_t capacity = parser->locals_capacity == 0 ? 16 : parser->locals_capacity * 2;
926 pm_local_t *locals = PM_ARENA_ALLOC_ARRAY(&parser->scratch_arena, pm_local_t, capacity);
927
928 if (parser->locals_capacity > 0) {
929 memcpy(locals, parser->locals, parser->locals_size * sizeof(pm_local_t));
930 }
931
932 parser->locals = locals;
933 parser->locals_capacity = capacity;
934}
935
951static void
952pm_locals_write(pm_parser_t *parser, pm_constant_id_t name, uint32_t start, uint32_t length, uint32_t reads) {
953 const pm_scope_t *scope = parser->current_scope;
954 uint32_t shadowed = pm_parser_binding(parser, name);
955
956 /* The locals of the current scope are at the top of the stack, so if it
957 * declares the name then that local is the innermost one. */
958 if (shadowed >= scope->locals_start) return;
959
960 if (name >= parser->bindings_capacity) pm_parser_bindings_grow(parser);
961 PRISM_ASSERT(name < parser->bindings_capacity);
962
963 if (parser->locals_size >= parser->locals_capacity) pm_parser_locals_grow(parser);
964
965 uint32_t index = parser->locals_size++;
966
967 parser->locals[index] = (pm_local_t) {
968 .name = name,
969 .location = { .start = start, .length = length },
970 .reads = reads,
971 .level = scope->level,
972 .shadowed = shadowed
973 };
974
975 parser->bindings[name] = index;
976}
977
984static pm_local_t *
985pm_locals_get(const pm_parser_t *parser, pm_constant_id_t name) {
986 uint32_t index = pm_parser_binding(parser, name);
987 PRISM_ASSERT(index >= parser->current_scope->floor);
988
989 return &parser->locals[index];
990}
991
996static void
997pm_locals_read(const pm_parser_t *parser, pm_constant_id_t name) {
998 pm_local_t *local = pm_locals_get(parser, name);
999 PRISM_ASSERT(local->reads < UINT32_MAX);
1000
1001 local->reads++;
1002}
1003
1008static void
1009pm_locals_unread(const pm_parser_t *parser, pm_constant_id_t name) {
1010 pm_local_t *local = pm_locals_get(parser, name);
1011 PRISM_ASSERT(local->reads > 0);
1012
1013 local->reads--;
1014}
1015
1019static uint32_t
1020pm_locals_reads(const pm_parser_t *parser, pm_constant_id_t name) {
1021 return pm_locals_get(parser, name)->reads;
1022}
1023
1033static void
1034pm_locals_order(pm_parser_t *parser, pm_constant_id_list_t *list, bool toplevel) {
1035 uint32_t start = parser->current_scope->locals_start;
1036 pm_constant_id_list_init(parser->arena, list, parser->locals_size - start);
1037
1038 // We will only warn for unused variables if we're not at the top level, or
1039 // if we're parsing a file outside of eval or -e.
1040 bool warn_unused = !toplevel || (!parser->parsing_eval && !PM_PARSER_COMMAND_LINE_OPTION_E(parser));
1041
1042 for (uint32_t index = start; index < parser->locals_size; index++) {
1043 pm_local_t *local = &parser->locals[index];
1044 pm_constant_id_list_insert(list, index - start, local->name);
1045
1046 if (warn_unused && local->reads == 0 && ((parser->start_line >= 0) || (pm_line_offset_list_line(&parser->line_offsets, local->location.start, parser->start_line) >= 0))) {
1047 pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, local->name);
1048
1049 if (constant->length >= 1 && *constant->start != '_') {
1050 PM_PARSER_WARN_FORMAT(
1051 parser,
1052 local->location.start,
1053 local->location.length,
1054 PM_WARN_UNUSED_LOCAL_VARIABLE,
1055 (int) constant->length,
1056 (const char *) constant->start
1057 );
1058 }
1059 }
1060 }
1061}
1062
1063/******************************************************************************/
1064/* Node-related functions */
1065/******************************************************************************/
1066
1071pm_parser_constant_id_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
1072 /* Fast path: if this is the same token as the last lookup (same pointer
1073 * range), return the cached result. */
1074 if (start == parser->constant_cache.start && end == parser->constant_cache.end) {
1075 return parser->constant_cache.id;
1076 }
1077
1078 pm_constant_id_t id = pm_constant_pool_insert_shared(&parser->metadata_arena, &parser->constant_pool, start, (size_t) (end - start));
1079
1080 parser->constant_cache.start = start;
1081 parser->constant_cache.end = end;
1082 parser->constant_cache.id = id;
1083
1084 return id;
1085}
1086
1091pm_parser_constant_id_owned(pm_parser_t *parser, uint8_t *start, size_t length) {
1092 return pm_constant_pool_insert_owned(&parser->metadata_arena, &parser->constant_pool, start, length);
1093}
1094
1099pm_parser_constant_id_constant(pm_parser_t *parser, const char *start, size_t length) {
1100 return pm_constant_pool_insert_constant(&parser->metadata_arena, &parser->constant_pool, (const uint8_t *) start, length);
1101}
1102
1107pm_parser_constant_id_token(pm_parser_t *parser, const pm_token_t *token) {
1108 return pm_parser_constant_id_raw(parser, token->start, token->end);
1109}
1110
1115#define PM_CASE_VOID_VALUE PM_RETURN_NODE: case PM_BREAK_NODE: case PM_NEXT_NODE: \
1116 case PM_REDO_NODE: case PM_RETRY_NODE: case PM_MATCH_REQUIRED_NODE
1117
1123static pm_node_t *
1124pm_check_value_expression(pm_parser_t *parser, pm_node_t *node) {
1125 pm_node_t *void_node = NULL;
1126
1127 while (node != NULL) {
1128 switch (PM_NODE_TYPE(node)) {
1129 case PM_CASE_VOID_VALUE:
1130 return void_node != NULL ? void_node : node;
1131 case PM_MATCH_PREDICATE_NODE:
1132 return NULL;
1133 case PM_BEGIN_NODE: {
1134 pm_begin_node_t *cast = (pm_begin_node_t *) node;
1135
1136 if (cast->rescue_clause != NULL) {
1137 /* Check the begin/rescue/else value before the ensure body. */
1138 node = UP(cast->ensure_clause);
1139 pm_node_t *rescue_void = NULL;
1140
1141 /* https://bugs.ruby-lang.org/issues/21669 */
1142 if (cast->else_clause == NULL || parser->version < PM_OPTIONS_VERSION_CRUBY_4_1) {
1143 rescue_void = pm_check_value_expression(parser, UP(cast->statements));
1144 if (rescue_void == NULL) break;
1145 }
1146
1147 bool rescue_has_value = false;
1148 for (pm_rescue_node_t *rescue_clause = cast->rescue_clause; rescue_clause != NULL; rescue_clause = rescue_clause->subsequent) {
1149 pm_node_t *vn = pm_check_value_expression(parser, UP(rescue_clause->statements));
1150
1151 if (vn == NULL) {
1152 rescue_has_value = true;
1153 break;
1154 }
1155 }
1156
1157 if (rescue_has_value && parser->version >= PM_OPTIONS_VERSION_CRUBY_4_1) break;
1158
1159 if (cast->else_clause != NULL) {
1160 pm_node_t *vn = pm_check_value_expression(parser, UP(cast->else_clause));
1161 if (vn != NULL) {
1162 if (parser->version < PM_OPTIONS_VERSION_CRUBY_4_1 && !rescue_has_value) {
1163 if (void_node != NULL) return void_node;
1164 if (rescue_void != NULL) return rescue_void;
1165 }
1166 return vn;
1167 }
1168 } else if (!rescue_has_value && rescue_void != NULL) {
1169 return void_node != NULL ? void_node : rescue_void;
1170 }
1171 } else if (cast->ensure_clause != NULL) {
1172 pm_node_t *vn = pm_check_value_expression(parser, UP(cast->statements));
1173 if (vn != NULL) return vn;
1174 node = UP(cast->ensure_clause);
1175 } else {
1176 node = UP(cast->statements);
1177 }
1178
1179 break;
1180 }
1181 case PM_CASE_NODE: {
1182 // https://bugs.ruby-lang.org/issues/21669
1183 if (parser->version < PM_OPTIONS_VERSION_CRUBY_4_1) {
1184 return NULL;
1185 }
1186
1187 pm_case_node_t *cast = (pm_case_node_t *) node;
1188 if (cast->else_clause == NULL) return NULL;
1189
1190 pm_node_t *condition;
1191 PM_NODE_LIST_FOREACH(&cast->conditions, index, condition) {
1192 PRISM_ASSERT(PM_NODE_TYPE_P(condition, PM_WHEN_NODE));
1193
1194 pm_when_node_t *cast = (pm_when_node_t *) condition;
1195 pm_node_t *vn = pm_check_value_expression(parser, UP(cast->statements));
1196 if (vn == NULL) return NULL;
1197 if (void_node == NULL) void_node = vn;
1198 }
1199
1200 node = UP(cast->else_clause);
1201 break;
1202 }
1203 case PM_CASE_MATCH_NODE: {
1204 // https://bugs.ruby-lang.org/issues/21669
1205 if (parser->version < PM_OPTIONS_VERSION_CRUBY_4_1) {
1206 return NULL;
1207 }
1208
1209 pm_case_match_node_t *cast = (pm_case_match_node_t *) node;
1210 if (cast->else_clause == NULL) return NULL;
1211
1212 pm_node_t *condition;
1213 PM_NODE_LIST_FOREACH(&cast->conditions, index, condition) {
1214 PRISM_ASSERT(PM_NODE_TYPE_P(condition, PM_IN_NODE));
1215
1216 pm_in_node_t *cast = (pm_in_node_t *) condition;
1217 pm_node_t *vn = pm_check_value_expression(parser, UP(cast->statements));
1218 if (vn == NULL) return NULL;
1219 if (void_node == NULL) void_node = vn;
1220 }
1221
1222 node = UP(cast->else_clause);
1223 break;
1224 }
1225 case PM_ENSURE_NODE: {
1226 pm_ensure_node_t *cast = (pm_ensure_node_t *) node;
1227 node = UP(cast->statements);
1228 break;
1229 }
1230 case PM_PARENTHESES_NODE: {
1231 pm_parentheses_node_t *cast = (pm_parentheses_node_t *) node;
1232 node = UP(cast->body);
1233 break;
1234 }
1235 case PM_STATEMENTS_NODE: {
1236 pm_statements_node_t *cast = (pm_statements_node_t *) node;
1237
1238 // https://bugs.ruby-lang.org/issues/21669
1239 if (parser->version >= PM_OPTIONS_VERSION_CRUBY_4_1) {
1240 pm_node_t *body_part;
1241 PM_NODE_LIST_FOREACH(&cast->body, index, body_part) {
1242 switch (PM_NODE_TYPE(body_part)) {
1243 case PM_CASE_VOID_VALUE:
1244 if (void_node == NULL) {
1245 void_node = body_part;
1246 }
1247 return void_node;
1248 default: break;
1249 }
1250 }
1251 }
1252
1253 node = cast->body.nodes[cast->body.size - 1];
1254 break;
1255 }
1256 case PM_IF_NODE: {
1257 pm_if_node_t *cast = (pm_if_node_t *) node;
1258 if (cast->statements == NULL || cast->subsequent == NULL) {
1259 return NULL;
1260 }
1261 pm_node_t *vn = pm_check_value_expression(parser, UP(cast->statements));
1262 if (vn == NULL) {
1263 return NULL;
1264 }
1265 if (void_node == NULL) {
1266 void_node = vn;
1267 }
1268 node = cast->subsequent;
1269 break;
1270 }
1271 case PM_UNLESS_NODE: {
1272 pm_unless_node_t *cast = (pm_unless_node_t *) node;
1273 if (cast->statements == NULL || cast->else_clause == NULL) {
1274 return NULL;
1275 }
1276 pm_node_t *vn = pm_check_value_expression(parser, UP(cast->statements));
1277 if (vn == NULL) {
1278 return NULL;
1279 }
1280 if (void_node == NULL) {
1281 void_node = vn;
1282 }
1283 node = UP(cast->else_clause);
1284 break;
1285 }
1286 case PM_ELSE_NODE: {
1287 pm_else_node_t *cast = (pm_else_node_t *) node;
1288 node = UP(cast->statements);
1289 break;
1290 }
1291 case PM_AND_NODE:
1292 case PM_OR_NODE:
1293 // The left operand of an and/or node was already checked for a
1294 // value when the node was created, so descending into it again
1295 // would re-report the same void value and, in a chain such as
1296 // `a && a && ...`, walk the whole left branch on every operator,
1297 // which is quadratic in the length of the chain.
1298 return NULL;
1299 case PM_LOCAL_VARIABLE_WRITE_NODE: {
1300 pm_local_variable_write_node_t *cast = (pm_local_variable_write_node_t *) node;
1301
1302 PRISM_ASSERT(pm_parser_local_depth_constant_id(parser, cast->name) == (int) cast->depth);
1303 pm_locals_read(parser, cast->name);
1304 return NULL;
1305 }
1306 default:
1307 return NULL;
1308 }
1309 }
1310
1311 return NULL;
1312}
1313
1314static PRISM_INLINE void
1315pm_assert_value_expression(pm_parser_t *parser, pm_node_t *node) {
1316 pm_node_t *void_node = pm_check_value_expression(parser, node);
1317 if (void_node != NULL) {
1318 pm_parser_err_node(parser, void_node, PM_ERR_VOID_EXPRESSION);
1319 }
1320}
1321
1325static void
1326pm_void_statement_check(pm_parser_t *parser, const pm_node_t *node) {
1327 const char *type = NULL;
1328 int length = 0;
1329
1330 switch (PM_NODE_TYPE(node)) {
1331 case PM_BACK_REFERENCE_READ_NODE:
1332 case PM_CLASS_VARIABLE_READ_NODE:
1333 case PM_GLOBAL_VARIABLE_READ_NODE:
1334 case PM_INSTANCE_VARIABLE_READ_NODE:
1335 case PM_IT_LOCAL_VARIABLE_READ_NODE:
1336 case PM_LOCAL_VARIABLE_READ_NODE:
1337 case PM_NUMBERED_REFERENCE_READ_NODE:
1338 type = "a variable";
1339 length = 10;
1340 break;
1341 case PM_CALL_NODE: {
1342 const pm_call_node_t *cast = (const pm_call_node_t *) node;
1343 if (cast->call_operator_loc.length > 0 || cast->message_loc.length == 0) break;
1344
1345 const pm_constant_t *message = pm_constant_pool_id_to_constant(&parser->constant_pool, cast->name);
1346 switch (message->length) {
1347 case 1:
1348 switch (message->start[0]) {
1349 case '+':
1350 case '-':
1351 case '*':
1352 case '/':
1353 case '%':
1354 case '|':
1355 case '^':
1356 case '&':
1357 case '>':
1358 case '<':
1359 type = (const char *) message->start;
1360 length = 1;
1361 break;
1362 }
1363 break;
1364 case 2:
1365 switch (message->start[1]) {
1366 case '=':
1367 if (message->start[0] == '<' || message->start[0] == '>' || message->start[0] == '!' || message->start[0] == '=') {
1368 type = (const char *) message->start;
1369 length = 2;
1370 }
1371 break;
1372 case '@':
1373 if (message->start[0] == '+' || message->start[0] == '-') {
1374 type = (const char *) message->start;
1375 length = 2;
1376 }
1377 break;
1378 case '*':
1379 if (message->start[0] == '*') {
1380 type = (const char *) message->start;
1381 length = 2;
1382 }
1383 break;
1384 }
1385 break;
1386 case 3:
1387 if (memcmp(message->start, "<=>", 3) == 0) {
1388 type = "<=>";
1389 length = 3;
1390 }
1391 break;
1392 }
1393
1394 break;
1395 }
1396 case PM_CONSTANT_PATH_NODE:
1397 type = "::";
1398 length = 2;
1399 break;
1400 case PM_CONSTANT_READ_NODE:
1401 type = "a constant";
1402 length = 10;
1403 break;
1404 case PM_DEFINED_NODE:
1405 type = "defined?";
1406 length = 8;
1407 break;
1408 case PM_FALSE_NODE:
1409 type = "false";
1410 length = 5;
1411 break;
1412 case PM_FLOAT_NODE:
1413 case PM_IMAGINARY_NODE:
1414 case PM_INTEGER_NODE:
1415 case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE:
1416 case PM_INTERPOLATED_STRING_NODE:
1417 case PM_RATIONAL_NODE:
1418 case PM_REGULAR_EXPRESSION_NODE:
1419 case PM_SOURCE_ENCODING_NODE:
1420 case PM_SOURCE_FILE_NODE:
1421 case PM_SOURCE_LINE_NODE:
1422 case PM_STRING_NODE:
1423 case PM_SYMBOL_NODE:
1424 type = "a literal";
1425 length = 9;
1426 break;
1427 case PM_NIL_NODE:
1428 type = "nil";
1429 length = 3;
1430 break;
1431 case PM_RANGE_NODE: {
1432 const pm_range_node_t *cast = (const pm_range_node_t *) node;
1433
1434 if (PM_NODE_FLAG_P(cast, PM_RANGE_FLAGS_EXCLUDE_END)) {
1435 type = "...";
1436 length = 3;
1437 } else {
1438 type = "..";
1439 length = 2;
1440 }
1441
1442 break;
1443 }
1444 case PM_SELF_NODE:
1445 type = "self";
1446 length = 4;
1447 break;
1448 case PM_TRUE_NODE:
1449 type = "true";
1450 length = 4;
1451 break;
1452 default:
1453 break;
1454 }
1455
1456 if (type != NULL) {
1457 PM_PARSER_WARN_NODE_FORMAT(parser, node, PM_WARN_VOID_STATEMENT, length, type);
1458 }
1459}
1460
1465static void
1466pm_void_statement_check_unwrap(pm_parser_t *parser, const pm_node_t *node) {
1467 switch (PM_NODE_TYPE(node)) {
1468 case PM_BEGIN_NODE: {
1469 const pm_begin_node_t *cast = (const pm_begin_node_t *) node;
1470 if (cast->statements == NULL || cast->rescue_clause != NULL || cast->else_clause != NULL || cast->ensure_clause != NULL) break;
1471
1472 // The rest of the body was already checked when it was parsed.
1473 const pm_node_list_t *body = &cast->statements->body;
1474 pm_void_statement_check_unwrap(parser, body->nodes[body->size - 1]);
1475 break;
1476 }
1477 case PM_PARENTHESES_NODE: {
1478 const pm_parentheses_node_t *cast = (const pm_parentheses_node_t *) node;
1479 if (cast->body == NULL || !PM_NODE_TYPE_P(cast->body, PM_STATEMENTS_NODE)) break;
1480
1481 // Unlike begin, parse.y only looks one level into parentheses, so
1482 // `((foo))` doesn't warn.
1483 const pm_statements_node_t *statements = (const pm_statements_node_t *) cast->body;
1484 if (statements->body.size == 1) pm_void_statement_check(parser, statements->body.nodes[0]);
1485 break;
1486 }
1487 default:
1488 pm_void_statement_check(parser, node);
1489 break;
1490 }
1491}
1492
1497static void
1498pm_void_statements_check(pm_parser_t *parser, const pm_statements_node_t *node, bool last_value) {
1499 PRISM_ASSERT(node->body.size > 0);
1500 const size_t size = node->body.size - (last_value ? 1 : 0);
1501 for (size_t index = 0; index < size; index++) {
1502 pm_void_statement_check_unwrap(parser, node->body.nodes[index]);
1503 }
1504}
1505
1511typedef enum {
1512 PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL,
1513 PM_CONDITIONAL_PREDICATE_TYPE_FLIP_FLOP,
1514 PM_CONDITIONAL_PREDICATE_TYPE_NOT
1515} pm_conditional_predicate_type_t;
1516
1520static void
1521pm_parser_warn_conditional_predicate_literal(pm_parser_t *parser, pm_node_t *node, pm_conditional_predicate_type_t type, pm_diagnostic_id_t diag_id, const char *prefix) {
1522 switch (type) {
1523 case PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL:
1524 PM_PARSER_WARN_NODE_FORMAT(parser, node, diag_id, prefix, "condition");
1525 break;
1526 case PM_CONDITIONAL_PREDICATE_TYPE_FLIP_FLOP:
1527 PM_PARSER_WARN_NODE_FORMAT(parser, node, diag_id, prefix, "flip-flop");
1528 break;
1529 case PM_CONDITIONAL_PREDICATE_TYPE_NOT:
1530 break;
1531 }
1532}
1533
1538static bool
1539pm_conditional_predicate_warn_write_literal_p(const pm_node_t *node) {
1540 switch (PM_NODE_TYPE(node)) {
1541 case PM_ARRAY_NODE: {
1542 if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) return true;
1543
1544 const pm_array_node_t *cast = (const pm_array_node_t *) node;
1545 for (size_t index = 0; index < cast->elements.size; index++) {
1546 if (!pm_conditional_predicate_warn_write_literal_p(cast->elements.nodes[index])) return false;
1547 }
1548
1549 return true;
1550 }
1551 case PM_HASH_NODE: {
1552 if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) return true;
1553
1554 const pm_hash_node_t *cast = (const pm_hash_node_t *) node;
1555 for (size_t index = 0; index < cast->elements.size; index++) {
1556 const pm_node_t *element = cast->elements.nodes[index];
1557 if (!PM_NODE_TYPE_P(element, PM_ASSOC_NODE)) return false;
1558
1559 const pm_assoc_node_t *assoc = (const pm_assoc_node_t *) element;
1560 if (!pm_conditional_predicate_warn_write_literal_p(assoc->key) || !pm_conditional_predicate_warn_write_literal_p(assoc->value)) return false;
1561 }
1562
1563 return true;
1564 }
1565 case PM_FALSE_NODE:
1566 case PM_FLOAT_NODE:
1567 case PM_IMAGINARY_NODE:
1568 case PM_INTEGER_NODE:
1569 case PM_NIL_NODE:
1570 case PM_RATIONAL_NODE:
1571 case PM_REGULAR_EXPRESSION_NODE:
1572 case PM_SOURCE_ENCODING_NODE:
1573 case PM_SOURCE_FILE_NODE:
1574 case PM_SOURCE_LINE_NODE:
1575 case PM_STRING_NODE:
1576 case PM_SYMBOL_NODE:
1577 case PM_TRUE_NODE:
1578 return true;
1579 default:
1580 return false;
1581 }
1582}
1583
1588static PRISM_INLINE void
1589pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, const pm_node_t *node) {
1590 if (pm_conditional_predicate_warn_write_literal_p(node)) {
1591 pm_parser_warn_node(parser, node, parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_WARN_EQUAL_IN_CONDITIONAL_3_3 : PM_WARN_EQUAL_IN_CONDITIONAL);
1592 }
1593}
1594
1607static void
1608pm_conditional_predicate(pm_parser_t *parser, pm_node_t *node, pm_conditional_predicate_type_t type) {
1609 switch (PM_NODE_TYPE(node)) {
1610 case PM_AND_NODE: {
1611 pm_and_node_t *cast = (pm_and_node_t *) node;
1612 pm_conditional_predicate(parser, cast->left, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
1613 pm_conditional_predicate(parser, cast->right, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
1614 break;
1615 }
1616 case PM_OR_NODE: {
1617 pm_or_node_t *cast = (pm_or_node_t *) node;
1618 pm_conditional_predicate(parser, cast->left, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
1619 pm_conditional_predicate(parser, cast->right, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
1620 break;
1621 }
1622 case PM_PARENTHESES_NODE: {
1623 pm_parentheses_node_t *cast = (pm_parentheses_node_t *) node;
1624
1625 if ((cast->body != NULL) && PM_NODE_TYPE_P(cast->body, PM_STATEMENTS_NODE)) {
1626 pm_statements_node_t *statements = (pm_statements_node_t *) cast->body;
1627 if (statements->body.size == 1) pm_conditional_predicate(parser, statements->body.nodes[0], type);
1628 }
1629
1630 break;
1631 }
1632 case PM_BEGIN_NODE: {
1633 pm_begin_node_t *cast = (pm_begin_node_t *) node;
1634 if (cast->statements != NULL) {
1635 pm_statements_node_t *statements = cast->statements;
1636 if (statements->body.size == 1) pm_conditional_predicate(parser, statements->body.nodes[0], type);
1637 }
1638 break;
1639 }
1640 case PM_RANGE_NODE: {
1641 pm_range_node_t *cast = (pm_range_node_t *) node;
1642
1643 if (cast->left != NULL) pm_conditional_predicate(parser, cast->left, PM_CONDITIONAL_PREDICATE_TYPE_FLIP_FLOP);
1644 if (cast->right != NULL) pm_conditional_predicate(parser, cast->right, PM_CONDITIONAL_PREDICATE_TYPE_FLIP_FLOP);
1645
1646 // Here we change the range node into a flip flop node. We can do
1647 // this since the nodes are exactly the same except for the type.
1648 // We're only asserting against the size when we should probably
1649 // assert against the entire layout, but we'll assume tests will
1650 // catch this.
1651 PRISM_ASSERT(sizeof(pm_range_node_t) == sizeof(pm_flip_flop_node_t));
1652 node->type = PM_FLIP_FLOP_NODE;
1653
1654 break;
1655 }
1656 case PM_REGULAR_EXPRESSION_NODE:
1657 // Here we change the regular expression node into a match last line
1658 // node. We can do this since the nodes are exactly the same except
1659 // for the type.
1660 PRISM_ASSERT(sizeof(pm_regular_expression_node_t) == sizeof(pm_match_last_line_node_t));
1661 node->type = PM_MATCH_LAST_LINE_NODE;
1662
1663 if (!PM_PARSER_COMMAND_LINE_OPTION_E(parser)) {
1664 pm_parser_warn_conditional_predicate_literal(parser, node, type, PM_WARN_LITERAL_IN_CONDITION_DEFAULT, "regex ");
1665 }
1666
1667 break;
1668 case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE:
1669 // Here we change the interpolated regular expression node into an
1670 // interpolated match last line node. We can do this since the nodes
1671 // are exactly the same except for the type.
1672 PRISM_ASSERT(sizeof(pm_interpolated_regular_expression_node_t) == sizeof(pm_interpolated_match_last_line_node_t));
1673 node->type = PM_INTERPOLATED_MATCH_LAST_LINE_NODE;
1674
1675 if (!PM_PARSER_COMMAND_LINE_OPTION_E(parser)) {
1676 pm_parser_warn_conditional_predicate_literal(parser, node, type, PM_WARN_LITERAL_IN_CONDITION_VERBOSE, "regex ");
1677 }
1678
1679 break;
1680 case PM_INTEGER_NODE:
1681 if (type == PM_CONDITIONAL_PREDICATE_TYPE_FLIP_FLOP) {
1682 if (!PM_PARSER_COMMAND_LINE_OPTION_E(parser)) {
1683 pm_parser_warn_node(parser, node, PM_WARN_INTEGER_IN_FLIP_FLOP);
1684 }
1685 } else {
1686 pm_parser_warn_conditional_predicate_literal(parser, node, type, PM_WARN_LITERAL_IN_CONDITION_VERBOSE, "");
1687 }
1688 break;
1689 case PM_STRING_NODE:
1690 case PM_SOURCE_FILE_NODE:
1691 case PM_INTERPOLATED_STRING_NODE:
1692 pm_parser_warn_conditional_predicate_literal(parser, node, type, PM_WARN_LITERAL_IN_CONDITION_DEFAULT, "string ");
1693 break;
1694 case PM_SYMBOL_NODE:
1695 case PM_INTERPOLATED_SYMBOL_NODE:
1696 pm_parser_warn_conditional_predicate_literal(parser, node, type, PM_WARN_LITERAL_IN_CONDITION_VERBOSE, "symbol ");
1697 break;
1698 case PM_SOURCE_LINE_NODE:
1699 case PM_SOURCE_ENCODING_NODE:
1700 case PM_FLOAT_NODE:
1701 case PM_RATIONAL_NODE:
1702 case PM_IMAGINARY_NODE:
1703 pm_parser_warn_conditional_predicate_literal(parser, node, type, PM_WARN_LITERAL_IN_CONDITION_VERBOSE, "");
1704 break;
1705 case PM_CLASS_VARIABLE_WRITE_NODE:
1706 pm_conditional_predicate_warn_write_literal(parser, ((pm_class_variable_write_node_t *) node)->value);
1707 break;
1708 case PM_CONSTANT_WRITE_NODE:
1709 pm_conditional_predicate_warn_write_literal(parser, ((pm_constant_write_node_t *) node)->value);
1710 break;
1711 case PM_GLOBAL_VARIABLE_WRITE_NODE:
1712 pm_conditional_predicate_warn_write_literal(parser, ((pm_global_variable_write_node_t *) node)->value);
1713 break;
1714 case PM_INSTANCE_VARIABLE_WRITE_NODE:
1715 pm_conditional_predicate_warn_write_literal(parser, ((pm_instance_variable_write_node_t *) node)->value);
1716 break;
1717 case PM_LOCAL_VARIABLE_WRITE_NODE:
1718 pm_conditional_predicate_warn_write_literal(parser, ((pm_local_variable_write_node_t *) node)->value);
1719 break;
1720 case PM_MULTI_WRITE_NODE:
1721 pm_conditional_predicate_warn_write_literal(parser, ((pm_multi_write_node_t *) node)->value);
1722 break;
1723 default:
1724 break;
1725 }
1726}
1727
1734typedef struct {
1737
1739 pm_node_list_t list;
1740
1743
1745 pm_node_flags_t list_flags;
1746
1749
1751 pm_node_t *block;
1752
1756
1760static PRISM_INLINE const pm_location_t *
1761pm_arguments_end(pm_arguments_t *arguments) {
1762 if (arguments->block != NULL) {
1763 uint32_t end = PM_NODE_END(arguments->block);
1764
1765 if (arguments->closing_loc.length > 0) {
1766 uint32_t arguments_end = PM_LOCATION_END(&arguments->closing_loc);
1767 if (arguments_end > end) {
1768 return &arguments->closing_loc;
1769 }
1770 }
1771 return &arguments->block->location;
1772 }
1773 if (arguments->closing_loc.length > 0) {
1774 return &arguments->closing_loc;
1775 }
1776 if (arguments->list.size > 0) {
1777 return &arguments->list_loc;
1778 }
1779 if (arguments->opening_loc.length > 0) {
1780 return &arguments->opening_loc;
1781 }
1782 return NULL;
1783}
1784
1789static void
1790pm_arguments_validate_block(pm_parser_t *parser, pm_arguments_t *arguments, pm_block_node_t *block) {
1791 // First, check that we have arguments and that we don't have a closing
1792 // location for them.
1793 if (arguments->list.size == 0 || arguments->closing_loc.length > 0) {
1794 return;
1795 }
1796
1797 // Next, check that we don't have a single parentheses argument. This would
1798 // look like:
1799 //
1800 // foo (1) {}
1801 //
1802 // In this case, it's actually okay for the block to be attached to the
1803 // call, even though it looks like it's attached to the argument.
1804 if (arguments->list.size == 1 && PM_NODE_TYPE_P(pm_node_list_open_last(&arguments->list), PM_PARENTHESES_NODE)) {
1805 return;
1806 }
1807
1808 // If we didn't hit a case before this check, then at this point we need to
1809 // add a syntax error.
1810 pm_parser_err_node(parser, UP(block), PM_ERR_ARGUMENT_UNEXPECTED_BLOCK);
1811}
1812
1813/******************************************************************************/
1814/* Basic character checks */
1815/******************************************************************************/
1816
1825static PRISM_INLINE size_t
1826char_is_identifier_start(const pm_parser_t *parser, const uint8_t *b, ptrdiff_t n) {
1827 if (n <= 0) return 0;
1828
1829 if (parser->encoding_changed && *b >= 0x80) {
1830 return pm_encoding_char_width(parser->encoding, b, n);
1831 } else if (*b < 0x80) {
1832 return pm_encoding_ascii_alpha_p(*b) || (*b == '_');
1833 } else {
1834 return pm_encoding_utf_8_char_width(b, n);
1835 }
1836}
1837
1843static PRISM_INLINE size_t
1844char_is_identifier_utf8(const uint8_t *b, ptrdiff_t n) {
1845 if (n <= 0) {
1846 return 0;
1847 } else if (*b < 0x80) {
1848 return (*b == '_') || pm_encoding_ascii_alnum_p(*b);
1849 } else {
1850 return pm_encoding_utf_8_char_width(b, n);
1851 }
1852}
1853
1866
1867#if defined(PRISM_HAS_NEON)
1868#include <arm_neon.h>
1869
1870static PRISM_INLINE size_t
1871scan_identifier_ascii(const uint8_t *start, const uint8_t *end) {
1872 const uint8_t *cursor = start;
1873
1874 // Nibble-based lookup tables for classifying [a-zA-Z0-9_].
1875 // Each high nibble is assigned a unique bit; the low nibble table
1876 // contains the OR of bits for all high nibbles that have an
1877 // identifier character at that low nibble position. A byte is an
1878 // identifier character iff (low_lut[lo] & high_lut[hi]) != 0.
1879 static const uint8_t low_lut_data[16] = {
1880 0x15, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F,
1881 0x1F, 0x1F, 0x1E, 0x0A, 0x0A, 0x0A, 0x0A, 0x0E
1882 };
1883 static const uint8_t high_lut_data[16] = {
1884 0x00, 0x00, 0x00, 0x01, 0x02, 0x04, 0x08, 0x10,
1885 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
1886 };
1887 const uint8x16_t low_lut = vld1q_u8(low_lut_data);
1888 const uint8x16_t high_lut = vld1q_u8(high_lut_data);
1889 const uint8x16_t mask_0f = vdupq_n_u8(0x0F);
1890
1891 while (cursor + 16 <= end) {
1892 uint8x16_t v = vld1q_u8(cursor);
1893
1894 uint8x16_t lo_class = vqtbl1q_u8(low_lut, vandq_u8(v, mask_0f));
1895 uint8x16_t hi_class = vqtbl1q_u8(high_lut, vshrq_n_u8(v, 4));
1896 uint8x16_t ident = vandq_u8(lo_class, hi_class);
1897
1898 // Fast check: if the per-byte minimum is nonzero, every byte matched.
1899 if (vminvq_u8(ident) != 0) {
1900 cursor += 16;
1901 continue;
1902 }
1903
1904 // Find the first non-identifier byte (zero in ident).
1905 uint8x16_t is_zero = vceqq_u8(ident, vdupq_n_u8(0));
1906 uint64_t lo = vgetq_lane_u64(vreinterpretq_u64_u8(is_zero), 0);
1907
1908 if (lo != 0) {
1909 cursor += pm_ctzll(lo) / 8;
1910 } else {
1911 uint64_t hi = vgetq_lane_u64(vreinterpretq_u64_u8(is_zero), 1);
1912 cursor += 8 + pm_ctzll(hi) / 8;
1913 }
1914
1915 return (size_t) (cursor - start);
1916 }
1917
1918 return (size_t) (cursor - start);
1919}
1920
1921#elif defined(PRISM_HAS_SSE2)
1922#include <emmintrin.h>
1923
1924static PRISM_INLINE size_t
1925scan_identifier_ascii(const uint8_t *start, const uint8_t *end) {
1926 const uint8_t *cursor = start;
1927
1928 while (cursor + 16 <= end) {
1929 __m128i v = _mm_loadu_si128((const __m128i *) cursor);
1930 __m128i zero = _mm_setzero_si128();
1931
1932 // Unsigned range check via saturating subtraction:
1933 // byte >= lo ⟺ saturate(lo - byte) == 0
1934 // byte <= hi ⟺ saturate(byte - hi) == 0
1935
1936 // Fold case: OR with 0x20 maps A-Z to a-z.
1937 __m128i lowered = _mm_or_si128(v, _mm_set1_epi8(0x20));
1938 __m128i letter = _mm_and_si128(
1939 _mm_cmpeq_epi8(_mm_subs_epu8(_mm_set1_epi8(0x61), lowered), zero),
1940 _mm_cmpeq_epi8(_mm_subs_epu8(lowered, _mm_set1_epi8(0x7A)), zero));
1941
1942 __m128i digit = _mm_and_si128(
1943 _mm_cmpeq_epi8(_mm_subs_epu8(_mm_set1_epi8(0x30), v), zero),
1944 _mm_cmpeq_epi8(_mm_subs_epu8(v, _mm_set1_epi8(0x39)), zero));
1945
1946 __m128i underscore = _mm_cmpeq_epi8(v, _mm_set1_epi8(0x5F));
1947
1948 __m128i ident = _mm_or_si128(_mm_or_si128(letter, digit), underscore);
1949 int mask = _mm_movemask_epi8(ident);
1950
1951 if (mask == 0xFFFF) {
1952 cursor += 16;
1953 continue;
1954 }
1955
1956 cursor += pm_ctzll((uint64_t) (~mask & 0xFFFF));
1957 return (size_t) (cursor - start);
1958 }
1959
1960 return (size_t) (cursor - start);
1961}
1962
1963// The SWAR path uses pm_ctzll to find the first non-matching byte within a
1964// word, which only yields the correct byte index on little-endian targets.
1965// We gate on a positive little-endian check so that unknown-endianness
1966// platforms safely fall through to the no-op fallback.
1967#elif defined(PRISM_HAS_SWAR)
1968
1978static PRISM_INLINE size_t
1979scan_identifier_ascii(const uint8_t *start, const uint8_t *end) {
1980 static const uint64_t ones = 0x0101010101010101ULL;
1981 static const uint64_t highs = 0x8080808080808080ULL;
1982 const uint8_t *cursor = start;
1983
1984 while (cursor + 8 <= end) {
1985 uint64_t word;
1986 memcpy(&word, cursor, 8);
1987
1988 // Bail on any non-ASCII byte.
1989 if (word & highs) break;
1990
1991 uint64_t digit = ((word | highs) - ones * 0x30) & ((ones * 0x39 | highs) - word) & highs;
1992
1993 // Fold upper- and lowercase together by forcing bit 5 (OR 0x20),
1994 // then check the lowercase range once. A-Z maps to a-z; the
1995 // only non-letter byte that could alias into [0x61,0x7A] is one
1996 // whose original value was in [0x41,0x5A] — which is exactly
1997 // the uppercase letters we want to match.
1998 uint64_t lowered = word | (ones * 0x20);
1999 uint64_t letter = ((lowered | highs) - ones * 0x61) & ((ones * 0x7A | highs) - lowered) & highs;
2000
2001 // Standard SWAR "has zero byte" idiom on (word XOR 0x5F) to find
2002 // bytes equal to underscore. Safe from cross-byte borrows because
2003 // the ASCII guard above ensures all bytes are < 0x80.
2004 uint64_t xor_us = word ^ (ones * 0x5F);
2005 uint64_t underscore = (xor_us - ones) & ~xor_us & highs;
2006
2007 uint64_t ident = digit | letter | underscore;
2008
2009 if (ident == highs) {
2010 cursor += 8;
2011 continue;
2012 }
2013
2014 // Find the first non-identifier byte. On little-endian the first
2015 // byte sits in the least-significant position.
2016 uint64_t not_ident = ~ident & highs;
2017 cursor += pm_ctzll(not_ident) / 8;
2018 return (size_t) (cursor - start);
2019 }
2020
2021 return (size_t) (cursor - start);
2022}
2023
2024#else
2025
2026// No-op fallback for big-endian or other unsupported platforms.
2027// The caller's byte-at-a-time loop handles everything.
2028#define scan_identifier_ascii(start, end) ((size_t) 0)
2029
2030#endif
2031
2037static PRISM_INLINE size_t
2038char_is_identifier(const pm_parser_t *parser, const uint8_t *b, ptrdiff_t n) {
2039 if (n <= 0) {
2040 return 0;
2041 } else if (parser->encoding_changed && *b >= 0x80) {
2042 return pm_encoding_char_width(parser->encoding, b, n);
2043 } else {
2044 return char_is_identifier_utf8(b, n);
2045 }
2046}
2047
2048// Here we're defining a perfect hash for the characters that are allowed in
2049// global names. This is used to quickly check the next character after a $ to
2050// see if it's a valid character for a global name.
2051#define BIT(c, idx) (((c) / 32 - 1 == idx) ? (1U << ((c) % 32)) : 0)
2052#define PUNCT(idx) ( \
2053 BIT('~', idx) | BIT('*', idx) | BIT('$', idx) | BIT('?', idx) | \
2054 BIT('!', idx) | BIT('@', idx) | BIT('/', idx) | BIT('\\', idx) | \
2055 BIT(';', idx) | BIT(',', idx) | BIT('.', idx) | BIT('=', idx) | \
2056 BIT(':', idx) | BIT('<', idx) | BIT('>', idx) | BIT('\"', idx) | \
2057 BIT('&', idx) | BIT('`', idx) | BIT('\'', idx) | BIT('+', idx) | \
2058 BIT('0', idx))
2059
2060const unsigned int pm_global_name_punctuation_hash[(0x7e - 0x20 + 31) / 32] = { PUNCT(0), PUNCT(1), PUNCT(2) };
2061
2062#undef BIT
2063#undef PUNCT
2064
2065static PRISM_INLINE bool
2066char_is_global_name_punctuation(const uint8_t b) {
2067 const unsigned int i = (const unsigned int) b;
2068 if (i <= 0x20 || 0x7e < i) return false;
2069
2070 return (pm_global_name_punctuation_hash[(i - 0x20) / 32] >> (i % 32)) & 1;
2071}
2072
2073static PRISM_INLINE bool
2074token_is_setter_name(pm_token_t *token) {
2075 return (
2076 (token->type == PM_TOKEN_BRACKET_LEFT_RIGHT_EQUAL) ||
2077 ((token->type == PM_TOKEN_IDENTIFIER) &&
2078 (token->end - token->start >= 2) &&
2079 (token->end[-1] == '='))
2080 );
2081}
2082
2086static bool
2087pm_local_is_keyword(const char *source, size_t length) {
2088#define KEYWORD(name) if (memcmp(source, name, length) == 0) return true
2089
2090 switch (length) {
2091 case 2:
2092 switch (source[0]) {
2093 case 'd': KEYWORD("do"); return false;
2094 case 'i': KEYWORD("if"); KEYWORD("in"); return false;
2095 case 'o': KEYWORD("or"); return false;
2096 default: return false;
2097 }
2098 case 3:
2099 switch (source[0]) {
2100 case 'a': KEYWORD("and"); return false;
2101 case 'd': KEYWORD("def"); return false;
2102 case 'e': KEYWORD("end"); return false;
2103 case 'f': KEYWORD("for"); return false;
2104 case 'n': KEYWORD("nil"); KEYWORD("not"); return false;
2105 default: return false;
2106 }
2107 case 4:
2108 switch (source[0]) {
2109 case 'c': KEYWORD("case"); return false;
2110 case 'e': KEYWORD("else"); return false;
2111 case 'n': KEYWORD("next"); return false;
2112 case 'r': KEYWORD("redo"); return false;
2113 case 's': KEYWORD("self"); return false;
2114 case 't': KEYWORD("then"); KEYWORD("true"); return false;
2115 case 'w': KEYWORD("when"); return false;
2116 default: return false;
2117 }
2118 case 5:
2119 switch (source[0]) {
2120 case 'a': KEYWORD("alias"); return false;
2121 case 'b': KEYWORD("begin"); KEYWORD("break"); return false;
2122 case 'c': KEYWORD("class"); return false;
2123 case 'e': KEYWORD("elsif"); return false;
2124 case 'f': KEYWORD("false"); return false;
2125 case 'r': KEYWORD("retry"); return false;
2126 case 's': KEYWORD("super"); return false;
2127 case 'u': KEYWORD("undef"); KEYWORD("until"); return false;
2128 case 'w': KEYWORD("while"); return false;
2129 case 'y': KEYWORD("yield"); return false;
2130 default: return false;
2131 }
2132 case 6:
2133 switch (source[0]) {
2134 case 'e': KEYWORD("ensure"); return false;
2135 case 'm': KEYWORD("module"); return false;
2136 case 'r': KEYWORD("rescue"); KEYWORD("return"); return false;
2137 case 'u': KEYWORD("unless"); return false;
2138 default: return false;
2139 }
2140 case 8:
2141 KEYWORD("__LINE__");
2142 KEYWORD("__FILE__");
2143 return false;
2144 case 12:
2145 KEYWORD("__ENCODING__");
2146 return false;
2147 default:
2148 return false;
2149 }
2150
2151#undef KEYWORD
2152}
2153
2154/******************************************************************************/
2155/* Node flag handling functions */
2156/******************************************************************************/
2157
2161static PRISM_INLINE void
2162pm_node_flag_set(pm_node_t *node, pm_node_flags_t flag) {
2163 node->flags |= flag;
2164}
2165
2169static PRISM_INLINE void
2170pm_node_flag_unset(pm_node_t *node, pm_node_flags_t flag) {
2171 node->flags &= (pm_node_flags_t) ~flag;
2172}
2173
2177static PRISM_INLINE void
2178pm_node_flag_set_repeated_parameter(pm_node_t *node) {
2179 PRISM_ASSERT(PM_NODE_TYPE(node) == PM_BLOCK_LOCAL_VARIABLE_NODE ||
2180 PM_NODE_TYPE(node) == PM_BLOCK_PARAMETER_NODE ||
2181 PM_NODE_TYPE(node) == PM_KEYWORD_REST_PARAMETER_NODE ||
2182 PM_NODE_TYPE(node) == PM_OPTIONAL_KEYWORD_PARAMETER_NODE ||
2183 PM_NODE_TYPE(node) == PM_OPTIONAL_PARAMETER_NODE ||
2184 PM_NODE_TYPE(node) == PM_REQUIRED_KEYWORD_PARAMETER_NODE ||
2185 PM_NODE_TYPE(node) == PM_REQUIRED_PARAMETER_NODE ||
2186 PM_NODE_TYPE(node) == PM_REST_PARAMETER_NODE);
2187
2188 pm_node_flag_set(node, PM_PARAMETER_FLAGS_REPEATED_PARAMETER);
2189}
2190
2191/******************************************************************************/
2192/* Node creation functions */
2193/******************************************************************************/
2194
2200#define PM_REGULAR_EXPRESSION_ENCODING_MASK ~(PM_REGULAR_EXPRESSION_FLAGS_EUC_JP | PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT | PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J | PM_REGULAR_EXPRESSION_FLAGS_UTF_8)
2201
2205static PRISM_INLINE pm_node_flags_t
2206pm_regular_expression_flags_create(pm_parser_t *parser, const pm_token_t *closing) {
2207 pm_node_flags_t flags = 0;
2208
2209 if (closing->type == PM_TOKEN_REGEXP_END) {
2210 pm_buffer_t unknown_flags = { 0 };
2211
2212 // The closing delimiter is normally a single byte, so the options
2213 // follow it. A `\r\n` newline delimiter is two bytes, however, so we
2214 // skip past it to avoid misreading the trailing `\n` as an option.
2215 const uint8_t *flag = closing->start + 1;
2216 if ((closing->end - closing->start) >= 2 && closing->start[0] == '\r' && closing->start[1] == '\n') {
2217 flag++;
2218 }
2219
2220 for (; flag < closing->end; flag++) {
2221 switch (*flag) {
2222 case 'i': flags |= PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE; break;
2223 case 'm': flags |= PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE; break;
2224 case 'x': flags |= PM_REGULAR_EXPRESSION_FLAGS_EXTENDED; break;
2225 case 'o': flags |= PM_REGULAR_EXPRESSION_FLAGS_ONCE; break;
2226
2227 case 'e': flags = (pm_node_flags_t) (((pm_node_flags_t) (flags & PM_REGULAR_EXPRESSION_ENCODING_MASK)) | PM_REGULAR_EXPRESSION_FLAGS_EUC_JP); break;
2228 case 'n': flags = (pm_node_flags_t) (((pm_node_flags_t) (flags & PM_REGULAR_EXPRESSION_ENCODING_MASK)) | PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT); break;
2229 case 's': flags = (pm_node_flags_t) (((pm_node_flags_t) (flags & PM_REGULAR_EXPRESSION_ENCODING_MASK)) | PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J); break;
2230 case 'u': flags = (pm_node_flags_t) (((pm_node_flags_t) (flags & PM_REGULAR_EXPRESSION_ENCODING_MASK)) | PM_REGULAR_EXPRESSION_FLAGS_UTF_8); break;
2231
2232 default: pm_buffer_append_byte(&unknown_flags, *flag);
2233 }
2234 }
2235
2236 size_t unknown_flags_length = pm_buffer_length(&unknown_flags);
2237 if (unknown_flags_length != 0) {
2238 const char *word = unknown_flags_length >= 2 ? "options" : "option";
2239 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags));
2240 }
2241 pm_buffer_cleanup(&unknown_flags);
2242 }
2243
2244 return flags;
2245}
2246
2247#undef PM_REGULAR_EXPRESSION_ENCODING_MASK
2248
2249static pm_statements_node_t *
2250pm_statements_node_single_create(pm_parser_t *parser, pm_node_t *statement, bool newline);
2251
2252static size_t
2253pm_statements_node_body_length(pm_statements_node_t *node);
2254
2258static pm_error_recovery_node_t *
2259pm_error_recovery_node_create(pm_parser_t *parser, uint32_t start, uint32_t length) {
2260 return pm_error_recovery_node_new(
2261 parser->arena,
2262 ++parser->node_id,
2263 0,
2264 ((pm_location_t) { .start = start, .length = length }),
2265 NULL
2266 );
2267}
2268
2272static pm_error_recovery_node_t *
2273pm_error_recovery_node_create_unexpected(pm_parser_t *parser, pm_node_t *unexpected) {
2274 return pm_error_recovery_node_new(
2275 parser->arena,
2276 ++parser->node_id,
2277 0,
2278 unexpected->location,
2279 unexpected
2280 );
2281}
2282
2286static pm_alias_global_variable_node_t *
2287pm_alias_global_variable_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *new_name, pm_node_t *old_name) {
2288 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_ALIAS);
2289
2290 return pm_alias_global_variable_node_new(
2291 parser->arena,
2292 ++parser->node_id,
2293 0,
2294 PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, old_name),
2295 new_name,
2296 old_name,
2297 TOK2LOC(parser, keyword)
2298 );
2299}
2300
2304static pm_alias_method_node_t *
2305pm_alias_method_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *new_name, pm_node_t *old_name) {
2306 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_ALIAS);
2307
2308 return pm_alias_method_node_new(
2309 parser->arena,
2310 ++parser->node_id,
2311 0,
2312 PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, old_name),
2313 new_name,
2314 old_name,
2315 TOK2LOC(parser, keyword)
2316 );
2317}
2318
2322static pm_alternation_pattern_node_t *
2323pm_alternation_pattern_node_create(pm_parser_t *parser, pm_node_t *left, pm_node_t *right, const pm_token_t *operator) {
2324 return pm_alternation_pattern_node_new(
2325 parser->arena,
2326 ++parser->node_id,
2327 0,
2328 PM_LOCATION_INIT_NODES(left, right),
2329 left,
2330 right,
2331 TOK2LOC(parser, operator)
2332 );
2333}
2334
2338static pm_and_node_t *
2339pm_and_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *operator, pm_node_t *right) {
2340 pm_assert_value_expression(parser, left);
2341
2342 return pm_and_node_new(
2343 parser->arena,
2344 ++parser->node_id,
2345 0,
2346 PM_LOCATION_INIT_NODES(left, right),
2347 left,
2348 right,
2349 TOK2LOC(parser, operator)
2350 );
2351}
2352
2358static pm_arguments_node_t *
2359pm_arguments_node_create(pm_parser_t *parser, pm_arguments_t *arguments) {
2360 if (arguments->list.size == 0) return NULL;
2361 pm_node_list_finalize(&parser->node_chunks, parser->arena, &arguments->list);
2362
2363 return pm_arguments_node_new(
2364 parser->arena,
2365 ++parser->node_id,
2366 arguments->list_flags,
2367 arguments->list_loc,
2368 arguments->list
2369 );
2370}
2371
2375static void
2376pm_arguments_flags_update(pm_node_flags_t *flags, const pm_node_t *argument) {
2377 if (PM_NODE_TYPE_P(argument, PM_SPLAT_NODE)) {
2378 if (*flags & PM_ARGUMENTS_NODE_FLAGS_CONTAINS_SPLAT) {
2379 *flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_MULTIPLE_SPLATS;
2380 } else {
2381 *flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_SPLAT;
2382 }
2383 }
2384}
2385
2389static pm_arguments_node_t *
2390pm_arguments_node_single_create(pm_parser_t *parser, pm_node_t *argument) {
2391 pm_node_flags_t flags = 0;
2392 pm_arguments_flags_update(&flags, argument);
2393
2394 return pm_arguments_node_new(
2395 parser->arena,
2396 ++parser->node_id,
2397 flags,
2398 PM_LOCATION_INIT_NODE(argument),
2399 pm_node_list_single(parser->arena, argument)
2400 );
2401}
2402
2407static pm_arguments_node_t *
2408pm_arguments_node_extend(pm_parser_t *parser, pm_arguments_node_t *node, pm_node_t *argument) {
2409 if (node == NULL) return pm_arguments_node_single_create(parser, argument);
2410
2411 if (PM_NODE_END(node) < PM_NODE_END(argument)) {
2412 PM_NODE_LENGTH_SET_NODE(node, argument);
2413 }
2414
2415 pm_arguments_flags_update(&node->base.flags, argument);
2416 pm_node_list_append(parser->arena, &node->arguments, argument);
2417 return node;
2418}
2419
2425typedef struct {
2427 pm_node_list_t nodes;
2428
2433 bool dynamic;
2434
2436 bool contains_splat;
2437} pm_array_elements_t;
2438
2443static PRISM_INLINE void
2444pm_array_elements_append(pm_parser_t *parser, pm_array_elements_t *elements, pm_node_t *element) {
2445 if (PM_NODE_TYPE_P(element, PM_ARRAY_NODE) || PM_NODE_TYPE_P(element, PM_HASH_NODE) || PM_NODE_TYPE_P(element, PM_RANGE_NODE) || !PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL)) {
2446 elements->dynamic = true;
2447 }
2448
2449 if (PM_NODE_TYPE_P(element, PM_SPLAT_NODE)) {
2450 elements->contains_splat = true;
2451 }
2452
2453 pm_node_list_open_append(&parser->node_chunks, &elements->nodes, element);
2454}
2455
2459static pm_array_node_t *
2460pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_array_elements_t *elements, const pm_token_t *closing) {
2461 PRISM_ASSERT(closing == NULL || closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == 0);
2462
2463 pm_node_list_t *nodes = &elements->nodes;
2464 pm_node_list_finalize(&parser->node_chunks, parser->arena, nodes);
2465
2466 pm_node_flags_t flags = (pm_node_flags_t) ((elements->dynamic ? 0 : PM_NODE_FLAG_STATIC_LITERAL) | (elements->contains_splat ? PM_ARRAY_NODE_FLAGS_CONTAINS_SPLAT : 0));
2467
2468 uint32_t start = 0;
2469 uint32_t end = 0;
2470
2471 if (opening != NULL) {
2472 start = PM_TOKEN_START(parser, opening);
2473 end = PM_TOKEN_END(parser, opening);
2474 } else if (nodes->size > 0) {
2475 start = PM_NODE_START(nodes->nodes[0]);
2476 }
2477
2478 if (nodes->size > 0) end = PM_NODE_END(nodes->nodes[nodes->size - 1]);
2479 if (closing != NULL) end = PM_TOKEN_END(parser, closing);
2480
2481 return pm_array_node_new(
2482 parser->arena,
2483 ++parser->node_id,
2484 flags,
2485 PM_LOCATION_INIT_BOUNDS(start, end),
2486 *nodes,
2487 NTOK2LOC(parser, opening),
2488 NTOK2LOC(parser, closing)
2489 );
2490}
2491
2497static pm_array_pattern_node_t *
2498pm_array_pattern_node_node_list_create(pm_parser_t *parser, const pm_node_list_t *nodes) {
2499 uint32_t rest_index = nodes->size;
2500 for (uint32_t idx = 0; idx < nodes->size; ++idx) {
2501 if (PM_NODE_TYPE_P(nodes->nodes[idx], PM_SPLAT_NODE) || PM_NODE_TYPE_P(nodes->nodes[idx], PM_IMPLICIT_REST_NODE)) {
2502 rest_index = idx;
2503 break;
2504 }
2505 }
2506
2507 pm_node_t *rest = NULL;
2508 pm_node_list_t posts = { 0 };
2509
2510 if (rest_index < nodes->size) {
2511 rest = nodes->nodes[rest_index];
2512 posts = pm_node_list_slice(nodes, rest_index + 1, nodes->size - rest_index - 1);
2513 }
2514
2515 return pm_array_pattern_node_new(
2516 parser->arena,
2517 ++parser->node_id,
2518 0,
2519 PM_LOCATION_INIT_NODES(nodes->nodes[0], nodes->nodes[nodes->size - 1]),
2520 NULL,
2521 pm_node_list_slice(nodes, 0, rest_index),
2522 rest,
2523 posts,
2524 ((pm_location_t) { 0 }),
2525 ((pm_location_t) { 0 })
2526 );
2527}
2528
2532static pm_array_pattern_node_t *
2533pm_array_pattern_node_rest_create(pm_parser_t *parser, pm_node_t *rest) {
2534 return pm_array_pattern_node_new(
2535 parser->arena,
2536 ++parser->node_id,
2537 0,
2538 PM_LOCATION_INIT_NODE(rest),
2539 NULL,
2540 ((pm_node_list_t) { 0 }),
2541 rest,
2542 ((pm_node_list_t) { 0 }),
2543 ((pm_location_t) { 0 }),
2544 ((pm_location_t) { 0 })
2545 );
2546}
2547
2552static pm_array_pattern_node_t *
2553pm_array_pattern_node_constant_create(pm_parser_t *parser, pm_node_t *constant, const pm_token_t *opening, pm_node_t *inner, const pm_token_t *closing) {
2554 return pm_array_pattern_node_new(
2555 parser->arena,
2556 ++parser->node_id,
2557 0,
2558 PM_LOCATION_INIT_NODE_TOKEN(parser, constant, closing),
2559 constant,
2560 inner == NULL ? ((pm_node_list_t) { 0 }) : pm_node_list_single(parser->arena, inner),
2561 NULL,
2562 ((pm_node_list_t) { 0 }),
2563 TOK2LOC(parser, opening),
2564 TOK2LOC(parser, closing)
2565 );
2566}
2567
2572static pm_array_pattern_node_t *
2573pm_array_pattern_node_bracket_create(pm_parser_t *parser, const pm_token_t *opening, pm_node_t *inner, const pm_token_t *closing) {
2574 return pm_array_pattern_node_new(
2575 parser->arena,
2576 ++parser->node_id,
2577 0,
2578 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
2579 NULL,
2580 inner == NULL ? ((pm_node_list_t) { 0 }) : pm_node_list_single(parser->arena, inner),
2581 NULL,
2582 ((pm_node_list_t) { 0 }),
2583 TOK2LOC(parser, opening),
2584 TOK2LOC(parser, closing)
2585 );
2586}
2587
2591static pm_assoc_node_t *
2592pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *operator, pm_node_t *value) {
2593 uint32_t end;
2594
2595 if (value != NULL && PM_NODE_END(value) > PM_NODE_END(key)) {
2596 end = PM_NODE_END(value);
2597 } else if (operator != NULL) {
2598 end = PM_TOKEN_END(parser, operator);
2599 } else {
2600 end = PM_NODE_END(key);
2601 }
2602
2603 // Hash string keys will be frozen, so we can mark them as frozen here so
2604 // that the compiler picks them up and also when we check for static literal
2605 // on the keys it gets factored in.
2606 if (PM_NODE_TYPE_P(key, PM_STRING_NODE)) {
2607 key->flags |= PM_STRING_FLAGS_FROZEN | PM_NODE_FLAG_STATIC_LITERAL;
2608 }
2609
2610 // If the key and value of this assoc node are both static literals, then
2611 // we can mark this node as a static literal.
2612 pm_node_flags_t flags = 0;
2613 if (
2614 !PM_NODE_TYPE_P(key, PM_ARRAY_NODE) && !PM_NODE_TYPE_P(key, PM_HASH_NODE) && !PM_NODE_TYPE_P(key, PM_RANGE_NODE) &&
2615 value && !PM_NODE_TYPE_P(value, PM_ARRAY_NODE) && !PM_NODE_TYPE_P(value, PM_HASH_NODE) && !PM_NODE_TYPE_P(value, PM_RANGE_NODE)
2616 ) {
2617 flags = key->flags & value->flags & PM_NODE_FLAG_STATIC_LITERAL;
2618 }
2619
2620 return pm_assoc_node_new(
2621 parser->arena,
2622 ++parser->node_id,
2623 flags,
2624 ((pm_location_t) { .start = PM_NODE_START(key), .length = U32(end - PM_NODE_START(key)) }),
2625 key,
2626 value,
2627 NTOK2LOC(parser, operator)
2628 );
2629}
2630
2634static pm_assoc_splat_node_t *
2635pm_assoc_splat_node_create(pm_parser_t *parser, pm_node_t *value, const pm_token_t *operator) {
2636 PRISM_ASSERT(operator->type == PM_TOKEN_USTAR_STAR);
2637
2638 return pm_assoc_splat_node_new(
2639 parser->arena,
2640 ++parser->node_id,
2641 0,
2642 (value == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKEN_NODE(parser, operator, value),
2643 value,
2644 TOK2LOC(parser, operator)
2645 );
2646}
2647
2651static pm_back_reference_read_node_t *
2652pm_back_reference_read_node_create(pm_parser_t *parser, const pm_token_t *name) {
2653 PRISM_ASSERT(name->type == PM_TOKEN_BACK_REFERENCE);
2654
2655 return pm_back_reference_read_node_new(
2656 parser->arena,
2657 ++parser->node_id,
2658 0,
2659 PM_LOCATION_INIT_TOKEN(parser, name),
2660 pm_parser_constant_id_token(parser, name)
2661 );
2662}
2663
2667static pm_begin_node_t *
2668pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_statements_node_t *statements) {
2669 uint32_t start = begin_keyword == NULL ? 0 : PM_TOKEN_START(parser, begin_keyword);
2670 uint32_t end = statements == NULL ? (begin_keyword == NULL ? 0 : PM_TOKEN_END(parser, begin_keyword)) : PM_NODE_END(statements);
2671
2672 return pm_begin_node_new(
2673 parser->arena,
2674 ++parser->node_id,
2675 0,
2676 ((pm_location_t) { .start = start, .length = U32(end - start) }),
2677 NTOK2LOC(parser, begin_keyword),
2678 statements,
2679 NULL,
2680 NULL,
2681 NULL,
2682 ((pm_location_t) { 0 })
2683 );
2684}
2685
2689static void
2690pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_clause) {
2691 if (node->begin_keyword_loc.length == 0) {
2692 PM_NODE_START_SET_NODE(node, rescue_clause);
2693 }
2694 PM_NODE_LENGTH_SET_NODE(node, rescue_clause);
2695 node->rescue_clause = rescue_clause;
2696}
2697
2701static void
2702pm_begin_node_else_clause_set(pm_begin_node_t *node, pm_else_node_t *else_clause) {
2703 if ((node->begin_keyword_loc.length == 0) && PM_NODE_START(node) == 0) {
2704 PM_NODE_START_SET_NODE(node, else_clause);
2705 }
2706 PM_NODE_LENGTH_SET_NODE(node, else_clause);
2707 node->else_clause = else_clause;
2708}
2709
2713static void
2714pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_clause) {
2715 if ((node->begin_keyword_loc.length == 0) && PM_NODE_START(node) == 0) {
2716 PM_NODE_START_SET_NODE(node, ensure_clause);
2717 }
2718 PM_NODE_LENGTH_SET_NODE(node, ensure_clause);
2719 node->ensure_clause = ensure_clause;
2720}
2721
2725static void
2726pm_begin_node_end_keyword_set(const pm_parser_t *parser, pm_begin_node_t *node, const pm_token_t *end_keyword) {
2727 PRISM_ASSERT(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == 0);
2728 PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword);
2729 node->end_keyword_loc = TOK2LOC(parser, end_keyword);
2730}
2731
2735static pm_block_argument_node_t *
2736pm_block_argument_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t *expression) {
2737 PRISM_ASSERT(operator->type == PM_TOKEN_UAMPERSAND);
2738
2739 return pm_block_argument_node_new(
2740 parser->arena,
2741 ++parser->node_id,
2742 0,
2743 (expression == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKEN_NODE(parser, operator, expression),
2744 expression,
2745 TOK2LOC(parser, operator)
2746 );
2747}
2748
2752static pm_block_node_t *
2753pm_block_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const pm_token_t *opening, pm_node_t *parameters, pm_node_t *body, const pm_token_t *closing) {
2754 return pm_block_node_new(
2755 parser->arena,
2756 ++parser->node_id,
2757 0,
2758 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
2759 *locals,
2760 parameters,
2761 body,
2762 TOK2LOC(parser, opening),
2763 TOK2LOC(parser, closing)
2764 );
2765}
2766
2770static pm_block_parameter_node_t *
2771pm_block_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, const pm_token_t *operator) {
2772 PRISM_ASSERT(operator->type == PM_TOKEN_UAMPERSAND || operator->type == PM_TOKEN_AMPERSAND);
2773
2774 return pm_block_parameter_node_new(
2775 parser->arena,
2776 ++parser->node_id,
2777 0,
2778 (name == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKENS(parser, operator, name),
2779 name == NULL ? 0 : pm_parser_constant_id_token(parser, name),
2780 NTOK2LOC(parser, name),
2781 TOK2LOC(parser, operator)
2782 );
2783}
2784
2788static pm_block_parameters_node_t *
2789pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *parameters, const pm_token_t *opening, pm_node_list_t *locals) {
2790 pm_node_list_finalize(&parser->node_chunks, parser->arena, locals);
2791
2792 uint32_t start;
2793 if (opening != NULL) {
2794 start = PM_TOKEN_START(parser, opening);
2795 } else if (parameters != NULL) {
2796 start = PM_NODE_START(parameters);
2797 } else {
2798 start = 0;
2799 }
2800
2801 uint32_t end;
2802 if (parameters != NULL) {
2803 end = PM_NODE_END(parameters);
2804 } else if (opening != NULL) {
2805 end = PM_TOKEN_END(parser, opening);
2806 } else {
2807 end = 0;
2808 }
2809
2810 /* Locals only follow an opening token, so they always extend the end of
2811 * a node that has already started. */
2812 if (locals->size > 0) {
2813 PRISM_ASSERT(opening != NULL);
2814 end = PM_NODE_END(locals->nodes[locals->size - 1]);
2815 }
2816
2817 return pm_block_parameters_node_new(
2818 parser->arena,
2819 ++parser->node_id,
2820 0,
2821 ((pm_location_t) { .start = start, .length = U32(end - start) }),
2822 parameters,
2823 *locals,
2824 NTOK2LOC(parser, opening),
2825 ((pm_location_t) { 0 })
2826 );
2827}
2828
2832static void
2833pm_block_parameters_node_closing_set(const pm_parser_t *parser, pm_block_parameters_node_t *node, const pm_token_t *closing) {
2834 PRISM_ASSERT(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == 0);
2835 PM_NODE_LENGTH_SET_TOKEN(parser, node, closing);
2836 node->closing_loc = TOK2LOC(parser, closing);
2837}
2838
2842static pm_block_local_variable_node_t *
2843pm_block_local_variable_node_create(pm_parser_t *parser, const pm_token_t *name) {
2844 return pm_block_local_variable_node_new(
2845 parser->arena,
2846 ++parser->node_id,
2847 0,
2848 PM_LOCATION_INIT_TOKEN(parser, name),
2849 pm_parser_constant_id_token(parser, name)
2850 );
2851}
2852
2856static pm_break_node_t *
2857pm_break_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments_node_t *arguments) {
2858 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_BREAK);
2859
2860 return pm_break_node_new(
2861 parser->arena,
2862 ++parser->node_id,
2863 0,
2864 (arguments == NULL) ? PM_LOCATION_INIT_TOKEN(parser, keyword) : PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, arguments),
2865 arguments,
2866 TOK2LOC(parser, keyword)
2867 );
2868}
2869
2870// There are certain flags that we want to use internally but don't want to
2871// expose because they are not relevant beyond parsing. Therefore we'll define
2872// them here and not define them in config.yml/a header file.
2873static const pm_node_flags_t PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY = (1 << 2);
2874
2875static const pm_node_flags_t PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY = ((PM_CALL_NODE_FLAGS_LAST - 1) << 1);
2876static const pm_node_flags_t PM_CALL_NODE_FLAGS_COMPARISON = ((PM_CALL_NODE_FLAGS_LAST - 1) << 2);
2877static const pm_node_flags_t PM_CALL_NODE_FLAGS_INDEX = ((PM_CALL_NODE_FLAGS_LAST - 1) << 3);
2878
2884static pm_call_node_t *
2885pm_call_node_create(pm_parser_t *parser, pm_node_flags_t flags) {
2886 return pm_call_node_new(
2887 parser->arena,
2888 ++parser->node_id,
2889 flags,
2890 PM_LOCATION_INIT_UNSET,
2891 NULL,
2892 ((pm_location_t) { 0 }),
2893 0,
2894 ((pm_location_t) { 0 }),
2895 ((pm_location_t) { 0 }),
2896 NULL,
2897 ((pm_location_t) { 0 }),
2898 ((pm_location_t) { 0 }),
2899 NULL
2900 );
2901}
2902
2907static PRISM_INLINE pm_node_flags_t
2908pm_call_node_ignore_visibility_flag(const pm_node_t *receiver) {
2909 return PM_NODE_TYPE_P(receiver, PM_SELF_NODE) ? PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY : 0;
2910}
2911
2916static pm_call_node_t *
2917pm_call_node_aref_create(pm_parser_t *parser, pm_node_t *receiver, pm_arguments_t *arguments) {
2918 pm_assert_value_expression(parser, receiver);
2919
2920 pm_node_flags_t flags = pm_call_node_ignore_visibility_flag(receiver);
2921 if (arguments->block == NULL || PM_NODE_TYPE_P(arguments->block, PM_BLOCK_ARGUMENT_NODE)) {
2922 flags |= PM_CALL_NODE_FLAGS_INDEX;
2923 }
2924
2925 pm_call_node_t *node = pm_call_node_create(parser, flags);
2926
2927 PM_NODE_START_SET_NODE(node, receiver);
2928
2929 const pm_location_t *end = pm_arguments_end(arguments);
2930 PRISM_ASSERT(end != NULL && "unreachable");
2931 PM_NODE_LENGTH_SET_LOCATION(node, end);
2932
2933 node->receiver = receiver;
2934 node->message_loc.start = arguments->opening_loc.start;
2935 node->message_loc.length = (arguments->closing_loc.start + arguments->closing_loc.length) - arguments->opening_loc.start;
2936
2937 node->opening_loc = arguments->opening_loc;
2938 node->arguments = pm_arguments_node_create(parser, arguments);
2939 node->closing_loc = arguments->closing_loc;
2940 node->block = arguments->block;
2941
2942 node->name = pm_parser_constant_id_constant(parser, "[]", 2);
2943 return node;
2944}
2945
2949static pm_call_node_t *
2950pm_call_node_binary_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *operator, pm_node_t *argument, pm_node_flags_t flags) {
2951 pm_assert_value_expression(parser, receiver);
2952 pm_assert_value_expression(parser, argument);
2953
2954 pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver) | flags);
2955
2956 PM_NODE_START_SET_NODE(node, PM_NODE_START(receiver) < PM_NODE_START(argument) ? receiver : argument);
2957 PM_NODE_LENGTH_SET_NODE(node, PM_NODE_END(receiver) > PM_NODE_END(argument) ? receiver : argument);
2958
2959 node->receiver = receiver;
2960 node->message_loc = TOK2LOC(parser, operator);
2961
2962 node->arguments = pm_arguments_node_single_create(parser, argument);
2963
2964 node->name = pm_parser_constant_id_token(parser, operator);
2965 return node;
2966}
2967
2968static const uint8_t * parse_operator_symbol_name(const pm_token_t *);
2969
2973static pm_call_node_t *
2974pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *operator, pm_token_t *message, pm_arguments_t *arguments) {
2975 pm_assert_value_expression(parser, receiver);
2976
2977 pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver));
2978
2979 PM_NODE_START_SET_NODE(node, receiver);
2980 const pm_location_t *end = pm_arguments_end(arguments);
2981 if (end == NULL) {
2982 PM_NODE_LENGTH_SET_TOKEN(parser, node, message);
2983 } else {
2984 PM_NODE_LENGTH_SET_LOCATION(node, end);
2985 }
2986
2987 node->receiver = receiver;
2988 node->call_operator_loc = TOK2LOC(parser, operator);
2989 node->message_loc = TOK2LOC(parser, message);
2990 node->opening_loc = arguments->opening_loc;
2991 node->arguments = pm_arguments_node_create(parser, arguments);
2992 node->closing_loc = arguments->closing_loc;
2993 node->block = arguments->block;
2994
2995 if (operator->type == PM_TOKEN_AMPERSAND_DOT) {
2996 pm_node_flag_set(UP(node), PM_CALL_NODE_FLAGS_SAFE_NAVIGATION);
2997 }
2998
3003 node->name = pm_parser_constant_id_raw(parser, message->start, parse_operator_symbol_name(message));
3004 return node;
3005}
3006
3010static pm_call_node_t *
3011pm_call_node_call_synthesized_create(pm_parser_t *parser, pm_node_t *receiver, const char *message, pm_arguments_node_t *arguments, pm_location_t location) {
3012 pm_call_node_t *node = pm_call_node_create(parser, 0);
3013 node->base.location = location;
3014
3015 node->receiver = receiver;
3016 node->arguments = arguments;
3017
3018 node->name = pm_parser_constant_id_constant(parser, message, strlen(message));
3019 return node;
3020}
3021
3026static pm_call_node_t *
3027pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments_t *arguments) {
3028 pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY);
3029
3030 PM_NODE_START_SET_TOKEN(parser, node, message);
3031 const pm_location_t *end = pm_arguments_end(arguments);
3032 PRISM_ASSERT(end != NULL && "unreachable");
3033 PM_NODE_LENGTH_SET_LOCATION(node, end);
3034
3035 node->message_loc = TOK2LOC(parser, message);
3036 node->opening_loc = arguments->opening_loc;
3037 node->arguments = pm_arguments_node_create(parser, arguments);
3038 node->closing_loc = arguments->closing_loc;
3039 node->block = arguments->block;
3040
3041 node->name = pm_parser_constant_id_token(parser, message);
3042 return node;
3043}
3044
3049static pm_call_node_t *
3050pm_call_node_fcall_synthesized_create(pm_parser_t *parser, pm_arguments_node_t *arguments, pm_constant_id_t name) {
3051 pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY);
3052
3053 node->base.location = (pm_location_t) { 0 };
3054 node->arguments = arguments;
3055
3056 node->name = name;
3057 return node;
3058}
3059
3063static pm_call_node_t *
3064pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *message, pm_arguments_t *arguments) {
3065 pm_assert_value_expression(parser, receiver);
3066 if (receiver != NULL) pm_conditional_predicate(parser, receiver, PM_CONDITIONAL_PREDICATE_TYPE_NOT);
3067
3068 pm_call_node_t *node = pm_call_node_create(parser, receiver == NULL ? 0 : pm_call_node_ignore_visibility_flag(receiver));
3069
3070 PM_NODE_START_SET_TOKEN(parser, node, message);
3071 if (arguments->closing_loc.length > 0) {
3072 PM_NODE_LENGTH_SET_LOCATION(node, &arguments->closing_loc);
3073 } else {
3074 PRISM_ASSERT(receiver != NULL);
3075 PM_NODE_LENGTH_SET_NODE(node, receiver);
3076 }
3077
3078 node->receiver = receiver;
3079 node->message_loc = TOK2LOC(parser, message);
3080 node->opening_loc = arguments->opening_loc;
3081 node->arguments = pm_arguments_node_create(parser, arguments);
3082 node->closing_loc = arguments->closing_loc;
3083
3084 node->name = pm_parser_constant_id_constant(parser, "!", 1);
3085 return node;
3086}
3087
3091static pm_call_node_t *
3092pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *operator, pm_arguments_t *arguments) {
3093 pm_assert_value_expression(parser, receiver);
3094
3095 pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver));
3096
3097 PM_NODE_START_SET_NODE(node, receiver);
3098 const pm_location_t *end = pm_arguments_end(arguments);
3099 PRISM_ASSERT(end != NULL && "unreachable");
3100 PM_NODE_LENGTH_SET_LOCATION(node, end);
3101
3102 node->receiver = receiver;
3103 node->call_operator_loc = TOK2LOC(parser, operator);
3104 node->opening_loc = arguments->opening_loc;
3105 node->arguments = pm_arguments_node_create(parser, arguments);
3106 node->closing_loc = arguments->closing_loc;
3107 node->block = arguments->block;
3108
3109 if (operator->type == PM_TOKEN_AMPERSAND_DOT) {
3110 pm_node_flag_set(UP(node), PM_CALL_NODE_FLAGS_SAFE_NAVIGATION);
3111 }
3112
3113 node->name = pm_parser_constant_id_constant(parser, "call", 4);
3114 return node;
3115}
3116
3120static pm_call_node_t *
3121pm_call_node_unary_create(pm_parser_t *parser, pm_token_t *operator, pm_node_t *receiver, const char *name) {
3122 pm_assert_value_expression(parser, receiver);
3123
3124 pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver));
3125
3126 PM_NODE_START_SET_TOKEN(parser, node, operator);
3127 PM_NODE_LENGTH_SET_NODE(node, receiver);
3128
3129 node->receiver = receiver;
3130 node->message_loc = TOK2LOC(parser, operator);
3131
3132 node->name = pm_parser_constant_id_constant(parser, name, strlen(name));
3133 return node;
3134}
3135
3140static pm_call_node_t *
3141pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) {
3142 pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY);
3143
3144 node->base.location = TOK2LOC(parser, message);
3145 node->message_loc = TOK2LOC(parser, message);
3146
3147 node->name = pm_parser_constant_id_token(parser, message);
3148 return node;
3149}
3150
3155static PRISM_INLINE bool
3156pm_call_node_writable_p(const pm_parser_t *parser, const pm_call_node_t *node) {
3157 return (
3158 (node->message_loc.length > 0) &&
3159 (parser->start[node->message_loc.start + node->message_loc.length - 1] != '!') &&
3160 (parser->start[node->message_loc.start + node->message_loc.length - 1] != '?') &&
3161 (node->receiver == NULL || node->call_operator_loc.length > 0) &&
3162 char_is_identifier_start(parser, parser->start + node->message_loc.start, (ptrdiff_t) node->message_loc.length) &&
3163 (node->opening_loc.length == 0) &&
3164 (node->arguments == NULL) &&
3165 (node->block == NULL)
3166 );
3167}
3168
3172static void
3173pm_call_write_read_name_init(pm_parser_t *parser, pm_constant_id_t *read_name, pm_constant_id_t *write_name) {
3174 pm_constant_t *write_constant = pm_constant_pool_id_to_constant(&parser->constant_pool, *write_name);
3175
3176 if (write_constant->length > 0) {
3177 size_t length = write_constant->length - 1;
3178
3179 uint8_t *memory = PM_ARENA_ALLOC_ARRAY(parser->arena, uint8_t, length);
3180 memcpy(memory, write_constant->start, length);
3181
3182 *read_name = pm_constant_pool_insert_owned(&parser->metadata_arena, &parser->constant_pool, memory, length);
3183 } else {
3184 // We can get here if the message was missing because of a syntax error.
3185 *read_name = pm_parser_constant_id_constant(parser, "", 0);
3186 }
3187}
3188
3192static pm_call_and_write_node_t *
3193pm_call_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3194 PRISM_ASSERT(target->block == NULL);
3195 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
3196
3197 pm_call_and_write_node_t *node = pm_call_and_write_node_new(
3198 parser->arena,
3199 ++parser->node_id,
3200 FL(target),
3201 PM_LOCATION_INIT_NODES(target, value),
3202 target->receiver,
3203 target->call_operator_loc,
3204 target->message_loc,
3205 0,
3206 target->name,
3207 TOK2LOC(parser, operator),
3208 value
3209 );
3210
3211 pm_call_write_read_name_init(parser, &node->read_name, &node->write_name);
3212
3213 // The target is no longer necessary because we've reused its children.
3214 // It is arena-allocated so no explicit free is needed.
3215
3216 return node;
3217}
3218
3223static void
3224pm_index_arguments_check(pm_parser_t *parser, const pm_arguments_node_t *arguments, const pm_node_t *block) {
3225 if (parser->version >= PM_OPTIONS_VERSION_CRUBY_3_4) {
3226 if (arguments != NULL && PM_NODE_FLAG_P(arguments, PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS)) {
3227 pm_node_t *node;
3228 PM_NODE_LIST_FOREACH(&arguments->arguments, index, node) {
3229 if (PM_NODE_TYPE_P(node, PM_KEYWORD_HASH_NODE)) {
3230 pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_INDEX_KEYWORDS);
3231 break;
3232 }
3233 }
3234 }
3235
3236 if (block != NULL) {
3237 pm_parser_err_node(parser, block, PM_ERR_UNEXPECTED_INDEX_BLOCK);
3238 }
3239 }
3240}
3241
3245static pm_index_and_write_node_t *
3246pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3247 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
3248
3249 pm_index_arguments_check(parser, target->arguments, target->block);
3250
3251 PRISM_ASSERT(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE));
3252
3253 pm_index_and_write_node_t *node = pm_index_and_write_node_new(
3254 parser->arena,
3255 ++parser->node_id,
3256 FL(target),
3257 PM_LOCATION_INIT_NODES(target, value),
3258 target->receiver,
3259 target->call_operator_loc,
3260 target->opening_loc,
3261 target->arguments,
3262 target->closing_loc,
3263 (pm_block_argument_node_t *) target->block,
3264 TOK2LOC(parser, operator),
3265 value
3266 );
3267
3268 // The target is no longer necessary because we've reused its children.
3269 // It is arena-allocated so no explicit free is needed.
3270
3271 return node;
3272}
3273
3277static pm_call_operator_write_node_t *
3278pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3279 PRISM_ASSERT(target->block == NULL);
3280
3281 pm_call_operator_write_node_t *node = pm_call_operator_write_node_new(
3282 parser->arena,
3283 ++parser->node_id,
3284 FL(target),
3285 PM_LOCATION_INIT_NODES(target, value),
3286 target->receiver,
3287 target->call_operator_loc,
3288 target->message_loc,
3289 0,
3290 target->name,
3291 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1),
3292 TOK2LOC(parser, operator),
3293 value
3294 );
3295
3296 pm_call_write_read_name_init(parser, &node->read_name, &node->write_name);
3297
3298 // The target is no longer necessary because we've reused its children.
3299 // It is arena-allocated so no explicit free is needed.
3300
3301 return node;
3302}
3303
3307static pm_index_operator_write_node_t *
3308pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3309 pm_index_arguments_check(parser, target->arguments, target->block);
3310
3311 PRISM_ASSERT(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE));
3312
3313 pm_index_operator_write_node_t *node = pm_index_operator_write_node_new(
3314 parser->arena,
3315 ++parser->node_id,
3316 FL(target),
3317 PM_LOCATION_INIT_NODES(target, value),
3318 target->receiver,
3319 target->call_operator_loc,
3320 target->opening_loc,
3321 target->arguments,
3322 target->closing_loc,
3323 (pm_block_argument_node_t *) target->block,
3324 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1),
3325 TOK2LOC(parser, operator),
3326 value
3327 );
3328
3329 // The target is no longer necessary because we've reused its children.
3330 // It is arena-allocated so no explicit free is needed.
3331
3332 return node;
3333}
3334
3338static pm_call_or_write_node_t *
3339pm_call_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3340 PRISM_ASSERT(target->block == NULL);
3341 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
3342
3343 pm_call_or_write_node_t *node = pm_call_or_write_node_new(
3344 parser->arena,
3345 ++parser->node_id,
3346 FL(target),
3347 PM_LOCATION_INIT_NODES(target, value),
3348 target->receiver,
3349 target->call_operator_loc,
3350 target->message_loc,
3351 0,
3352 target->name,
3353 TOK2LOC(parser, operator),
3354 value
3355 );
3356
3357 pm_call_write_read_name_init(parser, &node->read_name, &node->write_name);
3358
3359 // The target is no longer necessary because we've reused its children.
3360 // It is arena-allocated so no explicit free is needed.
3361
3362 return node;
3363}
3364
3368static pm_index_or_write_node_t *
3369pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3370 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
3371
3372 pm_index_arguments_check(parser, target->arguments, target->block);
3373
3374 PRISM_ASSERT(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE));
3375
3376 pm_index_or_write_node_t *node = pm_index_or_write_node_new(
3377 parser->arena,
3378 ++parser->node_id,
3379 FL(target),
3380 PM_LOCATION_INIT_NODES(target, value),
3381 target->receiver,
3382 target->call_operator_loc,
3383 target->opening_loc,
3384 target->arguments,
3385 target->closing_loc,
3386 (pm_block_argument_node_t *) target->block,
3387 TOK2LOC(parser, operator),
3388 value
3389 );
3390
3391 // The target is no longer necessary because we've reused its children.
3392 // It is arena-allocated so no explicit free is needed.
3393
3394 return node;
3395}
3396
3401static pm_call_target_node_t *
3402pm_call_target_node_create(pm_parser_t *parser, pm_call_node_t *target) {
3403 pm_call_target_node_t *node = pm_call_target_node_new(
3404 parser->arena,
3405 ++parser->node_id,
3406 FL(target),
3407 PM_LOCATION_INIT_NODE(target),
3408 target->receiver,
3409 target->call_operator_loc,
3410 target->name,
3411 target->message_loc
3412 );
3413
3414 /* It is possible to get here where we have parsed an invalid syntax tree
3415 * where the call operator was not present. In that case we will have a
3416 * problem because it is a required location. In this case we need to fill
3417 * it in with a fake location so that the syntax tree remains valid. */
3418 if (node->call_operator_loc.length == 0) {
3419 node->call_operator_loc = target->base.location;
3420 }
3421
3422 // The target is no longer necessary because we've reused its children.
3423 // It is arena-allocated so no explicit free is needed.
3424
3425 return node;
3426}
3427
3432static pm_index_target_node_t *
3433pm_index_target_node_create(pm_parser_t *parser, pm_call_node_t *target) {
3434 pm_index_arguments_check(parser, target->arguments, target->block);
3435 PRISM_ASSERT(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE));
3436
3437 pm_index_target_node_t *node = pm_index_target_node_new(
3438 parser->arena,
3439 ++parser->node_id,
3440 FL(target) | PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE,
3441 PM_LOCATION_INIT_NODE(target),
3442 target->receiver,
3443 target->opening_loc,
3444 target->arguments,
3445 target->closing_loc,
3446 (pm_block_argument_node_t *) target->block
3447 );
3448
3449 // The target is no longer necessary because we've reused its children.
3450 // It is arena-allocated so no explicit free is needed.
3451
3452 return node;
3453}
3454
3458static pm_capture_pattern_node_t *
3459pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_local_variable_target_node_t *target, const pm_token_t *operator) {
3460 return pm_capture_pattern_node_new(
3461 parser->arena,
3462 ++parser->node_id,
3463 0,
3464 PM_LOCATION_INIT_NODES(value, target),
3465 value,
3466 target,
3467 TOK2LOC(parser, operator)
3468 );
3469}
3470
3474static pm_case_node_t *
3475pm_case_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node_t *predicate, pm_node_list_t *conditions, pm_else_node_t *else_clause, const pm_token_t *end_keyword) {
3476 pm_node_list_finalize(&parser->node_chunks, parser->arena, conditions);
3477
3478 return pm_case_node_new(
3479 parser->arena,
3480 ++parser->node_id,
3481 0,
3482 PM_LOCATION_INIT_TOKENS(parser, case_keyword, end_keyword),
3483 predicate,
3484 *conditions,
3485 else_clause,
3486 TOK2LOC(parser, case_keyword),
3487 TOK2LOC(parser, end_keyword)
3488 );
3489}
3490
3494static pm_case_match_node_t *
3495pm_case_match_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node_t *predicate, pm_node_list_t *conditions, pm_else_node_t *else_clause, const pm_token_t *end_keyword) {
3496 pm_node_list_finalize(&parser->node_chunks, parser->arena, conditions);
3497
3498 return pm_case_match_node_new(
3499 parser->arena,
3500 ++parser->node_id,
3501 0,
3502 PM_LOCATION_INIT_TOKENS(parser, case_keyword, end_keyword),
3503 predicate,
3504 *conditions,
3505 else_clause,
3506 TOK2LOC(parser, case_keyword),
3507 TOK2LOC(parser, end_keyword)
3508 );
3509}
3510
3514static pm_class_node_t *
3515pm_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const pm_token_t *class_keyword, pm_node_t *constant_path, const pm_token_t *name, const pm_token_t *inheritance_operator, pm_node_t *superclass, pm_node_t *body, const pm_token_t *end_keyword) {
3516 return pm_class_node_new(
3517 parser->arena,
3518 ++parser->node_id,
3519 0,
3520 PM_LOCATION_INIT_TOKENS(parser, class_keyword, end_keyword),
3521 *locals,
3522 TOK2LOC(parser, class_keyword),
3523 constant_path,
3524 NTOK2LOC(parser, inheritance_operator),
3525 superclass,
3526 body,
3527 TOK2LOC(parser, end_keyword),
3528 pm_parser_constant_id_token(parser, name)
3529 );
3530}
3531
3535static pm_class_variable_and_write_node_t *
3536pm_class_variable_and_write_node_create(pm_parser_t *parser, pm_class_variable_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3537 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
3538
3539 return pm_class_variable_and_write_node_new(
3540 parser->arena,
3541 ++parser->node_id,
3542 0,
3543 PM_LOCATION_INIT_NODES(target, value),
3544 target->name,
3545 target->base.location,
3546 TOK2LOC(parser, operator),
3547 value
3548 );
3549}
3550
3554static pm_class_variable_operator_write_node_t *
3555pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_variable_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3556 return pm_class_variable_operator_write_node_new(
3557 parser->arena,
3558 ++parser->node_id,
3559 0,
3560 PM_LOCATION_INIT_NODES(target, value),
3561 target->name,
3562 target->base.location,
3563 TOK2LOC(parser, operator),
3564 value,
3565 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1)
3566 );
3567}
3568
3572static pm_class_variable_or_write_node_t *
3573pm_class_variable_or_write_node_create(pm_parser_t *parser, pm_class_variable_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3574 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
3575
3576 return pm_class_variable_or_write_node_new(
3577 parser->arena,
3578 ++parser->node_id,
3579 0,
3580 PM_LOCATION_INIT_NODES(target, value),
3581 target->name,
3582 target->base.location,
3583 TOK2LOC(parser, operator),
3584 value
3585 );
3586}
3587
3591static pm_class_variable_read_node_t *
3592pm_class_variable_read_node_create(pm_parser_t *parser, const pm_token_t *token) {
3593 PRISM_ASSERT(token->type == PM_TOKEN_CLASS_VARIABLE);
3594
3595 return pm_class_variable_read_node_new(
3596 parser->arena,
3597 ++parser->node_id,
3598 0,
3599 PM_LOCATION_INIT_TOKEN(parser, token),
3600 pm_parser_constant_id_token(parser, token)
3601 );
3602}
3603
3610static PRISM_INLINE pm_node_flags_t
3611pm_implicit_array_write_flags(const pm_node_t *node, pm_node_flags_t flags) {
3612 if (PM_NODE_TYPE_P(node, PM_ARRAY_NODE) && ((const pm_array_node_t *) node)->opening_loc.length == 0) {
3613 return flags;
3614 }
3615 return 0;
3616}
3617
3621static pm_class_variable_write_node_t *
3622pm_class_variable_write_node_create(pm_parser_t *parser, pm_class_variable_read_node_t *read_node, pm_token_t *operator, pm_node_t *value) {
3623 return pm_class_variable_write_node_new(
3624 parser->arena,
3625 ++parser->node_id,
3626 pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY),
3627 PM_LOCATION_INIT_NODES(read_node, value),
3628 read_node->name,
3629 read_node->base.location,
3630 value,
3631 TOK2LOC(parser, operator)
3632 );
3633}
3634
3638static pm_constant_path_and_write_node_t *
3639pm_constant_path_and_write_node_create(pm_parser_t *parser, pm_constant_path_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3640 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
3641
3642 return pm_constant_path_and_write_node_new(
3643 parser->arena,
3644 ++parser->node_id,
3645 0,
3646 PM_LOCATION_INIT_NODES(target, value),
3647 target,
3648 TOK2LOC(parser, operator),
3649 value
3650 );
3651}
3652
3656static pm_constant_path_operator_write_node_t *
3657pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_path_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3658 return pm_constant_path_operator_write_node_new(
3659 parser->arena,
3660 ++parser->node_id,
3661 0,
3662 PM_LOCATION_INIT_NODES(target, value),
3663 target,
3664 TOK2LOC(parser, operator),
3665 value,
3666 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1)
3667 );
3668}
3669
3673static pm_constant_path_or_write_node_t *
3674pm_constant_path_or_write_node_create(pm_parser_t *parser, pm_constant_path_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3675 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
3676
3677 return pm_constant_path_or_write_node_new(
3678 parser->arena,
3679 ++parser->node_id,
3680 0,
3681 PM_LOCATION_INIT_NODES(target, value),
3682 target,
3683 TOK2LOC(parser, operator),
3684 value
3685 );
3686}
3687
3691static pm_constant_path_node_t *
3692pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_token_t *delimiter, const pm_token_t *name_token) {
3693 pm_assert_value_expression(parser, parent);
3694
3695 pm_constant_id_t name = PM_CONSTANT_ID_UNSET;
3696 if (name_token->type == PM_TOKEN_CONSTANT) {
3697 name = pm_parser_constant_id_token(parser, name_token);
3698 }
3699
3700 return pm_constant_path_node_new(
3701 parser->arena,
3702 ++parser->node_id,
3703 0,
3704 (parent == NULL) ? PM_LOCATION_INIT_TOKENS(parser, delimiter, name_token) : PM_LOCATION_INIT_NODE_TOKEN(parser, parent, name_token),
3705 parent,
3706 name,
3707 TOK2LOC(parser, delimiter),
3708 TOK2LOC(parser, name_token)
3709 );
3710}
3711
3715static pm_constant_path_write_node_t *
3716pm_constant_path_write_node_create(pm_parser_t *parser, pm_constant_path_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3717 return pm_constant_path_write_node_new(
3718 parser->arena,
3719 ++parser->node_id,
3720 pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY),
3721 PM_LOCATION_INIT_NODES(target, value),
3722 target,
3723 TOK2LOC(parser, operator),
3724 value
3725 );
3726}
3727
3731static pm_constant_and_write_node_t *
3732pm_constant_and_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3733 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
3734
3735 return pm_constant_and_write_node_new(
3736 parser->arena,
3737 ++parser->node_id,
3738 0,
3739 PM_LOCATION_INIT_NODES(target, value),
3740 target->name,
3741 target->base.location,
3742 TOK2LOC(parser, operator),
3743 value
3744 );
3745}
3746
3750static pm_constant_operator_write_node_t *
3751pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3752 return pm_constant_operator_write_node_new(
3753 parser->arena,
3754 ++parser->node_id,
3755 0,
3756 PM_LOCATION_INIT_NODES(target, value),
3757 target->name,
3758 target->base.location,
3759 TOK2LOC(parser, operator),
3760 value,
3761 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1)
3762 );
3763}
3764
3768static pm_constant_or_write_node_t *
3769pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3770 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
3771
3772 return pm_constant_or_write_node_new(
3773 parser->arena,
3774 ++parser->node_id,
3775 0,
3776 PM_LOCATION_INIT_NODES(target, value),
3777 target->name,
3778 target->base.location,
3779 TOK2LOC(parser, operator),
3780 value
3781 );
3782}
3783
3787static pm_constant_read_node_t *
3788pm_constant_read_node_create(pm_parser_t *parser, const pm_token_t *name) {
3789 PRISM_ASSERT(name->type == PM_TOKEN_CONSTANT || name->type == 0);
3790
3791 return pm_constant_read_node_new(
3792 parser->arena,
3793 ++parser->node_id,
3794 0,
3795 PM_LOCATION_INIT_TOKEN(parser, name),
3796 pm_parser_constant_id_token(parser, name)
3797 );
3798}
3799
3803static pm_constant_write_node_t *
3804pm_constant_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
3805 return pm_constant_write_node_new(
3806 parser->arena,
3807 ++parser->node_id,
3808 pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY),
3809 PM_LOCATION_INIT_NODES(target, value),
3810 target->name,
3811 target->base.location,
3812 value,
3813 TOK2LOC(parser, operator)
3814 );
3815}
3816
3820static void
3821pm_def_node_receiver_check(pm_parser_t *parser, const pm_node_t *node) {
3822 switch (PM_NODE_TYPE(node)) {
3823 case PM_BEGIN_NODE: {
3824 const pm_begin_node_t *cast = (pm_begin_node_t *) node;
3825 if (cast->statements != NULL) pm_def_node_receiver_check(parser, UP(cast->statements));
3826 break;
3827 }
3828 case PM_PARENTHESES_NODE: {
3829 const pm_parentheses_node_t *cast = (const pm_parentheses_node_t *) node;
3830 if (cast->body != NULL) pm_def_node_receiver_check(parser, cast->body);
3831 break;
3832 }
3833 case PM_STATEMENTS_NODE: {
3834 const pm_statements_node_t *cast = (const pm_statements_node_t *) node;
3835 pm_def_node_receiver_check(parser, cast->body.nodes[cast->body.size - 1]);
3836 break;
3837 }
3838 case PM_ARRAY_NODE:
3839 case PM_FLOAT_NODE:
3840 case PM_IMAGINARY_NODE:
3841 case PM_INTEGER_NODE:
3842 case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE:
3843 case PM_INTERPOLATED_STRING_NODE:
3844 case PM_INTERPOLATED_SYMBOL_NODE:
3845 case PM_INTERPOLATED_X_STRING_NODE:
3846 case PM_RATIONAL_NODE:
3847 case PM_REGULAR_EXPRESSION_NODE:
3848 case PM_SOURCE_ENCODING_NODE:
3849 case PM_SOURCE_FILE_NODE:
3850 case PM_SOURCE_LINE_NODE:
3851 case PM_STRING_NODE:
3852 case PM_SYMBOL_NODE:
3853 case PM_X_STRING_NODE:
3854 pm_parser_err_node(parser, node, PM_ERR_SINGLETON_FOR_LITERALS);
3855 break;
3856 default:
3857 break;
3858 }
3859}
3860
3864static pm_def_node_t *
3865pm_def_node_create(
3866 pm_parser_t *parser,
3867 pm_constant_id_t name,
3868 const pm_token_t *name_loc,
3869 pm_node_t *receiver,
3870 pm_parameters_node_t *parameters,
3871 pm_node_t *body,
3872 pm_constant_id_list_t *locals,
3873 const pm_token_t *def_keyword,
3874 const pm_token_t *operator,
3875 const pm_token_t *lparen,
3876 const pm_token_t *rparen,
3877 const pm_token_t *equal,
3878 const pm_token_t *end_keyword
3879) {
3880 if (receiver != NULL) {
3881 pm_def_node_receiver_check(parser, receiver);
3882 }
3883
3884 return pm_def_node_new(
3885 parser->arena,
3886 ++parser->node_id,
3887 0,
3888 (end_keyword == NULL) ? PM_LOCATION_INIT_TOKEN_NODE(parser, def_keyword, body) : PM_LOCATION_INIT_TOKENS(parser, def_keyword, end_keyword),
3889 name,
3890 TOK2LOC(parser, name_loc),
3891 receiver,
3892 parameters,
3893 body,
3894 *locals,
3895 TOK2LOC(parser, def_keyword),
3896 NTOK2LOC(parser, operator),
3897 NTOK2LOC(parser, lparen),
3898 NTOK2LOC(parser, rparen),
3899 NTOK2LOC(parser, equal),
3900 NTOK2LOC(parser, end_keyword)
3901 );
3902}
3903
3907static pm_defined_node_t *
3908pm_defined_node_create(pm_parser_t *parser, const pm_token_t *lparen, pm_node_t *value, const pm_token_t *rparen, const pm_token_t *keyword) {
3909 return pm_defined_node_new(
3910 parser->arena,
3911 ++parser->node_id,
3912 0,
3913 (rparen == NULL) ? PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, value) : PM_LOCATION_INIT_TOKENS(parser, keyword, rparen),
3914 NTOK2LOC(parser, lparen),
3915 value,
3916 NTOK2LOC(parser, rparen),
3917 TOK2LOC(parser, keyword)
3918 );
3919}
3920
3924static pm_else_node_t *
3925pm_else_node_create(pm_parser_t *parser, const pm_token_t *else_keyword, pm_statements_node_t *statements, const pm_token_t *end_keyword) {
3926 return pm_else_node_new(
3927 parser->arena,
3928 ++parser->node_id,
3929 0,
3930 (end_keyword == NULL) ? PM_LOCATION_INIT_TOKEN_NODE(parser, else_keyword, statements) : PM_LOCATION_INIT_TOKENS(parser, else_keyword, end_keyword),
3931 TOK2LOC(parser, else_keyword),
3932 statements,
3933 NTOK2LOC(parser, end_keyword)
3934 );
3935}
3936
3940static pm_embedded_statements_node_t *
3941pm_embedded_statements_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_statements_node_t *statements, const pm_token_t *closing) {
3942 return pm_embedded_statements_node_new(
3943 parser->arena,
3944 ++parser->node_id,
3945 0,
3946 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
3947 TOK2LOC(parser, opening),
3948 statements,
3949 TOK2LOC(parser, closing)
3950 );
3951}
3952
3956static pm_embedded_variable_node_t *
3957pm_embedded_variable_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t *variable) {
3958 return pm_embedded_variable_node_new(
3959 parser->arena,
3960 ++parser->node_id,
3961 0,
3962 PM_LOCATION_INIT_TOKEN_NODE(parser, operator, variable),
3963 TOK2LOC(parser, operator),
3964 variable
3965 );
3966}
3967
3971static pm_ensure_node_t *
3972pm_ensure_node_create(pm_parser_t *parser, const pm_token_t *ensure_keyword, pm_statements_node_t *statements, const pm_token_t *end_keyword) {
3973 return pm_ensure_node_new(
3974 parser->arena,
3975 ++parser->node_id,
3976 0,
3977 PM_LOCATION_INIT_TOKENS(parser, ensure_keyword, end_keyword),
3978 TOK2LOC(parser, ensure_keyword),
3979 statements,
3980 TOK2LOC(parser, end_keyword)
3981 );
3982}
3983
3987static pm_false_node_t *
3988pm_false_node_create(pm_parser_t *parser, const pm_token_t *token) {
3989 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD_FALSE);
3990
3991 return pm_false_node_new(
3992 parser->arena,
3993 ++parser->node_id,
3994 PM_NODE_FLAG_STATIC_LITERAL,
3995 PM_LOCATION_INIT_TOKEN(parser, token)
3996 );
3997}
3998
4003static pm_find_pattern_node_t *
4004pm_find_pattern_node_create(pm_parser_t *parser, const pm_node_list_t *nodes) {
4005 PRISM_ASSERT(nodes->size >= 2);
4006 pm_node_t *left = nodes->nodes[0];
4007 pm_node_t *right = nodes->nodes[nodes->size - 1];
4008
4009 PRISM_ASSERT(PM_NODE_TYPE_P(left, PM_SPLAT_NODE));
4010 PRISM_ASSERT(PM_NODE_TYPE_P(right, PM_SPLAT_NODE));
4011
4012 return pm_find_pattern_node_new(
4013 parser->arena,
4014 ++parser->node_id,
4015 0,
4016 PM_LOCATION_INIT_NODES(left, right),
4017 NULL,
4018 (pm_splat_node_t *) left,
4019 pm_node_list_slice(nodes, 1, nodes->size - 2),
4020 (pm_splat_node_t *) right,
4021 ((pm_location_t) { 0 }),
4022 ((pm_location_t) { 0 })
4023 );
4024}
4025
4030static double
4031pm_float_node_value(pm_parser_t *parser, const pm_token_t *token) {
4032 bool out_of_range;
4033 double value = pm_double_parse(token->start, token->end, &out_of_range);
4034
4035 if (out_of_range) {
4036 size_t length = PM_TOKEN_LENGTH(token);
4037 int warn_width;
4038 const char *ellipsis;
4039
4040 if (length > 20) {
4041 warn_width = 20;
4042 ellipsis = "...";
4043 } else {
4044 warn_width = (int) length;
4045 ellipsis = "";
4046 }
4047
4048 pm_diagnostic_list_append_format(&parser->metadata_arena, &parser->warning_list, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), PM_WARN_FLOAT_OUT_OF_RANGE, warn_width, (const char *) token->start, ellipsis);
4049 }
4050
4051 return value;
4052}
4053
4057static pm_float_node_t *
4058pm_float_node_create(pm_parser_t *parser, const pm_token_t *token) {
4059 PRISM_ASSERT(token->type == PM_TOKEN_FLOAT);
4060
4061 return pm_float_node_new(
4062 parser->arena,
4063 ++parser->node_id,
4064 PM_NODE_FLAG_STATIC_LITERAL,
4065 PM_LOCATION_INIT_TOKEN(parser, token),
4066 pm_float_node_value(parser, token)
4067 );
4068}
4069
4073static pm_imaginary_node_t *
4074pm_float_node_imaginary_create(pm_parser_t *parser, const pm_token_t *token) {
4075 PRISM_ASSERT(token->type == PM_TOKEN_FLOAT_IMAGINARY);
4076
4077 return pm_imaginary_node_new(
4078 parser->arena,
4079 ++parser->node_id,
4080 PM_NODE_FLAG_STATIC_LITERAL,
4081 PM_LOCATION_INIT_TOKEN(parser, token),
4082 UP(pm_float_node_create(parser, &((pm_token_t) {
4083 .type = PM_TOKEN_FLOAT,
4084 .start = token->start,
4085 .end = token->end - 1
4086 })))
4087 );
4088}
4089
4093static pm_rational_node_t *
4094pm_float_node_rational_create(pm_parser_t *parser, const pm_token_t *token) {
4095 PRISM_ASSERT(token->type == PM_TOKEN_FLOAT_RATIONAL);
4096
4097 pm_rational_node_t *node = pm_rational_node_new(
4098 parser->arena,
4099 ++parser->node_id,
4100 PM_INTEGER_BASE_FLAGS_DECIMAL | PM_NODE_FLAG_STATIC_LITERAL,
4101 PM_LOCATION_INIT_TOKEN(parser, token),
4102 ((pm_integer_t) { 0 }),
4103 ((pm_integer_t) { 0 })
4104 );
4105
4106 pm_integer_parse_rational(parser->arena, &node->numerator, &node->denominator, token->start, token->end - 1);
4107 return node;
4108}
4109
4114static pm_imaginary_node_t *
4115pm_float_node_rational_imaginary_create(pm_parser_t *parser, const pm_token_t *token) {
4116 PRISM_ASSERT(token->type == PM_TOKEN_FLOAT_RATIONAL_IMAGINARY);
4117
4118 return pm_imaginary_node_new(
4119 parser->arena,
4120 ++parser->node_id,
4121 PM_NODE_FLAG_STATIC_LITERAL,
4122 PM_LOCATION_INIT_TOKEN(parser, token),
4123 UP(pm_float_node_rational_create(parser, &((pm_token_t) {
4124 .type = PM_TOKEN_FLOAT_RATIONAL,
4125 .start = token->start,
4126 .end = token->end - 1
4127 })))
4128 );
4129}
4130
4134static pm_for_node_t *
4135pm_for_node_create(
4136 pm_parser_t *parser,
4137 pm_node_t *index,
4138 pm_node_t *collection,
4139 pm_statements_node_t *statements,
4140 const pm_token_t *for_keyword,
4141 const pm_token_t *in_keyword,
4142 const pm_token_t *do_keyword,
4143 const pm_token_t *end_keyword
4144) {
4145 return pm_for_node_new(
4146 parser->arena,
4147 ++parser->node_id,
4148 0,
4149 PM_LOCATION_INIT_TOKENS(parser, for_keyword, end_keyword),
4150 index,
4151 collection,
4152 statements,
4153 TOK2LOC(parser, for_keyword),
4154 TOK2LOC(parser, in_keyword),
4155 NTOK2LOC(parser, do_keyword),
4156 TOK2LOC(parser, end_keyword)
4157 );
4158}
4159
4163static pm_forwarding_arguments_node_t *
4164pm_forwarding_arguments_node_create(pm_parser_t *parser, const pm_token_t *token) {
4165 PRISM_ASSERT(token->type == PM_TOKEN_UDOT_DOT_DOT);
4166
4167 return pm_forwarding_arguments_node_new(
4168 parser->arena,
4169 ++parser->node_id,
4170 0,
4171 PM_LOCATION_INIT_TOKEN(parser, token)
4172 );
4173}
4174
4178static pm_forwarding_parameter_node_t *
4179pm_forwarding_parameter_node_create(pm_parser_t *parser, const pm_token_t *token) {
4180 PRISM_ASSERT(token->type == PM_TOKEN_UDOT_DOT_DOT);
4181
4182 return pm_forwarding_parameter_node_new(
4183 parser->arena,
4184 ++parser->node_id,
4185 0,
4186 PM_LOCATION_INIT_TOKEN(parser, token)
4187 );
4188}
4189
4193static pm_forwarding_super_node_t *
4194pm_forwarding_super_node_create(pm_parser_t *parser, const pm_token_t *token, pm_arguments_t *arguments) {
4195 PRISM_ASSERT(arguments->block == NULL || PM_NODE_TYPE_P(arguments->block, PM_BLOCK_NODE));
4196 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD_SUPER);
4197
4198 pm_block_node_t *block = NULL;
4199 if (arguments->block != NULL) {
4200 block = (pm_block_node_t *) arguments->block;
4201 }
4202
4203 return pm_forwarding_super_node_new(
4204 parser->arena,
4205 ++parser->node_id,
4206 0,
4207 (block == NULL) ? PM_LOCATION_INIT_TOKEN(parser, token) : PM_LOCATION_INIT_TOKEN_NODE(parser, token, block),
4208 PM_LOCATION_INIT_TOKEN(parser, token),
4209 block
4210 );
4211}
4212
4217static pm_hash_pattern_node_t *
4218pm_hash_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *closing) {
4219 return pm_hash_pattern_node_new(
4220 parser->arena,
4221 ++parser->node_id,
4222 0,
4223 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
4224 NULL,
4225 ((pm_node_list_t) { 0 }),
4226 NULL,
4227 TOK2LOC(parser, opening),
4228 TOK2LOC(parser, closing)
4229 );
4230}
4231
4236static pm_hash_pattern_node_t *
4237pm_hash_pattern_node_node_list_create(pm_parser_t *parser, pm_node_list_t *elements, pm_node_t *rest) {
4238 pm_node_list_finalize(&parser->node_chunks, parser->arena, elements);
4239
4240 uint32_t start;
4241 uint32_t end;
4242
4243 if (elements->size > 0) {
4244 if (rest) {
4245 start = MIN(PM_NODE_START(rest), PM_NODE_START(elements->nodes[0]));
4246 end = MAX(PM_NODE_END(rest), PM_NODE_END(elements->nodes[elements->size - 1]));
4247 } else {
4248 start = PM_NODE_START(elements->nodes[0]);
4249 end = PM_NODE_END(elements->nodes[elements->size - 1]);
4250 }
4251 } else {
4252 PRISM_ASSERT(rest != NULL);
4253 start = PM_NODE_START(rest);
4254 end = PM_NODE_END(rest);
4255 }
4256
4257 return pm_hash_pattern_node_new(
4258 parser->arena,
4259 ++parser->node_id,
4260 0,
4261 PM_LOCATION_INIT_BOUNDS(start, end),
4262 NULL,
4263 *elements,
4264 rest,
4265 ((pm_location_t) { 0 }),
4266 ((pm_location_t) { 0 })
4267 );
4268}
4269
4273static pm_constant_id_t
4274pm_global_variable_write_name(pm_parser_t *parser, const pm_node_t *target) {
4275 switch (PM_NODE_TYPE(target)) {
4276 case PM_GLOBAL_VARIABLE_READ_NODE:
4277 return ((pm_global_variable_read_node_t *) target)->name;
4278 case PM_BACK_REFERENCE_READ_NODE:
4279 return ((pm_back_reference_read_node_t *) target)->name;
4280 case PM_NUMBERED_REFERENCE_READ_NODE:
4281 // This will only ever happen in the event of a syntax error, but we
4282 // still need to provide something for the node.
4283 return pm_parser_constant_id_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target));
4284 default:
4285 PRISM_ASSERT(false && "unreachable");
4286 return (pm_constant_id_t) -1;
4287 }
4288}
4289
4293static pm_global_variable_and_write_node_t *
4294pm_global_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, const pm_token_t *operator, pm_node_t *value) {
4295 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
4296
4297 return pm_global_variable_and_write_node_new(
4298 parser->arena,
4299 ++parser->node_id,
4300 0,
4301 PM_LOCATION_INIT_NODES(target, value),
4302 pm_global_variable_write_name(parser, target),
4303 target->location,
4304 TOK2LOC(parser, operator),
4305 value
4306 );
4307}
4308
4312static pm_global_variable_operator_write_node_t *
4313pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *target, const pm_token_t *operator, pm_node_t *value) {
4314 return pm_global_variable_operator_write_node_new(
4315 parser->arena,
4316 ++parser->node_id,
4317 0,
4318 PM_LOCATION_INIT_NODES(target, value),
4319 pm_global_variable_write_name(parser, target),
4320 target->location,
4321 TOK2LOC(parser, operator),
4322 value,
4323 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1)
4324 );
4325}
4326
4330static pm_global_variable_or_write_node_t *
4331pm_global_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, const pm_token_t *operator, pm_node_t *value) {
4332 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
4333
4334 return pm_global_variable_or_write_node_new(
4335 parser->arena,
4336 ++parser->node_id,
4337 0,
4338 PM_LOCATION_INIT_NODES(target, value),
4339 pm_global_variable_write_name(parser, target),
4340 target->location,
4341 TOK2LOC(parser, operator),
4342 value
4343 );
4344}
4345
4349static pm_global_variable_read_node_t *
4350pm_global_variable_read_node_create(pm_parser_t *parser, const pm_token_t *name) {
4351 return pm_global_variable_read_node_new(
4352 parser->arena,
4353 ++parser->node_id,
4354 0,
4355 PM_LOCATION_INIT_TOKEN(parser, name),
4356 pm_parser_constant_id_token(parser, name)
4357 );
4358}
4359
4363static pm_global_variable_read_node_t *
4364pm_global_variable_read_node_synthesized_create(pm_parser_t *parser, pm_constant_id_t name) {
4365 return pm_global_variable_read_node_new(
4366 parser->arena,
4367 ++parser->node_id,
4368 0,
4369 PM_LOCATION_INIT_UNSET,
4370 name
4371 );
4372}
4373
4377static pm_global_variable_write_node_t *
4378pm_global_variable_write_node_create(pm_parser_t *parser, pm_node_t *target, const pm_token_t *operator, pm_node_t *value) {
4379 return pm_global_variable_write_node_new(
4380 parser->arena,
4381 ++parser->node_id,
4382 pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY),
4383 PM_LOCATION_INIT_NODES(target, value),
4384 pm_global_variable_write_name(parser, target),
4385 target->location,
4386 value,
4387 TOK2LOC(parser, operator)
4388 );
4389}
4390
4394static pm_global_variable_write_node_t *
4395pm_global_variable_write_node_synthesized_create(pm_parser_t *parser, pm_constant_id_t name, pm_node_t *value) {
4396 return pm_global_variable_write_node_new(
4397 parser->arena,
4398 ++parser->node_id,
4399 0,
4400 PM_LOCATION_INIT_UNSET,
4401 name,
4402 ((pm_location_t) { 0 }),
4403 value,
4404 ((pm_location_t) { 0 })
4405 );
4406}
4407
4414typedef struct {
4416 pm_node_list_t nodes;
4417
4423 bool dynamic;
4424
4426 bool non_symbol_keys;
4427} pm_hash_elements_t;
4428
4433static PRISM_INLINE void
4434pm_hash_elements_append(pm_parser_t *parser, pm_hash_elements_t *elements, pm_node_t *element) {
4435 if (!PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL)) {
4436 elements->dynamic = true;
4437 }
4438
4439 if (!PM_NODE_TYPE_P(element, PM_ASSOC_NODE) || !PM_NODE_TYPE_P(((const pm_assoc_node_t *) element)->key, PM_SYMBOL_NODE)) {
4440 elements->non_symbol_keys = true;
4441 }
4442
4443 pm_node_list_open_append(&parser->node_chunks, &elements->nodes, element);
4444}
4445
4449static pm_hash_node_t *
4450pm_hash_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_hash_elements_t *elements, const pm_token_t *closing) {
4451 pm_node_list_finalize(&parser->node_chunks, parser->arena, &elements->nodes);
4452
4453 return pm_hash_node_new(
4454 parser->arena,
4455 ++parser->node_id,
4456 elements->dynamic ? 0 : PM_NODE_FLAG_STATIC_LITERAL,
4457 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
4458 TOK2LOC(parser, opening),
4459 elements->nodes,
4460 TOK2LOC(parser, closing)
4461 );
4462}
4463
4467static pm_if_node_t *
4468pm_if_node_create(pm_parser_t *parser,
4469 const pm_token_t *if_keyword,
4470 pm_node_t *predicate,
4471 const pm_token_t *then_keyword,
4472 pm_statements_node_t *statements,
4473 pm_node_t *subsequent,
4474 const pm_token_t *end_keyword
4475) {
4476 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
4477
4478 uint32_t start = PM_TOKEN_START(parser, if_keyword);
4479 uint32_t end;
4480
4481 if (end_keyword != NULL) {
4482 end = PM_TOKEN_END(parser, end_keyword);
4483 } else if (subsequent != NULL) {
4484 end = PM_NODE_END(subsequent);
4485 } else if (pm_statements_node_body_length(statements) != 0) {
4486 end = PM_NODE_END(statements);
4487 } else {
4488 end = PM_NODE_END(predicate);
4489 }
4490
4491 return pm_if_node_new(
4492 parser->arena,
4493 ++parser->node_id,
4494 PM_NODE_FLAG_NEWLINE,
4495 ((pm_location_t) { .start = start, .length = U32(end - start) }),
4496 TOK2LOC(parser, if_keyword),
4497 predicate,
4498 NTOK2LOC(parser, then_keyword),
4499 statements,
4500 subsequent,
4501 NTOK2LOC(parser, end_keyword)
4502 );
4503}
4504
4508static pm_if_node_t *
4509pm_if_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const pm_token_t *if_keyword, pm_node_t *predicate) {
4510 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
4511
4512 pm_statements_node_t *statements = pm_statements_node_single_create(parser, statement, true);
4513
4514 return pm_if_node_new(
4515 parser->arena,
4516 ++parser->node_id,
4517 PM_NODE_FLAG_NEWLINE,
4518 PM_LOCATION_INIT_NODES(statement, predicate),
4519 TOK2LOC(parser, if_keyword),
4520 predicate,
4521 ((pm_location_t) { 0 }),
4522 statements,
4523 NULL,
4524 ((pm_location_t) { 0 })
4525 );
4526}
4527
4531static pm_if_node_t *
4532pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_token_t *qmark, pm_node_t *true_expression, const pm_token_t *colon, pm_node_t *false_expression) {
4533 pm_assert_value_expression(parser, predicate);
4534 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
4535
4536 pm_statements_node_t *if_statements = pm_statements_node_single_create(parser, true_expression, true);
4537 pm_statements_node_t *else_statements = pm_statements_node_single_create(parser, false_expression, true);
4538
4539 pm_else_node_t *else_node = pm_else_node_create(parser, colon, else_statements, NULL);
4540 return pm_if_node_new(
4541 parser->arena,
4542 ++parser->node_id,
4543 PM_NODE_FLAG_NEWLINE,
4544 PM_LOCATION_INIT_NODES(predicate, false_expression),
4545 ((pm_location_t) { 0 }),
4546 predicate,
4547 TOK2LOC(parser, qmark),
4548 if_statements,
4549 UP(else_node),
4550 ((pm_location_t) { 0 })
4551 );
4552}
4553
4554static PRISM_INLINE void
4555pm_if_node_end_keyword_loc_set(const pm_parser_t *parser, pm_if_node_t *node, const pm_token_t *keyword) {
4556 PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword);
4557 node->end_keyword_loc = TOK2LOC(parser, keyword);
4558}
4559
4560static PRISM_INLINE void
4561pm_else_node_end_keyword_loc_set(const pm_parser_t *parser, pm_else_node_t *node, const pm_token_t *keyword) {
4562 PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword);
4563 node->end_keyword_loc = TOK2LOC(parser, keyword);
4564}
4565
4569static pm_implicit_node_t *
4570pm_implicit_node_create(pm_parser_t *parser, pm_node_t *value) {
4571 return pm_implicit_node_new(
4572 parser->arena,
4573 ++parser->node_id,
4574 0,
4575 PM_LOCATION_INIT_NODE(value),
4576 value
4577 );
4578}
4579
4583static pm_implicit_rest_node_t *
4584pm_implicit_rest_node_create(pm_parser_t *parser, const pm_token_t *token) {
4585 PRISM_ASSERT(token->type == PM_TOKEN_COMMA);
4586
4587 return pm_implicit_rest_node_new(
4588 parser->arena,
4589 ++parser->node_id,
4590 0,
4591 PM_LOCATION_INIT_TOKEN(parser, token)
4592 );
4593}
4594
4598static pm_integer_node_t *
4599pm_integer_node_create(pm_parser_t *parser, pm_node_flags_t base, const pm_token_t *token) {
4600 PRISM_ASSERT(token->type == PM_TOKEN_INTEGER);
4601
4602 pm_integer_node_t *node = pm_integer_node_new(
4603 parser->arena,
4604 ++parser->node_id,
4605 base | PM_NODE_FLAG_STATIC_LITERAL,
4606 PM_LOCATION_INIT_TOKEN(parser, token),
4607 ((pm_integer_t) { 0 })
4608 );
4609
4610 if (parser->integer.start == token->start) {
4611 // The value was already computed during lexing.
4612 node->value = pm_integer_uint64(parser->integer.value);
4613 } else {
4614 pm_integer_base_t integer_base = PM_INTEGER_BASE_DECIMAL;
4615 switch (base) {
4616 case PM_INTEGER_BASE_FLAGS_BINARY: integer_base = PM_INTEGER_BASE_BINARY; break;
4617 case PM_INTEGER_BASE_FLAGS_OCTAL: integer_base = PM_INTEGER_BASE_OCTAL; break;
4618 case PM_INTEGER_BASE_FLAGS_DECIMAL: break;
4619 case PM_INTEGER_BASE_FLAGS_HEXADECIMAL: integer_base = PM_INTEGER_BASE_HEXADECIMAL; break;
4620 default: PRISM_ASSERT(false && "unreachable"); break;
4621 }
4622
4623 pm_integer_parse(parser->arena, &node->value, integer_base, token->start, token->end);
4624 }
4625
4626 return node;
4627}
4628
4633static pm_imaginary_node_t *
4634pm_integer_node_imaginary_create(pm_parser_t *parser, pm_node_flags_t base, const pm_token_t *token) {
4635 PRISM_ASSERT(token->type == PM_TOKEN_INTEGER_IMAGINARY);
4636
4637 return pm_imaginary_node_new(
4638 parser->arena,
4639 ++parser->node_id,
4640 PM_NODE_FLAG_STATIC_LITERAL,
4641 PM_LOCATION_INIT_TOKEN(parser, token),
4642 UP(pm_integer_node_create(parser, base, &((pm_token_t) {
4643 .type = PM_TOKEN_INTEGER,
4644 .start = token->start,
4645 .end = token->end - 1
4646 })))
4647 );
4648}
4649
4654static pm_rational_node_t *
4655pm_integer_node_rational_create(pm_parser_t *parser, pm_node_flags_t base, const pm_token_t *token) {
4656 PRISM_ASSERT(token->type == PM_TOKEN_INTEGER_RATIONAL);
4657
4658 pm_rational_node_t *node = pm_rational_node_new(
4659 parser->arena,
4660 ++parser->node_id,
4661 base | PM_NODE_FLAG_STATIC_LITERAL,
4662 PM_LOCATION_INIT_TOKEN(parser, token),
4663 ((pm_integer_t) { 0 }),
4664 pm_integer_uint64(1)
4665 );
4666
4667 pm_integer_base_t integer_base = PM_INTEGER_BASE_DECIMAL;
4668 switch (base) {
4669 case PM_INTEGER_BASE_FLAGS_BINARY: integer_base = PM_INTEGER_BASE_BINARY; break;
4670 case PM_INTEGER_BASE_FLAGS_OCTAL: integer_base = PM_INTEGER_BASE_OCTAL; break;
4671 case PM_INTEGER_BASE_FLAGS_DECIMAL: break;
4672 case PM_INTEGER_BASE_FLAGS_HEXADECIMAL: integer_base = PM_INTEGER_BASE_HEXADECIMAL; break;
4673 default: PRISM_ASSERT(false && "unreachable"); break;
4674 }
4675
4676 pm_integer_parse(parser->arena, &node->numerator, integer_base, token->start, token->end - 1);
4677
4678 return node;
4679}
4680
4685static pm_imaginary_node_t *
4686pm_integer_node_rational_imaginary_create(pm_parser_t *parser, pm_node_flags_t base, const pm_token_t *token) {
4687 PRISM_ASSERT(token->type == PM_TOKEN_INTEGER_RATIONAL_IMAGINARY);
4688
4689 return pm_imaginary_node_new(
4690 parser->arena,
4691 ++parser->node_id,
4692 PM_NODE_FLAG_STATIC_LITERAL,
4693 PM_LOCATION_INIT_TOKEN(parser, token),
4694 UP(pm_integer_node_rational_create(parser, base, &((pm_token_t) {
4695 .type = PM_TOKEN_INTEGER_RATIONAL,
4696 .start = token->start,
4697 .end = token->end - 1
4698 })))
4699 );
4700}
4701
4705static pm_in_node_t *
4706pm_in_node_create(pm_parser_t *parser, pm_node_t *pattern, pm_statements_node_t *statements, const pm_token_t *in_keyword, const pm_token_t *then_keyword) {
4707 uint32_t start = PM_TOKEN_START(parser, in_keyword);
4708 uint32_t end;
4709
4710 if (statements != NULL) {
4711 end = PM_NODE_END(statements);
4712 } else if (then_keyword != NULL) {
4713 end = PM_TOKEN_END(parser, then_keyword);
4714 } else {
4715 end = PM_NODE_END(pattern);
4716 }
4717
4718 return pm_in_node_new(
4719 parser->arena,
4720 ++parser->node_id,
4721 0,
4722 PM_LOCATION_INIT_BOUNDS(start, end),
4723 pattern,
4724 statements,
4725 TOK2LOC(parser, in_keyword),
4726 NTOK2LOC(parser, then_keyword)
4727 );
4728}
4729
4733static pm_instance_variable_and_write_node_t *
4734pm_instance_variable_and_write_node_create(pm_parser_t *parser, pm_instance_variable_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
4735 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
4736
4737 return pm_instance_variable_and_write_node_new(
4738 parser->arena,
4739 ++parser->node_id,
4740 0,
4741 PM_LOCATION_INIT_NODES(target, value),
4742 target->name,
4743 target->base.location,
4744 TOK2LOC(parser, operator),
4745 value
4746 );
4747}
4748
4752static pm_instance_variable_operator_write_node_t *
4753pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance_variable_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
4754 return pm_instance_variable_operator_write_node_new(
4755 parser->arena,
4756 ++parser->node_id,
4757 0,
4758 PM_LOCATION_INIT_NODES(target, value),
4759 target->name,
4760 target->base.location,
4761 TOK2LOC(parser, operator),
4762 value,
4763 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1)
4764 );
4765}
4766
4770static pm_instance_variable_or_write_node_t *
4771pm_instance_variable_or_write_node_create(pm_parser_t *parser, pm_instance_variable_read_node_t *target, const pm_token_t *operator, pm_node_t *value) {
4772 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
4773
4774 return pm_instance_variable_or_write_node_new(
4775 parser->arena,
4776 ++parser->node_id,
4777 0,
4778 PM_LOCATION_INIT_NODES(target, value),
4779 target->name,
4780 target->base.location,
4781 TOK2LOC(parser, operator),
4782 value
4783 );
4784}
4785
4789static pm_instance_variable_read_node_t *
4790pm_instance_variable_read_node_create(pm_parser_t *parser, const pm_token_t *token) {
4791 PRISM_ASSERT(token->type == PM_TOKEN_INSTANCE_VARIABLE);
4792
4793 return pm_instance_variable_read_node_new(
4794 parser->arena,
4795 ++parser->node_id,
4796 0,
4797 PM_LOCATION_INIT_TOKEN(parser, token),
4798 pm_parser_constant_id_token(parser, token)
4799 );
4800}
4801
4806static pm_instance_variable_write_node_t *
4807pm_instance_variable_write_node_create(pm_parser_t *parser, pm_instance_variable_read_node_t *read_node, pm_token_t *operator, pm_node_t *value) {
4808 return pm_instance_variable_write_node_new(
4809 parser->arena,
4810 ++parser->node_id,
4811 pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY),
4812 PM_LOCATION_INIT_NODES(read_node, value),
4813 read_node->name,
4814 read_node->base.location,
4815 value,
4816 TOK2LOC(parser, operator)
4817 );
4818}
4819
4825static void
4826pm_interpolated_part_flags(pm_node_flags_t *flags, pm_node_t *part) {
4827 switch (PM_NODE_TYPE(part)) {
4828 case PM_STRING_NODE:
4829 pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN);
4830 break;
4831 case PM_EMBEDDED_STATEMENTS_NODE: {
4832 pm_embedded_statements_node_t *cast = (pm_embedded_statements_node_t *) part;
4833 pm_node_t *embedded = (cast->statements != NULL && cast->statements->body.size == 1) ? cast->statements->body.nodes[0] : NULL;
4834
4835 if (embedded == NULL) {
4836 // If there are no statements or more than one statement, then
4837 // we lose the static literal flag.
4838 *flags &= (pm_node_flags_t) ~PM_NODE_FLAG_STATIC_LITERAL;
4839 } else if (PM_NODE_TYPE_P(embedded, PM_STRING_NODE)) {
4840 // If the embedded statement is a string, then we can keep the
4841 // static literal flag and mark the string as frozen.
4842 pm_node_flag_set(embedded, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN);
4843 } else if (PM_NODE_TYPE_P(embedded, PM_INTERPOLATED_STRING_NODE) && PM_NODE_FLAG_P(embedded, PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE)) {
4844 /* An interpolated string whose value is known while parsing
4845 * keeps the static literal flag, frozen or not, since only its
4846 * value is interpolated. */
4847 } else {
4848 // Otherwise we lose the static literal flag.
4849 *flags &= (pm_node_flags_t) ~PM_NODE_FLAG_STATIC_LITERAL;
4850 }
4851
4852 break;
4853 }
4854 case PM_EMBEDDED_VARIABLE_NODE:
4855 *flags &= (pm_node_flags_t) ~PM_NODE_FLAG_STATIC_LITERAL;
4856 break;
4857 default:
4858 PRISM_ASSERT(false && "unexpected node type");
4859 break;
4860 }
4861}
4862
4868static pm_location_t
4869pm_interpolated_location(const pm_parser_t *parser, const pm_token_t *opening, const pm_node_list_t *parts, const pm_token_t *closing) {
4870 uint32_t start = 0;
4871 uint32_t end = closing == NULL ? 0 : PM_TOKEN_END(parser, closing);
4872
4873 if (opening != NULL) {
4874 start = PM_TOKEN_START(parser, opening);
4875 if (closing == NULL) end = PM_TOKEN_END(parser, opening);
4876 } else if (parts->size > 0) {
4877 start = PM_NODE_START(parts->nodes[0]);
4878 }
4879
4880 if (parts->size > 0 && PM_NODE_END(parts->nodes[parts->size - 1]) > end) {
4881 end = PM_NODE_END(parts->nodes[parts->size - 1]);
4882 }
4883
4884 return PM_LOCATION_INIT_BOUNDS(start, end);
4885}
4886
4891static pm_interpolated_regular_expression_node_t *
4892pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_node_list_t *parts, const pm_token_t *closing) {
4893 pm_node_list_finalize(&parser->node_chunks, parser->arena, parts);
4894
4895 pm_node_flags_t flags = PM_NODE_FLAG_STATIC_LITERAL;
4896 for (size_t idx = 0; idx < parts->size; ++idx) {
4897 pm_interpolated_part_flags(&flags, parts->nodes[idx]);
4898 }
4899 flags |= pm_regular_expression_flags_create(parser, closing);
4900
4901 return pm_interpolated_regular_expression_node_new(
4902 parser->arena,
4903 ++parser->node_id,
4904 flags,
4905 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
4906 TOK2LOC(parser, opening),
4907 *parts,
4908 TOK2LOC(parser, closing)
4909 );
4910}
4911
4940static pm_interpolated_string_node_t *
4941pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_node_list_t *parts, const pm_token_t *closing) {
4942#define CLEAR_FLAGS() \
4943 flags = (pm_node_flags_t) (flags & ~(PM_NODE_FLAG_STATIC_LITERAL | PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE))
4944
4945#define MUTABLE_FLAGS() \
4946 flags = (pm_node_flags_t) ((flags | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE) & ~PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN)
4947
4948 pm_node_list_finalize(&parser->node_chunks, parser->arena, parts);
4949 pm_location_t location = pm_interpolated_location(parser, opening, parts, closing);
4950
4951 pm_node_flags_t flags = PM_NODE_FLAG_STATIC_LITERAL | PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE;
4952
4953 switch (parser->frozen_string_literal) {
4954 case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED:
4955 flags |= PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE;
4956 break;
4957 case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED:
4958 flags |= PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN;
4959 break;
4960 }
4961
4962 for (size_t idx = 0; idx < parts->size; ++idx) {
4963 pm_node_t *part = parts->nodes[idx];
4964
4965 switch (PM_NODE_TYPE(part)) {
4966 case PM_STRING_NODE:
4967 // If inner string is not frozen, it stops being a static literal. We should *not* clear other flags,
4968 // because concatenating two frozen strings (`'foo' 'bar'`) is still frozen. This holds true for
4969 // as long as this interpolation only consists of other string literals.
4970 if (!PM_NODE_FLAG_P(part, PM_STRING_FLAGS_FROZEN)) {
4971 flags &= (pm_node_flags_t) ~PM_NODE_FLAG_STATIC_LITERAL;
4972 }
4973 part->flags = (pm_node_flags_t) ((part->flags | PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN) & ~PM_STRING_FLAGS_MUTABLE);
4974 break;
4975 case PM_INTERPOLATED_STRING_NODE:
4976 if (PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) {
4977 // If the string that we're concatenating is a static literal,
4978 // then we can keep the static literal flag for this string.
4979 } else {
4980 // Otherwise, we lose the static literal flag here and we should
4981 // also clear the mutability flags.
4982 CLEAR_FLAGS();
4983 }
4984
4985 if (!PM_NODE_FLAG_P(part, PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE)) {
4986 flags &= (pm_node_flags_t) ~PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE;
4987 }
4988 break;
4989 case PM_EMBEDDED_STATEMENTS_NODE: {
4990 pm_embedded_statements_node_t *cast = (pm_embedded_statements_node_t *) part;
4991 pm_node_t *embedded = (cast->statements != NULL && cast->statements->body.size == 1) ? cast->statements->body.nodes[0] : NULL;
4992
4993 if (embedded == NULL) {
4994 // If we're embedding multiple statements or no statements, then
4995 // the string is not longer a static literal.
4996 CLEAR_FLAGS();
4997 } else if (PM_NODE_TYPE_P(embedded, PM_STRING_NODE)) {
4998 // If the embedded statement is a string, then we can make that
4999 // string as frozen and static literal, and not touch the static
5000 // literal status of this string.
5001 embedded->flags = (pm_node_flags_t) ((embedded->flags | PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN) & ~PM_STRING_FLAGS_MUTABLE);
5002
5003 if (flags & PM_NODE_FLAG_STATIC_LITERAL) {
5004 MUTABLE_FLAGS();
5005 }
5006 } else if (PM_NODE_TYPE_P(embedded, PM_INTERPOLATED_STRING_NODE) && PM_NODE_FLAG_P(embedded, PM_NODE_FLAG_STATIC_LITERAL)) {
5007 // If the embedded statement is an interpolated string, but that
5008 // string is marked as static literal, then we can keep our
5009 // static literal status for this string.
5010 if (flags & PM_NODE_FLAG_STATIC_LITERAL) {
5011 MUTABLE_FLAGS();
5012 }
5013 } else {
5014 // In all other cases, we lose the static literal flag here and
5015 // become mutable.
5016 CLEAR_FLAGS();
5017 }
5018
5019 if (embedded == NULL || !(PM_NODE_TYPE_P(embedded, PM_STRING_NODE) || (PM_NODE_TYPE_P(embedded, PM_INTERPOLATED_STRING_NODE) && PM_NODE_FLAG_P(embedded, PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE)))) {
5020 flags &= (pm_node_flags_t) ~PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE;
5021 }
5022
5023 break;
5024 }
5025 case PM_EMBEDDED_VARIABLE_NODE:
5026 // Embedded variables clear static literal, which means we also
5027 // should clear the mutability flags.
5028 CLEAR_FLAGS();
5029 flags &= (pm_node_flags_t) ~PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE;
5030 break;
5031 case PM_X_STRING_NODE:
5032 case PM_INTERPOLATED_X_STRING_NODE:
5033 case PM_SYMBOL_NODE:
5034 case PM_INTERPOLATED_SYMBOL_NODE:
5035 // These will only happen in error cases. But we want to handle it
5036 // here so that we don't fail the assertion.
5037 CLEAR_FLAGS();
5038 flags &= (pm_node_flags_t) ~PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE;
5039 parts->nodes[idx] = UP(pm_error_recovery_node_create_unexpected(parser, part));
5040 break;
5041 case PM_ERROR_RECOVERY_NODE:
5042 CLEAR_FLAGS();
5043 flags &= (pm_node_flags_t) ~PM_INTERPOLATED_STRING_NODE_FLAGS_STATIC_VALUE;
5044 break;
5045 default:
5046 PRISM_ASSERT(false && "unexpected node type");
5047 break;
5048 }
5049 }
5050
5051 return pm_interpolated_string_node_new(
5052 parser->arena,
5053 ++parser->node_id,
5054 flags,
5055 location,
5056 NTOK2LOC(parser, opening),
5057 *parts,
5058 NTOK2LOC(parser, closing)
5059 );
5060
5061#undef CLEAR_FLAGS
5062#undef MUTABLE_FLAGS
5063}
5064
5069static pm_interpolated_symbol_node_t *
5070pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_node_list_t *parts, const pm_token_t *closing) {
5071 pm_node_list_finalize(&parser->node_chunks, parser->arena, parts);
5072
5073 pm_node_flags_t flags = PM_NODE_FLAG_STATIC_LITERAL;
5074 for (size_t idx = 0; idx < parts->size; ++idx) {
5075 pm_interpolated_part_flags(&flags, parts->nodes[idx]);
5076 }
5077
5078 return pm_interpolated_symbol_node_new(
5079 parser->arena,
5080 ++parser->node_id,
5081 flags,
5082 pm_interpolated_location(parser, opening, parts, closing),
5083 NTOK2LOC(parser, opening),
5084 *parts,
5085 NTOK2LOC(parser, closing)
5086 );
5087}
5088
5094static pm_interpolated_x_string_node_t *
5095pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_node_list_t *parts, const pm_token_t *closing) {
5096 pm_node_list_finalize(&parser->node_chunks, parser->arena, parts);
5097
5098 pm_node_flags_t flags = 0;
5099 for (size_t idx = 0; idx < parts->size; ++idx) {
5100 pm_interpolated_part_flags(&flags, parts->nodes[idx]);
5101 }
5102
5103 return pm_interpolated_x_string_node_new(
5104 parser->arena,
5105 ++parser->node_id,
5106 flags,
5107 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
5108 TOK2LOC(parser, opening),
5109 *parts,
5110 TOK2LOC(parser, closing)
5111 );
5112}
5113
5117static pm_it_local_variable_read_node_t *
5118pm_it_local_variable_read_node_create(pm_parser_t *parser, const pm_token_t *name) {
5119 return pm_it_local_variable_read_node_new(
5120 parser->arena,
5121 ++parser->node_id,
5122 0,
5123 PM_LOCATION_INIT_TOKEN(parser, name)
5124 );
5125}
5126
5130static pm_it_parameters_node_t *
5131pm_it_parameters_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *closing) {
5132 return pm_it_parameters_node_new(
5133 parser->arena,
5134 ++parser->node_id,
5135 0,
5136 PM_LOCATION_INIT_TOKENS(parser, opening, closing)
5137 );
5138}
5139
5143static pm_keyword_hash_node_t *
5144pm_keyword_hash_node_create(pm_parser_t *parser, pm_hash_elements_t *elements) {
5145 pm_node_list_t *nodes = &elements->nodes;
5146 pm_node_list_finalize(&parser->node_chunks, parser->arena, nodes);
5147
5148 uint32_t start = 0;
5149 uint32_t end = 0;
5150
5151 if (nodes->size > 0) {
5152 start = PM_NODE_START(nodes->nodes[0]);
5153 end = PM_NODE_END(nodes->nodes[nodes->size - 1]);
5154 }
5155
5156 return pm_keyword_hash_node_new(
5157 parser->arena,
5158 ++parser->node_id,
5159 elements->non_symbol_keys ? 0 : PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS,
5160 PM_LOCATION_INIT_BOUNDS(start, end),
5161 *nodes
5162 );
5163}
5164
5168static pm_required_keyword_parameter_node_t *
5169pm_required_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *name) {
5170 return pm_required_keyword_parameter_node_new(
5171 parser->arena,
5172 ++parser->node_id,
5173 0,
5174 PM_LOCATION_INIT_TOKEN(parser, name),
5175 pm_parser_constant_id_raw(parser, name->start, name->end - 1),
5176 TOK2LOC(parser, name)
5177 );
5178}
5179
5183static pm_optional_keyword_parameter_node_t *
5184pm_optional_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, pm_node_t *value) {
5185 return pm_optional_keyword_parameter_node_new(
5186 parser->arena,
5187 ++parser->node_id,
5188 0,
5189 PM_LOCATION_INIT_TOKEN_NODE(parser, name, value),
5190 pm_parser_constant_id_raw(parser, name->start, name->end - 1),
5191 TOK2LOC(parser, name),
5192 value
5193 );
5194}
5195
5199static pm_keyword_rest_parameter_node_t *
5200pm_keyword_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, const pm_token_t *name) {
5201 return pm_keyword_rest_parameter_node_new(
5202 parser->arena,
5203 ++parser->node_id,
5204 0,
5205 (name == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKENS(parser, operator, name),
5206 name == NULL ? 0 : pm_parser_constant_id_token(parser, name),
5207 NTOK2LOC(parser, name),
5208 TOK2LOC(parser, operator)
5209 );
5210}
5211
5215static pm_lambda_node_t *
5216pm_lambda_node_create(
5217 pm_parser_t *parser,
5218 pm_constant_id_list_t *locals,
5219 const pm_token_t *operator,
5220 const pm_token_t *opening,
5221 const pm_token_t *closing,
5222 pm_node_t *parameters,
5223 pm_node_t *body
5224) {
5225 return pm_lambda_node_new(
5226 parser->arena,
5227 ++parser->node_id,
5228 0,
5229 PM_LOCATION_INIT_TOKENS(parser, operator, closing),
5230 *locals,
5231 TOK2LOC(parser, operator),
5232 TOK2LOC(parser, opening),
5233 TOK2LOC(parser, closing),
5234 parameters,
5235 body
5236 );
5237}
5238
5242static pm_local_variable_and_write_node_t *
5243pm_local_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, const pm_token_t *operator, pm_node_t *value, pm_constant_id_t name, uint32_t depth) {
5244 PRISM_ASSERT(PM_NODE_TYPE_P(target, PM_LOCAL_VARIABLE_READ_NODE) || PM_NODE_TYPE_P(target, PM_IT_LOCAL_VARIABLE_READ_NODE) || PM_NODE_TYPE_P(target, PM_CALL_NODE));
5245 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
5246
5247 return pm_local_variable_and_write_node_new(
5248 parser->arena,
5249 ++parser->node_id,
5250 0,
5251 PM_LOCATION_INIT_NODES(target, value),
5252 target->location,
5253 TOK2LOC(parser, operator),
5254 value,
5255 name,
5256 depth
5257 );
5258}
5259
5263static pm_local_variable_operator_write_node_t *
5264pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *target, const pm_token_t *operator, pm_node_t *value, pm_constant_id_t name, uint32_t depth) {
5265 return pm_local_variable_operator_write_node_new(
5266 parser->arena,
5267 ++parser->node_id,
5268 0,
5269 PM_LOCATION_INIT_NODES(target, value),
5270 target->location,
5271 TOK2LOC(parser, operator),
5272 value,
5273 name,
5274 pm_parser_constant_id_raw(parser, operator->start, operator->end - 1),
5275 depth
5276 );
5277}
5278
5282static pm_local_variable_or_write_node_t *
5283pm_local_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, const pm_token_t *operator, pm_node_t *value, pm_constant_id_t name, uint32_t depth) {
5284 PRISM_ASSERT(PM_NODE_TYPE_P(target, PM_LOCAL_VARIABLE_READ_NODE) || PM_NODE_TYPE_P(target, PM_IT_LOCAL_VARIABLE_READ_NODE) || PM_NODE_TYPE_P(target, PM_CALL_NODE));
5285 PRISM_ASSERT(operator->type == PM_TOKEN_PIPE_PIPE_EQUAL);
5286
5287 return pm_local_variable_or_write_node_new(
5288 parser->arena,
5289 ++parser->node_id,
5290 0,
5291 PM_LOCATION_INIT_NODES(target, value),
5292 target->location,
5293 TOK2LOC(parser, operator),
5294 value,
5295 name,
5296 depth
5297 );
5298}
5299
5303static pm_local_variable_read_node_t *
5304pm_local_variable_read_node_create_constant_id(pm_parser_t *parser, const pm_token_t *name, pm_constant_id_t name_id, uint32_t depth, bool missing) {
5305 if (!missing) {
5306 PRISM_ASSERT(pm_parser_local_depth_constant_id(parser, name_id) == (int) depth);
5307 pm_locals_read(parser, name_id);
5308 }
5309
5310 return pm_local_variable_read_node_new(
5311 parser->arena,
5312 ++parser->node_id,
5313 0,
5314 PM_LOCATION_INIT_TOKEN(parser, name),
5315 name_id,
5316 depth
5317 );
5318}
5319
5323static pm_local_variable_read_node_t *
5324pm_local_variable_read_node_create(pm_parser_t *parser, const pm_token_t *name, uint32_t depth) {
5325 pm_constant_id_t name_id = pm_parser_constant_id_token(parser, name);
5326 return pm_local_variable_read_node_create_constant_id(parser, name, name_id, depth, false);
5327}
5328
5333static pm_local_variable_read_node_t *
5334pm_local_variable_read_node_missing_create(pm_parser_t *parser, const pm_token_t *name, uint32_t depth) {
5335 pm_constant_id_t name_id = pm_parser_constant_id_token(parser, name);
5336 return pm_local_variable_read_node_create_constant_id(parser, name, name_id, depth, true);
5337}
5338
5342static pm_local_variable_write_node_t *
5343pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, uint32_t depth, pm_node_t *value, const pm_location_t *name_loc, const pm_token_t *operator) {
5344 return pm_local_variable_write_node_new(
5345 parser->arena,
5346 ++parser->node_id,
5347 pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY),
5348 ((pm_location_t) { .start = name_loc->start, .length = PM_NODE_END(value) - name_loc->start }),
5349 name,
5350 depth,
5351 *name_loc,
5352 value,
5353 TOK2LOC(parser, operator)
5354 );
5355}
5356
5360static PRISM_INLINE bool
5361pm_token_is_it(const uint8_t *start, const uint8_t *end) {
5362 return (end - start == 2) && (start[0] == 'i') && (start[1] == 't');
5363}
5364
5369static PRISM_INLINE bool
5370pm_token_is_numbered_parameter(const pm_parser_t *parser, uint32_t start, uint32_t length) {
5371 return (
5372 (length == 2) &&
5373 (parser->start[start] == '_') &&
5374 (parser->start[start + 1] != '0') &&
5375 pm_char_is_decimal_digit(parser->start[start + 1])
5376 );
5377}
5378
5383static PRISM_INLINE void
5384pm_refute_numbered_parameter(pm_parser_t *parser, uint32_t start, uint32_t length) {
5385 if (pm_token_is_numbered_parameter(parser, start, length)) {
5386 PM_PARSER_ERR_FORMAT(parser, start, length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + start);
5387 }
5388}
5389
5394static pm_local_variable_target_node_t *
5395pm_local_variable_target_node_create(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) {
5396 pm_refute_numbered_parameter(parser, location->start, location->length);
5397
5398 return pm_local_variable_target_node_new(
5399 parser->arena,
5400 ++parser->node_id,
5401 0,
5402 ((pm_location_t) { .start = location->start, .length = location->length }),
5403 name,
5404 depth
5405 );
5406}
5407
5411static pm_match_predicate_node_t *
5412pm_match_predicate_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t *pattern, const pm_token_t *operator) {
5413 pm_assert_value_expression(parser, value);
5414
5415 return pm_match_predicate_node_new(
5416 parser->arena,
5417 ++parser->node_id,
5418 0,
5419 PM_LOCATION_INIT_NODES(value, pattern),
5420 value,
5421 pattern,
5422 TOK2LOC(parser, operator)
5423 );
5424}
5425
5429static pm_match_required_node_t *
5430pm_match_required_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t *pattern, const pm_token_t *operator) {
5431 pm_assert_value_expression(parser, value);
5432
5433 return pm_match_required_node_new(
5434 parser->arena,
5435 ++parser->node_id,
5436 0,
5437 PM_LOCATION_INIT_NODES(value, pattern),
5438 value,
5439 pattern,
5440 TOK2LOC(parser, operator)
5441 );
5442}
5443
5447static pm_match_write_node_t *
5448pm_match_write_node_create(pm_parser_t *parser, pm_call_node_t *call, pm_node_list_t *targets) {
5449 pm_node_list_finalize(&parser->node_chunks, parser->arena, targets);
5450
5451 return pm_match_write_node_new(
5452 parser->arena,
5453 ++parser->node_id,
5454 0,
5455 PM_LOCATION_INIT_NODE(call),
5456 call,
5457 *targets
5458 );
5459}
5460
5464static pm_module_node_t *
5465pm_module_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const pm_token_t *module_keyword, pm_node_t *constant_path, const pm_token_t *name, pm_node_t *body, const pm_token_t *end_keyword) {
5466 pm_constant_id_list_t module_locals = { .ids = NULL, .size = 0 };
5467 if (locals != NULL) module_locals = *locals;
5468
5469 return pm_module_node_new(
5470 parser->arena,
5471 ++parser->node_id,
5472 0,
5473 PM_LOCATION_INIT_TOKENS(parser, module_keyword, end_keyword),
5474 module_locals,
5475 TOK2LOC(parser, module_keyword),
5476 constant_path,
5477 body,
5478 TOK2LOC(parser, end_keyword),
5479 pm_parser_constant_id_token(parser, name)
5480 );
5481}
5482
5487typedef struct {
5488 /* The open list of every target, including the rests. */
5489 pm_node_list_t nodes;
5490
5491 /* The first splat or implicit rest, or NULL if there has not been one. */
5492 pm_node_t *rest;
5493
5494 /* The index of the first rest in nodes, which is only set with rest. */
5495 uint32_t rest_index;
5496} pm_multi_targets_t;
5497
5502static void
5503pm_multi_targets_append(pm_parser_t *parser, pm_multi_targets_t *targets, pm_node_t *target) {
5504 if (PM_NODE_TYPE_P(target, PM_SPLAT_NODE) || PM_NODE_TYPE_P(target, PM_IMPLICIT_REST_NODE)) {
5505 if (targets->rest == NULL) {
5506 targets->rest = target;
5507 targets->rest_index = targets->nodes.size;
5508 } else if (PM_NODE_TYPE_P(target, PM_SPLAT_NODE)) {
5509 pm_parser_err_node(parser, target, PM_ERR_MULTI_ASSIGN_MULTI_SPLATS);
5510 } else {
5511 PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST);
5512 }
5513 }
5514
5515 pm_node_list_open_append(&parser->node_chunks, &targets->nodes, target);
5516}
5517
5523static pm_multi_target_node_t *
5524pm_multi_target_node_create(pm_parser_t *parser, pm_multi_targets_t *targets, const pm_token_t *lparen, const pm_token_t *rparen) {
5525 pm_node_list_t *nodes = &targets->nodes;
5526 pm_node_list_finalize(&parser->node_chunks, parser->arena, nodes);
5527 PRISM_ASSERT(nodes->size > 0);
5528
5529 pm_node_list_t lefts = *nodes;
5530 pm_node_list_t rights = { 0 };
5531
5532 if (targets->rest != NULL) {
5533 lefts = pm_node_list_slice(nodes, 0, targets->rest_index);
5534 rights = pm_node_list_slice(nodes, targets->rest_index + 1, nodes->size - targets->rest_index - 1);
5535 }
5536
5537 pm_location_t location;
5538 if (lparen != NULL) {
5539 PRISM_ASSERT(rparen != NULL);
5540 location = PM_LOCATION_INIT_TOKENS(parser, lparen, rparen);
5541 } else {
5542 location = PM_LOCATION_INIT_NODES(nodes->nodes[0], nodes->nodes[nodes->size - 1]);
5543 }
5544
5545 return pm_multi_target_node_new(
5546 parser->arena,
5547 ++parser->node_id,
5548 0,
5549 location,
5550 lefts,
5551 targets->rest,
5552 rights,
5553 NTOK2LOC(parser, lparen),
5554 NTOK2LOC(parser, rparen)
5555 );
5556}
5557
5561static pm_multi_write_node_t *
5562pm_multi_write_node_create(pm_parser_t *parser, pm_multi_target_node_t *target, const pm_token_t *operator, pm_node_t *value) {
5563 /* The target is no longer necessary because we have reused its children. It
5564 * is arena-allocated so no explicit free is needed. */
5565 return pm_multi_write_node_new(
5566 parser->arena,
5567 ++parser->node_id,
5568 pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY),
5569 PM_LOCATION_INIT_NODES(target, value),
5570 target->lefts,
5571 target->rest,
5572 target->rights,
5573 target->lparen_loc,
5574 target->rparen_loc,
5575 TOK2LOC(parser, operator),
5576 value
5577 );
5578}
5579
5583static pm_next_node_t *
5584pm_next_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments_node_t *arguments) {
5585 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_NEXT);
5586
5587 return pm_next_node_new(
5588 parser->arena,
5589 ++parser->node_id,
5590 0,
5591 (arguments == NULL) ? PM_LOCATION_INIT_TOKEN(parser, keyword) : PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, arguments),
5592 arguments,
5593 TOK2LOC(parser, keyword)
5594 );
5595}
5596
5600static pm_nil_node_t *
5601pm_nil_node_create(pm_parser_t *parser, const pm_token_t *token) {
5602 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD_NIL);
5603
5604 return pm_nil_node_new(
5605 parser->arena,
5606 ++parser->node_id,
5607 PM_NODE_FLAG_STATIC_LITERAL,
5608 PM_LOCATION_INIT_TOKEN(parser, token)
5609 );
5610}
5611
5615static pm_no_block_parameter_node_t *
5616pm_no_block_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, const pm_token_t *keyword) {
5617 PRISM_ASSERT(operator->type == PM_TOKEN_AMPERSAND || operator->type == PM_TOKEN_UAMPERSAND);
5618 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_NIL);
5619
5620 return pm_no_block_parameter_node_new(
5621 parser->arena,
5622 ++parser->node_id,
5623 0,
5624 PM_LOCATION_INIT_TOKENS(parser, operator, keyword),
5625 TOK2LOC(parser, operator),
5626 TOK2LOC(parser, keyword)
5627 );
5628}
5629
5633static pm_no_keywords_parameter_node_t *
5634pm_no_keywords_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, const pm_token_t *keyword) {
5635 PRISM_ASSERT(operator->type == PM_TOKEN_USTAR_STAR || operator->type == PM_TOKEN_STAR_STAR);
5636 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_NIL);
5637
5638 return pm_no_keywords_parameter_node_new(
5639 parser->arena,
5640 ++parser->node_id,
5641 0,
5642 PM_LOCATION_INIT_TOKENS(parser, operator, keyword),
5643 TOK2LOC(parser, operator),
5644 TOK2LOC(parser, keyword)
5645 );
5646}
5647
5651static pm_numbered_parameters_node_t *
5652pm_numbered_parameters_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *closing, uint8_t maximum) {
5653 return pm_numbered_parameters_node_new(
5654 parser->arena,
5655 ++parser->node_id,
5656 0,
5657 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
5658 maximum
5659 );
5660}
5661
5668static uint32_t
5669pm_numbered_reference_read_node_number(pm_parser_t *parser, const pm_token_t *token) {
5674#define NTH_REF_MAX ((uint64_t) (uint32_t) (INT_MAX >> 1))
5675
5676 const uint8_t *start = token->start + 1;
5677 ptrdiff_t diff = token->end - start;
5678 PRISM_ASSERT(diff > 0);
5679
5680 size_t length = (size_t) diff;
5681 uint64_t value = 0;
5682
5683 for (size_t idx = 0; idx < length; ++idx) {
5684 value = value * 10 + (uint64_t) (start[idx] - '0');
5685
5686 if (value > NTH_REF_MAX) {
5687 PM_PARSER_WARN_FORMAT(parser, U32(start - parser->start), U32(length), PM_WARN_INVALID_NUMBERED_REFERENCE, (int) (length + 1), (const char *) token->start);
5688 value = 0;
5689 break;
5690 }
5691 }
5692
5693 return (uint32_t) value;
5694
5695#undef NTH_REF_MAX
5696}
5697
5701static pm_numbered_reference_read_node_t *
5702pm_numbered_reference_read_node_create(pm_parser_t *parser, const pm_token_t *name) {
5703 PRISM_ASSERT(name->type == PM_TOKEN_NUMBERED_REFERENCE);
5704
5705 return pm_numbered_reference_read_node_new(
5706 parser->arena,
5707 ++parser->node_id,
5708 0,
5709 PM_LOCATION_INIT_TOKEN(parser, name),
5710 pm_numbered_reference_read_node_number(parser, name)
5711 );
5712}
5713
5717static pm_optional_parameter_node_t *
5718pm_optional_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, const pm_token_t *operator, pm_node_t *value) {
5719 return pm_optional_parameter_node_new(
5720 parser->arena,
5721 ++parser->node_id,
5722 0,
5723 PM_LOCATION_INIT_TOKEN_NODE(parser, name, value),
5724 pm_parser_constant_id_token(parser, name),
5725 TOK2LOC(parser, name),
5726 TOK2LOC(parser, operator),
5727 value
5728 );
5729}
5730
5734static pm_or_node_t *
5735pm_or_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *operator, pm_node_t *right) {
5736 pm_assert_value_expression(parser, left);
5737
5738 return pm_or_node_new(
5739 parser->arena,
5740 ++parser->node_id,
5741 0,
5742 PM_LOCATION_INIT_NODES(left, right),
5743 left,
5744 right,
5745 TOK2LOC(parser, operator)
5746 );
5747}
5748
5753typedef struct {
5754 /* The location covering every parameter added so far. */
5755 pm_location_t location;
5756
5757 /* The required parameters before any optional or rest parameter. */
5758 pm_node_list_t requireds;
5759
5760 /* The optional parameters. */
5761 pm_node_list_t optionals;
5762
5763 /* The rest parameter, or NULL if there is not one. */
5764 pm_node_t *rest;
5765
5766 /* The required parameters after an optional or rest parameter. */
5767 pm_node_list_t posts;
5768
5769 /* The keyword parameters. */
5770 pm_node_list_t keywords;
5771
5772 /* The keyword rest parameter, or NULL if there is not one. */
5773 pm_node_t *keyword_rest;
5774
5775 /* The block parameter, or NULL if there is not one. */
5776 pm_node_t *block;
5777} pm_parameters_t;
5778
5784static void
5785pm_parameters_location_extend(pm_parameters_t *params, const pm_node_t *param) {
5786 params->location = params->location.length == 0
5787 ? param->location
5788 : pm_location_join(params->location, param->location);
5789}
5790
5794static void
5795pm_parameters_requireds_append(pm_parser_t *parser, pm_parameters_t *params, pm_node_t *param) {
5796 pm_parameters_location_extend(params, param);
5797 pm_node_list_open_append(&parser->node_chunks, &params->requireds, param);
5798}
5799
5803static void
5804pm_parameters_optionals_append(pm_parser_t *parser, pm_parameters_t *params, pm_optional_parameter_node_t *param) {
5805 pm_parameters_location_extend(params, UP(param));
5806 pm_node_list_open_append(&parser->node_chunks, &params->optionals, UP(param));
5807}
5808
5812static void
5813pm_parameters_posts_append(pm_parser_t *parser, pm_parameters_t *params, pm_node_t *param) {
5814 pm_parameters_location_extend(params, param);
5815 pm_node_list_open_append(&parser->node_chunks, &params->posts, param);
5816}
5817
5821static void
5822pm_parameters_rest_set(pm_parameters_t *params, pm_node_t *param) {
5823 pm_parameters_location_extend(params, param);
5824 params->rest = param;
5825}
5826
5830static void
5831pm_parameters_keywords_append(pm_parser_t *parser, pm_parameters_t *params, pm_node_t *param) {
5832 pm_parameters_location_extend(params, param);
5833 pm_node_list_open_append(&parser->node_chunks, &params->keywords, param);
5834}
5835
5839static void
5840pm_parameters_keyword_rest_set(pm_parameters_t *params, pm_node_t *param) {
5841 PRISM_ASSERT(params->keyword_rest == NULL);
5842 pm_parameters_location_extend(params, param);
5843 params->keyword_rest = param;
5844}
5845
5849static void
5850pm_parameters_block_set(pm_parameters_t *params, pm_node_t *param) {
5851 PRISM_ASSERT(params->block == NULL);
5852 pm_parameters_location_extend(params, param);
5853 params->block = param;
5854}
5855
5859static pm_parameters_node_t *
5860pm_parameters_node_create(pm_parser_t *parser, pm_parameters_t *params) {
5861 pm_node_list_finalize(&parser->node_chunks, parser->arena, &params->requireds);
5862 pm_node_list_finalize(&parser->node_chunks, parser->arena, &params->optionals);
5863 pm_node_list_finalize(&parser->node_chunks, parser->arena, &params->posts);
5864 pm_node_list_finalize(&parser->node_chunks, parser->arena, &params->keywords);
5865
5866 return pm_parameters_node_new(
5867 parser->arena,
5868 ++parser->node_id,
5869 0,
5870 params->location,
5871 params->requireds,
5872 params->optionals,
5873 params->rest,
5874 params->posts,
5875 params->keywords,
5876 params->keyword_rest,
5877 params->block
5878 );
5879}
5880
5884static pm_program_node_t *
5885pm_program_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, pm_statements_node_t *statements) {
5886 return pm_program_node_new(
5887 parser->arena,
5888 ++parser->node_id,
5889 0,
5890 PM_LOCATION_INIT_NODE(statements),
5891 *locals,
5892 statements
5893 );
5894}
5895
5899static pm_parentheses_node_t *
5900pm_parentheses_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_node_t *body, const pm_token_t *closing, pm_node_flags_t flags) {
5901 return pm_parentheses_node_new(
5902 parser->arena,
5903 ++parser->node_id,
5904 flags,
5905 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
5906 body,
5907 TOK2LOC(parser, opening),
5908 TOK2LOC(parser, closing)
5909 );
5910}
5911
5915static pm_pinned_expression_node_t *
5916pm_pinned_expression_node_create(pm_parser_t *parser, pm_node_t *expression, const pm_token_t *operator, const pm_token_t *lparen, const pm_token_t *rparen) {
5917 return pm_pinned_expression_node_new(
5918 parser->arena,
5919 ++parser->node_id,
5920 0,
5921 PM_LOCATION_INIT_TOKENS(parser, operator, rparen),
5922 expression,
5923 TOK2LOC(parser, operator),
5924 TOK2LOC(parser, lparen),
5925 TOK2LOC(parser, rparen)
5926 );
5927}
5928
5932static pm_pinned_variable_node_t *
5933pm_pinned_variable_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t *variable) {
5934 return pm_pinned_variable_node_new(
5935 parser->arena,
5936 ++parser->node_id,
5937 0,
5938 PM_LOCATION_INIT_TOKEN_NODE(parser, operator, variable),
5939 variable,
5940 TOK2LOC(parser, operator)
5941 );
5942}
5943
5947static pm_post_execution_node_t *
5948pm_post_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_token_t *opening, pm_statements_node_t *statements, const pm_token_t *closing) {
5949 return pm_post_execution_node_new(
5950 parser->arena,
5951 ++parser->node_id,
5952 0,
5953 PM_LOCATION_INIT_TOKENS(parser, keyword, closing),
5954 statements,
5955 TOK2LOC(parser, keyword),
5956 TOK2LOC(parser, opening),
5957 TOK2LOC(parser, closing)
5958 );
5959}
5960
5964static pm_pre_execution_node_t *
5965pm_pre_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_token_t *opening, pm_statements_node_t *statements, const pm_token_t *closing) {
5966 return pm_pre_execution_node_new(
5967 parser->arena,
5968 ++parser->node_id,
5969 0,
5970 PM_LOCATION_INIT_TOKENS(parser, keyword, closing),
5971 statements,
5972 TOK2LOC(parser, keyword),
5973 TOK2LOC(parser, opening),
5974 TOK2LOC(parser, closing)
5975 );
5976}
5977
5981static pm_range_node_t *
5982pm_range_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *operator, pm_node_t *right) {
5983 pm_assert_value_expression(parser, left);
5984 pm_assert_value_expression(parser, right);
5985 pm_node_flags_t flags = 0;
5986
5987 // Indicate that this node is an exclusive range if the operator is `...`.
5988 if (operator->type == PM_TOKEN_DOT_DOT_DOT || operator->type == PM_TOKEN_UDOT_DOT_DOT) {
5989 flags |= PM_RANGE_FLAGS_EXCLUDE_END;
5990 }
5991
5992 // Indicate that this node is a static literal (i.e., can be compiled with
5993 // a putobject in CRuby) if the left and right are implicit nil, explicit
5994 // nil, or integers.
5995 if (
5996 (left == NULL || PM_NODE_TYPE_P(left, PM_NIL_NODE) || PM_NODE_TYPE_P(left, PM_INTEGER_NODE)) &&
5997 (right == NULL || PM_NODE_TYPE_P(right, PM_NIL_NODE) || PM_NODE_TYPE_P(right, PM_INTEGER_NODE))
5998 ) {
5999 flags |= PM_NODE_FLAG_STATIC_LITERAL;
6000 }
6001
6002 uint32_t start = left == NULL ? PM_TOKEN_START(parser, operator) : PM_NODE_START(left);
6003 uint32_t end = right == NULL ? PM_TOKEN_END(parser, operator) : PM_NODE_END(right);
6004
6005 return pm_range_node_new(
6006 parser->arena,
6007 ++parser->node_id,
6008 flags,
6009 ((pm_location_t) { .start = start, .length = U32(end - start) }),
6010 left,
6011 right,
6012 TOK2LOC(parser, operator)
6013 );
6014}
6015
6019static pm_redo_node_t *
6020pm_redo_node_create(pm_parser_t *parser, const pm_token_t *token) {
6021 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD_REDO);
6022
6023 return pm_redo_node_new(
6024 parser->arena,
6025 ++parser->node_id,
6026 0,
6027 PM_LOCATION_INIT_TOKEN(parser, token)
6028 );
6029}
6030
6035static pm_regular_expression_node_t *
6036pm_regular_expression_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *content, const pm_token_t *closing, const pm_string_t *unescaped) {
6037 return pm_regular_expression_node_new(
6038 parser->arena,
6039 ++parser->node_id,
6040 pm_regular_expression_flags_create(parser, closing) | PM_NODE_FLAG_STATIC_LITERAL,
6041 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
6042 TOK2LOC(parser, opening),
6043 TOK2LOC(parser, content),
6044 TOK2LOC(parser, closing),
6045 *unescaped
6046 );
6047}
6048
6052static PRISM_INLINE pm_regular_expression_node_t *
6053pm_regular_expression_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *content, const pm_token_t *closing) {
6054 return pm_regular_expression_node_create_unescaped(parser, opening, content, closing, &PM_STRING_EMPTY);
6055}
6056
6060static pm_required_parameter_node_t *
6061pm_required_parameter_node_create(pm_parser_t *parser, const pm_token_t *token) {
6062 return pm_required_parameter_node_new(
6063 parser->arena,
6064 ++parser->node_id,
6065 0,
6066 PM_LOCATION_INIT_TOKEN(parser, token),
6067 pm_parser_constant_id_token(parser, token)
6068 );
6069}
6070
6074static pm_rescue_modifier_node_t *
6075pm_rescue_modifier_node_create(pm_parser_t *parser, pm_node_t *expression, const pm_token_t *keyword, pm_node_t *rescue_expression) {
6076 return pm_rescue_modifier_node_new(
6077 parser->arena,
6078 ++parser->node_id,
6079 0,
6080 PM_LOCATION_INIT_NODES(expression, rescue_expression),
6081 expression,
6082 TOK2LOC(parser, keyword),
6083 rescue_expression
6084 );
6085}
6086
6090static pm_rescue_node_t *
6091pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_list_t *exceptions, const pm_token_t *operator, pm_node_t *reference, const pm_token_t *then_keyword, pm_statements_node_t *statements) {
6092 pm_node_list_finalize(&parser->node_chunks, parser->arena, exceptions);
6093
6094 uint32_t start = PM_TOKEN_START(parser, keyword);
6095 uint32_t end = PM_TOKEN_END(parser, keyword);
6096
6097 if (exceptions->size > 0) end = PM_NODE_END(exceptions->nodes[exceptions->size - 1]);
6098 if (reference != NULL) end = PM_NODE_END(reference);
6099 if (statements != NULL && pm_statements_node_body_length(statements) > 0) end = PM_NODE_END(statements);
6100
6101 return pm_rescue_node_new(
6102 parser->arena,
6103 ++parser->node_id,
6104 0,
6105 PM_LOCATION_INIT_BOUNDS(start, end),
6106 TOK2LOC(parser, keyword),
6107 *exceptions,
6108 NTOK2LOC(parser, operator),
6109 reference,
6110 NTOK2LOC(parser, then_keyword),
6111 statements,
6112 NULL
6113 );
6114}
6115
6119static void
6120pm_rescue_node_subsequent_set(pm_rescue_node_t *node, pm_rescue_node_t *subsequent) {
6121 node->subsequent = subsequent;
6122 PM_NODE_LENGTH_SET_NODE(node, subsequent);
6123}
6124
6128static pm_rest_parameter_node_t *
6129pm_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, const pm_token_t *name) {
6130 return pm_rest_parameter_node_new(
6131 parser->arena,
6132 ++parser->node_id,
6133 0,
6134 (name == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKENS(parser, operator, name),
6135 name == NULL ? 0 : pm_parser_constant_id_token(parser, name),
6136 NTOK2LOC(parser, name),
6137 TOK2LOC(parser, operator)
6138 );
6139}
6140
6144static pm_retry_node_t *
6145pm_retry_node_create(pm_parser_t *parser, const pm_token_t *token) {
6146 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD_RETRY);
6147
6148 return pm_retry_node_new(
6149 parser->arena,
6150 ++parser->node_id,
6151 0,
6152 PM_LOCATION_INIT_TOKEN(parser, token)
6153 );
6154}
6155
6159static pm_return_node_t *
6160pm_return_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments_node_t *arguments) {
6161 return pm_return_node_new(
6162 parser->arena,
6163 ++parser->node_id,
6164 0,
6165 (arguments == NULL) ? PM_LOCATION_INIT_TOKEN(parser, keyword) : PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, arguments),
6166 TOK2LOC(parser, keyword),
6167 arguments
6168 );
6169}
6170
6174static pm_self_node_t *
6175pm_self_node_create(pm_parser_t *parser, const pm_token_t *token) {
6176 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD_SELF);
6177
6178 return pm_self_node_new(
6179 parser->arena,
6180 ++parser->node_id,
6181 0,
6182 PM_LOCATION_INIT_TOKEN(parser, token)
6183 );
6184}
6185
6189static pm_shareable_constant_node_t *
6190pm_shareable_constant_node_create(pm_parser_t *parser, pm_node_t *write, pm_shareable_constant_value_t value) {
6191 return pm_shareable_constant_node_new(
6192 parser->arena,
6193 ++parser->node_id,
6194 (pm_node_flags_t) value,
6195 PM_LOCATION_INIT_NODE(write),
6196 write
6197 );
6198}
6199
6203static pm_singleton_class_node_t *
6204pm_singleton_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const pm_token_t *class_keyword, const pm_token_t *operator, pm_node_t *expression, pm_node_t *body, const pm_token_t *end_keyword) {
6205 return pm_singleton_class_node_new(
6206 parser->arena,
6207 ++parser->node_id,
6208 0,
6209 PM_LOCATION_INIT_TOKENS(parser, class_keyword, end_keyword),
6210 *locals,
6211 TOK2LOC(parser, class_keyword),
6212 TOK2LOC(parser, operator),
6213 expression,
6214 body,
6215 TOK2LOC(parser, end_keyword)
6216 );
6217}
6218
6222static pm_source_encoding_node_t *
6223pm_source_encoding_node_create(pm_parser_t *parser, const pm_token_t *token) {
6224 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD___ENCODING__);
6225
6226 return pm_source_encoding_node_new(
6227 parser->arena,
6228 ++parser->node_id,
6229 PM_NODE_FLAG_STATIC_LITERAL,
6230 PM_LOCATION_INIT_TOKEN(parser, token)
6231 );
6232}
6233
6237static pm_source_file_node_t*
6238pm_source_file_node_create(pm_parser_t *parser, const pm_token_t *file_keyword) {
6239 PRISM_ASSERT(file_keyword->type == PM_TOKEN_KEYWORD___FILE__);
6240
6241 pm_node_flags_t flags = 0;
6242
6243 switch (parser->frozen_string_literal) {
6244 case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED:
6245 flags |= PM_STRING_FLAGS_MUTABLE;
6246 break;
6247 case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED:
6248 flags |= PM_STRING_FLAGS_FROZEN;
6249 if (parser->version < PM_OPTIONS_VERSION_CRUBY_4_1) {
6250 flags |= PM_NODE_FLAG_STATIC_LITERAL;
6251 }
6252 break;
6253 }
6254
6255 return pm_source_file_node_new(
6256 parser->arena,
6257 ++parser->node_id,
6258 flags,
6259 PM_LOCATION_INIT_TOKEN(parser, file_keyword),
6260 parser->filepath
6261 );
6262}
6263
6267static pm_source_line_node_t *
6268pm_source_line_node_create(pm_parser_t *parser, const pm_token_t *token) {
6269 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD___LINE__);
6270
6271 return pm_source_line_node_new(
6272 parser->arena,
6273 ++parser->node_id,
6274 PM_NODE_FLAG_STATIC_LITERAL,
6275 PM_LOCATION_INIT_TOKEN(parser, token)
6276 );
6277}
6278
6282static pm_splat_node_t *
6283pm_splat_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t *expression) {
6284 return pm_splat_node_new(
6285 parser->arena,
6286 ++parser->node_id,
6287 0,
6288 (expression == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKEN_NODE(parser, operator, expression),
6289 TOK2LOC(parser, operator),
6290 expression
6291 );
6292}
6293
6297static void
6298pm_statements_unreachable_check(pm_parser_t *parser, const pm_node_t *previous, pm_node_t *statement) {
6299 switch (PM_NODE_TYPE(previous)) {
6300 case PM_BREAK_NODE:
6301 case PM_NEXT_NODE:
6302 case PM_REDO_NODE:
6303 case PM_RETRY_NODE:
6304 case PM_RETURN_NODE:
6305 pm_parser_warn_node(parser, statement, PM_WARN_UNREACHABLE_STATEMENT);
6306 break;
6307 default:
6308 break;
6309 }
6310}
6311
6315typedef struct {
6317 pm_node_list_t nodes;
6318
6324 pm_location_t location;
6325
6327 pm_node_flags_t flags;
6328} pm_statements_t;
6329
6334static void
6335pm_statements_body_append(pm_parser_t *parser, pm_statements_t *body, pm_node_t *statement, bool newline) {
6336 body->location = body->nodes.size == 0
6337 ? statement->location
6338 : pm_location_join(body->location, statement->location);
6339
6340 if (body->nodes.size > 0) pm_statements_unreachable_check(parser, pm_node_list_open_last(&body->nodes), statement);
6341 pm_node_list_open_append(&parser->node_chunks, &body->nodes, statement);
6342 if (PM_NODE_TYPE_P(statement, PM_PRE_EXECUTION_NODE)) body->flags |= PM_STATEMENTS_FLAGS_CONTAINS_PRE_EXECUTION;
6343 if (newline) pm_node_flag_set(statement, PM_NODE_FLAG_NEWLINE);
6344}
6345
6349static pm_statements_node_t *
6350pm_statements_node_create(pm_parser_t *parser, pm_statements_t *body) {
6351 pm_node_list_finalize(&parser->node_chunks, parser->arena, &body->nodes);
6352
6353 return pm_statements_node_new(
6354 parser->arena,
6355 ++parser->node_id,
6356 body->flags,
6357 body->location,
6358 body->nodes
6359 );
6360}
6361
6365static pm_statements_node_t *
6366pm_statements_node_single_create(pm_parser_t *parser, pm_node_t *statement, bool newline) {
6367 if (newline) pm_node_flag_set(statement, PM_NODE_FLAG_NEWLINE);
6368
6369 return pm_statements_node_new(
6370 parser->arena,
6371 ++parser->node_id,
6372 PM_NODE_TYPE_P(statement, PM_PRE_EXECUTION_NODE) ? PM_STATEMENTS_FLAGS_CONTAINS_PRE_EXECUTION : 0,
6373 PM_LOCATION_INIT_NODE(statement),
6374 pm_node_list_single(parser->arena, statement)
6375 );
6376}
6377
6381static size_t
6382pm_statements_node_body_length(pm_statements_node_t *node) {
6383 return node && node->body.size;
6384}
6385
6390static PRISM_INLINE void
6391pm_statements_node_body_update(pm_statements_node_t *node, pm_node_t *statement) {
6392 node->base.location = pm_statements_node_body_length(node) == 0
6393 ? statement->location
6394 : pm_location_join(node->base.location, statement->location);
6395 if (PM_NODE_TYPE_P(statement, PM_PRE_EXECUTION_NODE)) pm_node_flag_set(UP(node), PM_STATEMENTS_FLAGS_CONTAINS_PRE_EXECUTION);
6396}
6397
6401static void
6402pm_statements_node_body_append(pm_parser_t *parser, pm_statements_node_t *node, pm_node_t *statement) {
6403 pm_statements_node_body_update(node, statement);
6404 if (node->body.size > 0) pm_statements_unreachable_check(parser, node->body.nodes[node->body.size - 1], statement);
6405
6406 pm_node_list_append(parser->arena, &node->body, statement);
6407 pm_node_flag_set(statement, PM_NODE_FLAG_NEWLINE);
6408}
6409
6413static void
6414pm_statements_node_body_prepend(pm_arena_t *arena, pm_statements_node_t *node, pm_node_t *statement) {
6415 pm_statements_node_body_update(node, statement);
6416 pm_node_list_prepend_finalized(arena, &node->body, statement);
6417 pm_node_flag_set(statement, PM_NODE_FLAG_NEWLINE);
6418}
6419
6423static PRISM_INLINE pm_string_node_t *
6424pm_string_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *content, const pm_token_t *closing, const pm_string_t *string) {
6425 pm_node_flags_t flags = 0;
6426
6427 switch (parser->frozen_string_literal) {
6428 case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED:
6429 flags = PM_STRING_FLAGS_MUTABLE;
6430 break;
6431 case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED:
6432 flags = PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN;
6433 break;
6434 }
6435
6436 uint32_t start = PM_TOKEN_START(parser, opening == NULL ? content : opening);
6437 uint32_t end = PM_TOKEN_END(parser, closing == NULL ? content : closing);
6438
6439 return pm_string_node_new(
6440 parser->arena,
6441 ++parser->node_id,
6442 flags,
6443 ((pm_location_t) { .start = start, .length = U32(end - start) }),
6444 NTOK2LOC(parser, opening),
6445 TOK2LOC(parser, content),
6446 NTOK2LOC(parser, closing),
6447 *string
6448 );
6449}
6450
6454static pm_string_node_t *
6455pm_string_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *content, const pm_token_t *closing) {
6456 return pm_string_node_create_unescaped(parser, opening, content, closing, &PM_STRING_EMPTY);
6457}
6458
6463static pm_string_node_t *
6464pm_string_node_create_current_string(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *content, const pm_token_t *closing) {
6465 pm_string_node_t *node = pm_string_node_create_unescaped(parser, opening, content, closing, &parser->current_string);
6466 parser->current_string = PM_STRING_EMPTY;
6467 return node;
6468}
6469
6473static pm_super_node_t *
6474pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments_t *arguments) {
6475 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_SUPER);
6476
6477 const pm_location_t *end = pm_arguments_end(arguments);
6478 PRISM_ASSERT(end != NULL && "unreachable");
6479
6480 return pm_super_node_new(
6481 parser->arena,
6482 ++parser->node_id,
6483 0,
6484 ((pm_location_t) { .start = PM_TOKEN_START(parser, keyword), .length = PM_LOCATION_END(end) - PM_TOKEN_START(parser, keyword) }),
6485 TOK2LOC(parser, keyword),
6486 arguments->opening_loc,
6487 pm_arguments_node_create(parser, arguments),
6488 arguments->closing_loc,
6489 arguments->block
6490 );
6491}
6492
6496static void
6497parse_symbol_encoding_validate_utf8(pm_parser_t *parser, const pm_token_t *location, const pm_string_t *contents) {
6498 for (const uint8_t *cursor = pm_string_source(contents), *end = cursor + pm_string_length(contents); cursor < end;) {
6499 size_t width = pm_encoding_utf_8_char_width(cursor, end - cursor);
6500
6501 if (width == 0) {
6502 pm_parser_err(parser, PM_TOKEN_START(parser, location), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL);
6503 break;
6504 }
6505
6506 cursor += width;
6507 }
6508}
6509
6514static void
6515parse_symbol_encoding_validate_other(pm_parser_t *parser, const pm_token_t *location, const pm_string_t *contents) {
6516 const pm_encoding_t *encoding = parser->encoding;
6517
6518 for (const uint8_t *cursor = pm_string_source(contents), *end = cursor + pm_string_length(contents); cursor < end;) {
6519 size_t width = pm_encoding_char_width(encoding, cursor, end - cursor);
6520
6521 if (width == 0) {
6522 pm_parser_err(parser, PM_TOKEN_START(parser, location), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL);
6523 break;
6524 }
6525
6526 cursor += width;
6527 }
6528}
6529
6539static PRISM_INLINE pm_node_flags_t
6540parse_symbol_encoding(pm_parser_t *parser, const pm_encoding_t *explicit_encoding, const pm_token_t *location, const pm_string_t *contents, bool validate) {
6541 if (explicit_encoding != NULL) {
6542 // A Symbol may optionally have its encoding explicitly set. This will
6543 // happen if an escape sequence results in a non-ASCII code point.
6544 if (explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
6545 if (validate) parse_symbol_encoding_validate_utf8(parser, location, contents);
6546 return PM_SYMBOL_FLAGS_FORCED_UTF8_ENCODING;
6547 } else if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
6548 return PM_SYMBOL_FLAGS_FORCED_BINARY_ENCODING;
6549 } else if (validate) {
6550 parse_symbol_encoding_validate_other(parser, location, contents);
6551 }
6552 } else if (pm_string_ascii_only_p(contents)) {
6553 // Ruby stipulates that all source files must use an ASCII-compatible
6554 // encoding. Thus, all symbols appearing in source are eligible for
6555 // "downgrading" to US-ASCII.
6556 return PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING;
6557 } else if (validate) {
6558 parse_symbol_encoding_validate_other(parser, location, contents);
6559 }
6560
6561 return 0;
6562}
6563
6568static pm_symbol_node_t *
6569pm_symbol_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *value, const pm_token_t *closing, const pm_string_t *unescaped, pm_node_flags_t flags) {
6570 uint32_t start = opening == NULL ? PM_TOKEN_START(parser, value) : PM_TOKEN_START(parser, opening);
6571 uint32_t end = closing == NULL ? PM_TOKEN_END(parser, value) : PM_TOKEN_END(parser, closing);
6572
6573 return pm_symbol_node_new(
6574 parser->arena,
6575 ++parser->node_id,
6576 PM_NODE_FLAG_STATIC_LITERAL | flags,
6577 ((pm_location_t) { .start = start, .length = U32(end - start) }),
6578 NTOK2LOC(parser, opening),
6579 NTOK2LOC(parser, value),
6580 NTOK2LOC(parser, closing),
6581 *unescaped
6582 );
6583}
6584
6588static PRISM_INLINE pm_symbol_node_t *
6589pm_symbol_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *value, const pm_token_t *closing) {
6590 return pm_symbol_node_create_unescaped(parser, opening, value, closing, &PM_STRING_EMPTY, 0);
6591}
6592
6596static pm_symbol_node_t *
6597pm_symbol_node_create_current_string(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *value, const pm_token_t *closing) {
6598 pm_symbol_node_t *node = pm_symbol_node_create_unescaped(parser, opening, value, closing, &parser->current_string, parse_symbol_encoding(parser, parser->explicit_encoding, value, &parser->current_string, false));
6599 parser->current_string = PM_STRING_EMPTY;
6600 return node;
6601}
6602
6606static pm_symbol_node_t *
6607pm_symbol_node_label_create(pm_parser_t *parser, const pm_token_t *token) {
6608 PRISM_ASSERT(token->type == PM_TOKEN_LABEL);
6609
6610 pm_token_t closing = { .type = PM_TOKEN_LABEL_END, .start = token->end - 1, .end = token->end };
6611 pm_token_t label = { .type = PM_TOKEN_LABEL, .start = token->start, .end = token->end - 1 };
6612 pm_symbol_node_t *node = pm_symbol_node_create(parser, NULL, &label, &closing);
6613
6614 PRISM_ASSERT((label.end - label.start) >= 0);
6615 pm_string_shared_init(&node->unescaped, label.start, label.end);
6616 pm_node_flag_set(UP(node), parse_symbol_encoding(parser, parser->explicit_encoding, &label, &node->unescaped, false));
6617
6618 return node;
6619}
6620
6624static pm_symbol_node_t *
6625pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) {
6626 pm_symbol_node_t *node = pm_symbol_node_new(
6627 parser->arena,
6628 ++parser->node_id,
6629 PM_NODE_FLAG_STATIC_LITERAL | PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING,
6630 PM_LOCATION_INIT_UNSET,
6631 ((pm_location_t) { 0 }),
6632 ((pm_location_t) { 0 }),
6633 ((pm_location_t) { 0 }),
6634 ((pm_string_t) { 0 })
6635 );
6636
6637 pm_string_init(&node->unescaped, (const uint8_t *) content, strlen(content));
6638 return node;
6639}
6640
6644static bool
6645pm_symbol_node_label_p(const pm_parser_t *parser, const pm_node_t *node) {
6646 const pm_location_t *closing;
6647 uint32_t expected_length;
6648
6649 switch (PM_NODE_TYPE(node)) {
6650 case PM_SYMBOL_NODE: {
6651 const pm_symbol_node_t *cast = (pm_symbol_node_t *) node;
6652 closing = &cast->closing_loc;
6653 expected_length = cast->opening_loc.length == 0 ? 1 : 2;
6654 break;
6655 }
6656 case PM_INTERPOLATED_SYMBOL_NODE: {
6657 const pm_interpolated_symbol_node_t *cast = (pm_interpolated_symbol_node_t *) node;
6658 closing = &cast->closing_loc;
6659 expected_length = 2;
6660 break;
6661 }
6662 default:
6663 return false;
6664 }
6665
6666 /* An unquoted label ends with a colon alone. A quoted label ends with
6667 * its closing quote followed by a colon. */
6668 return closing->length == expected_length && parser->start[PM_LOCATION_END(closing) - 1] == ':';
6669}
6670
6674static pm_symbol_node_t *
6675pm_string_node_to_symbol_node(pm_parser_t *parser, pm_string_node_t *node, const pm_token_t *opening, const pm_token_t *closing) {
6676 pm_symbol_node_t *new_node = pm_symbol_node_new(
6677 parser->arena,
6678 ++parser->node_id,
6679 PM_NODE_FLAG_STATIC_LITERAL,
6680 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
6681 TOK2LOC(parser, opening),
6682 node->content_loc,
6683 TOK2LOC(parser, closing),
6684 node->unescaped
6685 );
6686
6687 pm_token_t content = {
6688 .type = PM_TOKEN_IDENTIFIER,
6689 .start = parser->start + node->content_loc.start,
6690 .end = parser->start + node->content_loc.start + node->content_loc.length
6691 };
6692
6693 pm_node_flag_set(UP(new_node), parse_symbol_encoding(parser, parser->explicit_encoding, &content, &node->unescaped, true));
6694
6695 /* The old node is arena-allocated so no explicit free is needed. */
6696 return new_node;
6697}
6698
6702static pm_string_node_t *
6703pm_symbol_node_to_string_node(pm_parser_t *parser, pm_symbol_node_t *node) {
6704 pm_node_flags_t flags = 0;
6705
6706 switch (parser->frozen_string_literal) {
6707 case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED:
6708 flags = PM_STRING_FLAGS_MUTABLE;
6709 break;
6710 case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED:
6711 flags = PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN;
6712 break;
6713 }
6714
6715 pm_string_node_t *new_node = pm_string_node_new(
6716 parser->arena,
6717 ++parser->node_id,
6718 flags,
6719 PM_LOCATION_INIT_NODE(node),
6720 node->opening_loc,
6721 node->content_loc,
6722 node->closing_loc,
6723 node->unescaped
6724 );
6725
6726 /* The old node is arena-allocated so no explicit free is needed. */
6727 return new_node;
6728}
6729
6733static pm_true_node_t *
6734pm_true_node_create(pm_parser_t *parser, const pm_token_t *token) {
6735 PRISM_ASSERT(token->type == PM_TOKEN_KEYWORD_TRUE);
6736
6737 return pm_true_node_new(
6738 parser->arena,
6739 ++parser->node_id,
6740 PM_NODE_FLAG_STATIC_LITERAL,
6741 PM_LOCATION_INIT_TOKEN(parser, token)
6742 );
6743}
6744
6748static pm_true_node_t *
6749pm_true_node_synthesized_create(pm_parser_t *parser) {
6750 return pm_true_node_new(
6751 parser->arena,
6752 ++parser->node_id,
6753 PM_NODE_FLAG_STATIC_LITERAL,
6754 PM_LOCATION_INIT_UNSET
6755 );
6756}
6757
6761static pm_undef_node_t *
6762pm_undef_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_list_t *names) {
6763 PRISM_ASSERT(keyword->type == PM_TOKEN_KEYWORD_UNDEF);
6764 pm_node_list_finalize(&parser->node_chunks, parser->arena, names);
6765
6766 return pm_undef_node_new(
6767 parser->arena,
6768 ++parser->node_id,
6769 0,
6770 names->size == 0 ? PM_LOCATION_INIT_TOKEN(parser, keyword) : PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, names->nodes[names->size - 1]),
6771 *names,
6772 TOK2LOC(parser, keyword)
6773 );
6774}
6775
6779static pm_unless_node_t *
6780pm_unless_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *predicate, const pm_token_t *then_keyword, pm_statements_node_t *statements) {
6781 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
6782 pm_node_t *end = statements == NULL ? predicate : UP(statements);
6783
6784 return pm_unless_node_new(
6785 parser->arena,
6786 ++parser->node_id,
6787 PM_NODE_FLAG_NEWLINE,
6788 PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, end),
6789 TOK2LOC(parser, keyword),
6790 predicate,
6791 NTOK2LOC(parser, then_keyword),
6792 statements,
6793 NULL,
6794 ((pm_location_t) { 0 })
6795 );
6796}
6797
6801static pm_unless_node_t *
6802pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const pm_token_t *unless_keyword, pm_node_t *predicate) {
6803 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
6804
6805 pm_statements_node_t *statements = pm_statements_node_single_create(parser, statement, true);
6806
6807 return pm_unless_node_new(
6808 parser->arena,
6809 ++parser->node_id,
6810 PM_NODE_FLAG_NEWLINE,
6811 PM_LOCATION_INIT_NODES(statement, predicate),
6812 TOK2LOC(parser, unless_keyword),
6813 predicate,
6814 ((pm_location_t) { 0 }),
6815 statements,
6816 NULL,
6817 ((pm_location_t) { 0 })
6818 );
6819}
6820
6821static PRISM_INLINE void
6822pm_unless_node_end_keyword_loc_set(const pm_parser_t *parser, pm_unless_node_t *node, const pm_token_t *end_keyword) {
6823 node->end_keyword_loc = TOK2LOC(parser, end_keyword);
6824 PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword);
6825}
6826
6832static void
6833pm_loop_modifier_block_exits(pm_parser_t *parser, pm_statements_node_t *statements) {
6834 PRISM_ASSERT(parser->current_block_exits != NULL);
6835
6836 // All of the block exits that we want to remove should be within the
6837 // statements, and since we are modifying the statements, we shouldn't have
6838 // to check the end location.
6839 uint32_t start = statements->base.location.start;
6840
6841 for (size_t index = parser->current_block_exits->size; index > 0; index--) {
6842 pm_node_t *block_exit = parser->current_block_exits->nodes[index - 1];
6843 if (block_exit->location.start < start) break;
6844
6845 // Implicitly remove from the list by lowering the size.
6846 parser->current_block_exits->size--;
6847 }
6848}
6849
6853static pm_until_node_t *
6854pm_until_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_token_t *do_keyword, const pm_token_t *closing, pm_node_t *predicate, pm_statements_node_t *statements, pm_node_flags_t flags) {
6855 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
6856
6857 return pm_until_node_new(
6858 parser->arena,
6859 ++parser->node_id,
6860 flags,
6861 PM_LOCATION_INIT_TOKENS(parser, keyword, closing),
6862 TOK2LOC(parser, keyword),
6863 NTOK2LOC(parser, do_keyword),
6864 TOK2LOC(parser, closing),
6865 predicate,
6866 statements
6867 );
6868}
6869
6873static pm_until_node_t *
6874pm_until_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *predicate, pm_statements_node_t *statements, pm_node_flags_t flags) {
6875 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
6876 pm_loop_modifier_block_exits(parser, statements);
6877
6878 return pm_until_node_new(
6879 parser->arena,
6880 ++parser->node_id,
6881 flags,
6882 PM_LOCATION_INIT_NODES(statements, predicate),
6883 TOK2LOC(parser, keyword),
6884 ((pm_location_t) { 0 }),
6885 ((pm_location_t) { 0 }),
6886 predicate,
6887 statements
6888 );
6889}
6890
6894static pm_when_node_t *
6895pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_list_t *conditions, const pm_token_t *then_keyword, pm_statements_node_t *statements) {
6896 PRISM_ASSERT(conditions->size > 0);
6897 pm_node_list_finalize(&parser->node_chunks, parser->arena, conditions);
6898
6899 uint32_t start = PM_TOKEN_START(parser, keyword);
6900 uint32_t end = PM_NODE_END(conditions->nodes[conditions->size - 1]);
6901
6902 if (then_keyword != NULL) end = PM_TOKEN_END(parser, then_keyword);
6903 if (statements != NULL && PM_NODE_END(statements) > end) end = PM_NODE_END(statements);
6904
6905 return pm_when_node_new(
6906 parser->arena,
6907 ++parser->node_id,
6908 0,
6909 PM_LOCATION_INIT_BOUNDS(start, end),
6910 TOK2LOC(parser, keyword),
6911 *conditions,
6912 NTOK2LOC(parser, then_keyword),
6913 statements
6914 );
6915}
6916
6920static pm_while_node_t *
6921pm_while_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_token_t *do_keyword, const pm_token_t *closing, pm_node_t *predicate, pm_statements_node_t *statements, pm_node_flags_t flags) {
6922 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
6923
6924 return pm_while_node_new(
6925 parser->arena,
6926 ++parser->node_id,
6927 flags,
6928 PM_LOCATION_INIT_TOKENS(parser, keyword, closing),
6929 TOK2LOC(parser, keyword),
6930 NTOK2LOC(parser, do_keyword),
6931 TOK2LOC(parser, closing),
6932 predicate,
6933 statements
6934 );
6935}
6936
6940static pm_while_node_t *
6941pm_while_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *predicate, pm_statements_node_t *statements, pm_node_flags_t flags) {
6942 pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL);
6943 pm_loop_modifier_block_exits(parser, statements);
6944
6945 return pm_while_node_new(
6946 parser->arena,
6947 ++parser->node_id,
6948 flags,
6949 PM_LOCATION_INIT_NODES(statements, predicate),
6950 TOK2LOC(parser, keyword),
6951 ((pm_location_t) { 0 }),
6952 ((pm_location_t) { 0 }),
6953 predicate,
6954 statements
6955 );
6956}
6957
6961static pm_while_node_t *
6962pm_while_node_synthesized_create(pm_parser_t *parser, pm_node_t *predicate, pm_statements_node_t *statements) {
6963 return pm_while_node_new(
6964 parser->arena,
6965 ++parser->node_id,
6966 0,
6967 PM_LOCATION_INIT_UNSET,
6968 ((pm_location_t) { 0 }),
6969 ((pm_location_t) { 0 }),
6970 ((pm_location_t) { 0 }),
6971 predicate,
6972 statements
6973 );
6974}
6975
6980static pm_x_string_node_t *
6981pm_xstring_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *content, const pm_token_t *closing, const pm_string_t *unescaped) {
6982 return pm_x_string_node_new(
6983 parser->arena,
6984 ++parser->node_id,
6985 PM_STRING_FLAGS_FROZEN,
6986 PM_LOCATION_INIT_TOKENS(parser, opening, closing),
6987 TOK2LOC(parser, opening),
6988 TOK2LOC(parser, content),
6989 TOK2LOC(parser, closing),
6990 *unescaped
6991 );
6992}
6993
6997static PRISM_INLINE pm_x_string_node_t *
6998pm_xstring_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *content, const pm_token_t *closing) {
6999 return pm_xstring_node_create_unescaped(parser, opening, content, closing, &PM_STRING_EMPTY);
7000}
7001
7005static pm_yield_node_t *
7006pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_location_t *lparen_loc, pm_arguments_node_t *arguments, const pm_location_t *rparen_loc) {
7007 uint32_t start = PM_TOKEN_START(parser, keyword);
7008 uint32_t end;
7009
7010 if (rparen_loc->length > 0) {
7011 end = PM_LOCATION_END(rparen_loc);
7012 } else if (arguments != NULL) {
7013 end = PM_NODE_END(arguments);
7014 } else if (lparen_loc->length > 0) {
7015 end = PM_LOCATION_END(lparen_loc);
7016 } else {
7017 end = PM_TOKEN_END(parser, keyword);
7018 }
7019
7020 return pm_yield_node_new(
7021 parser->arena,
7022 ++parser->node_id,
7023 0,
7024 ((pm_location_t) { .start = start, .length = U32(end - start) }),
7025 TOK2LOC(parser, keyword),
7026 *lparen_loc,
7027 arguments,
7028 *rparen_loc
7029 );
7030}
7031
7037static PRISM_INLINE int
7038pm_parser_local_depth(pm_parser_t *parser, pm_token_t *token) {
7039 return pm_parser_local_depth_constant_id(parser, pm_parser_constant_id_token(parser, token));
7040}
7041
7045static PRISM_INLINE void
7046pm_parser_local_add(pm_parser_t *parser, pm_constant_id_t constant_id, const uint8_t *start, const uint8_t *end, uint32_t reads) {
7047 pm_locals_write(parser, constant_id, U32(start - parser->start), U32(end - start), reads);
7048}
7049
7053static pm_constant_id_t
7054pm_parser_local_add_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, uint32_t reads) {
7055 pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, start, end);
7056 if (constant_id != 0) pm_parser_local_add(parser, constant_id, start, end, reads);
7057 return constant_id;
7058}
7059
7063static PRISM_INLINE pm_constant_id_t
7064pm_parser_local_add_location(pm_parser_t *parser, pm_location_t *location, uint32_t reads) {
7065 return pm_parser_local_add_raw(parser, parser->start + location->start, parser->start + location->start + location->length, reads);
7066}
7067
7071static PRISM_INLINE pm_constant_id_t
7072pm_parser_local_add_token(pm_parser_t *parser, pm_token_t *token, uint32_t reads) {
7073 return pm_parser_local_add_raw(parser, token->start, token->end, reads);
7074}
7075
7079static pm_constant_id_t
7080pm_parser_local_add_owned(pm_parser_t *parser, uint8_t *start, size_t length) {
7081 pm_constant_id_t constant_id = pm_parser_constant_id_owned(parser, start, length);
7082 if (constant_id != 0) pm_parser_local_add(parser, constant_id, parser->start, parser->start, 1);
7083 return constant_id;
7084}
7085
7089static pm_constant_id_t
7090pm_parser_local_add_constant(pm_parser_t *parser, const char *start, size_t length) {
7091 pm_constant_id_t constant_id = pm_parser_constant_id_constant(parser, start, length);
7092 if (constant_id != 0) pm_parser_local_add(parser, constant_id, parser->start, parser->start, 1);
7093 return constant_id;
7094}
7095
7103static bool
7104pm_parser_parameter_name_check(pm_parser_t *parser, const pm_token_t *name) {
7105 // We want to check whether the parameter name is a numbered parameter or
7106 // not.
7107 pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, name), PM_TOKEN_LENGTH(name));
7108
7109 // Otherwise we'll fetch the constant id for the parameter name and check
7110 // whether it's already in the current scope.
7111 pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, name);
7112
7113 if (pm_parser_binding(parser, constant_id) >= parser->current_scope->locals_start) {
7114 // Add an error if the parameter doesn't start with _ and has been seen before
7115 if ((name->start < name->end) && (*name->start != '_')) {
7116 pm_parser_err_token(parser, name, PM_ERR_PARAMETER_NAME_DUPLICATED);
7117 }
7118 return true;
7119 }
7120 return false;
7121}
7122
7126static void
7127pm_parser_scope_pop(pm_parser_t *parser) {
7128 pm_scope_t *scope = parser->current_scope;
7129
7130 /* Restore the bindings that the locals of this scope hid, in the reverse of
7131 * the order they were declared, then discard the locals. */
7132 for (uint32_t index = parser->locals_size; index > scope->locals_start; index--) {
7133 const pm_local_t *local = &parser->locals[index - 1];
7134 parser->bindings[local->name] = local->shadowed;
7135 }
7136
7137 parser->locals_size = scope->locals_start;
7138
7139 parser->current_scope = scope->previous;
7140 scope->previous = parser->free_scopes;
7141 parser->free_scopes = scope;
7142}
7143
7144/******************************************************************************/
7145/* Stack helpers */
7146/******************************************************************************/
7147
7152static void
7153pm_state_stack_spill(pm_arena_t *arena, pm_state_stack_t *stack) {
7154 uint32_t size = (stack->depth >> 6) - 1;
7155
7156 if (size == stack->capacity) {
7157 uint32_t capacity = stack->capacity == 0 ? 2 : stack->capacity * 2;
7158 stack->spill = PM_ARENA_REALLOC(arena, stack->spill, capacity * sizeof(uint64_t), stack->capacity * sizeof(uint64_t), uint64_t);
7159 stack->capacity = capacity;
7160 }
7161
7162 stack->spill[size] = stack->top;
7163 stack->top = 0;
7164}
7165
7170static PRISM_INLINE void
7171pm_state_stack_push(pm_arena_t *arena, pm_state_stack_t *stack, bool value) {
7172 if (PRISM_UNLIKELY(stack->depth && !(stack->depth & 63))) {
7173 pm_state_stack_spill(arena, stack);
7174 }
7175
7176 stack->top = (stack->top << 1) | value;
7177 stack->depth++;
7178}
7179
7185static PRISM_INLINE void
7186pm_state_stack_pop(pm_state_stack_t *stack) {
7187 /* A closing delimiter without a matching opening one is a syntax error that
7188 * the parser reports and recovers from, but the lexer still pops for it, so
7189 * popping an empty stack has to leave it empty. */
7190 if (PRISM_UNLIKELY(stack->depth == 0)) return;
7191
7192 stack->top >>= 1;
7193 stack->depth--;
7194
7195 if (PRISM_UNLIKELY(stack->depth && !(stack->depth & 63))) {
7196 stack->top = stack->spill[(stack->depth >> 6) - 1];
7197 }
7198}
7199
7204static PRISM_INLINE bool
7205pm_state_stack_p(const pm_state_stack_t *stack) {
7206 return stack->top & 1;
7207}
7208
7213static PRISM_INLINE void
7214pm_accepts_block_stack_push(pm_parser_t *parser, bool value) {
7215 pm_state_stack_push(&parser->scratch_arena, &parser->accepts_block_stack, !value);
7216}
7217
7218static PRISM_INLINE void
7219pm_accepts_block_stack_pop(pm_parser_t *parser) {
7220 pm_state_stack_pop(&parser->accepts_block_stack);
7221}
7222
7223static PRISM_INLINE bool
7224pm_accepts_block_stack_p(pm_parser_t *parser) {
7225 return !pm_state_stack_p(&parser->accepts_block_stack);
7226}
7227
7228static PRISM_INLINE void
7229pm_do_loop_stack_push(pm_parser_t *parser, bool value) {
7230 pm_state_stack_push(&parser->scratch_arena, &parser->do_loop_stack, value);
7231}
7232
7233static PRISM_INLINE void
7234pm_do_loop_stack_pop(pm_parser_t *parser) {
7235 pm_state_stack_pop(&parser->do_loop_stack);
7236}
7237
7238static PRISM_INLINE bool
7239pm_do_loop_stack_p(pm_parser_t *parser) {
7240 return pm_state_stack_p(&parser->do_loop_stack);
7241}
7242
7272static PRISM_INLINE void
7273pm_enclosure_frame_push(pm_parser_t *parser) {
7274 pm_do_loop_stack_push(parser, false);
7275 pm_accepts_block_stack_push(parser, true);
7276}
7277
7278static PRISM_INLINE void
7279pm_enclosure_frame_pop(pm_parser_t *parser) {
7280 pm_do_loop_stack_pop(parser);
7281 pm_accepts_block_stack_pop(parser);
7282}
7283
7284/******************************************************************************/
7285/* Lexer check helpers */
7286/******************************************************************************/
7287
7292static PRISM_INLINE uint8_t
7293peek_at(const pm_parser_t *parser, const uint8_t *cursor) {
7294 if (cursor < parser->end) {
7295 return *cursor;
7296 } else {
7297 return '\0';
7298 }
7299}
7300
7306static PRISM_INLINE uint8_t
7307peek_offset(pm_parser_t *parser, ptrdiff_t offset) {
7308 return peek_at(parser, parser->current.end + offset);
7309}
7310
7315static PRISM_INLINE uint8_t
7316peek(const pm_parser_t *parser) {
7317 return peek_at(parser, parser->current.end);
7318}
7319
7324static PRISM_INLINE bool
7325match(pm_parser_t *parser, uint8_t value) {
7326 if (peek(parser) == value) {
7327 parser->current.end++;
7328 return true;
7329 }
7330 return false;
7331}
7332
7337static PRISM_INLINE size_t
7338match_eol_at(pm_parser_t *parser, const uint8_t *cursor) {
7339 if (peek_at(parser, cursor) == '\n') {
7340 return 1;
7341 }
7342 if (peek_at(parser, cursor) == '\r' && peek_at(parser, cursor + 1) == '\n') {
7343 return 2;
7344 }
7345 return 0;
7346}
7347
7353static PRISM_INLINE size_t
7354match_eol_offset(pm_parser_t *parser, ptrdiff_t offset) {
7355 return match_eol_at(parser, parser->current.end + offset);
7356}
7357
7363static PRISM_INLINE size_t
7364match_eol(pm_parser_t *parser) {
7365 return match_eol_at(parser, parser->current.end);
7366}
7367
7371static PRISM_INLINE const uint8_t *
7372next_newline(const uint8_t *cursor, ptrdiff_t length) {
7373 PRISM_ASSERT(length >= 0);
7374
7375 // Note that it's okay for us to use memchr here to look for \n because none
7376 // of the encodings that we support have \n as a component of a multi-byte
7377 // character.
7378 return memchr(cursor, '\n', (size_t) length);
7379}
7380
7384static PRISM_INLINE bool
7385ambiguous_operator_p(const pm_parser_t *parser, bool space_seen) {
7386 return !lex_state_p(parser, PM_LEX_STATE_CLASS | PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME | PM_LEX_STATE_ENDFN) && space_seen && !pm_char_is_whitespace(peek(parser));
7387}
7388
7393static bool
7394parser_lex_magic_comment_encoding_value(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
7395 const pm_encoding_t *encoding = pm_encoding_find(start, end);
7396
7397 if (encoding != NULL) {
7398 if (parser->encoding != encoding) {
7399 parser->encoding = encoding;
7400 if (parser->encoding_changed_callback != NULL) parser->encoding_changed_callback(parser);
7401 }
7402
7403 parser->encoding_changed = (encoding != PM_ENCODING_UTF_8_ENTRY);
7404 return true;
7405 }
7406
7407 return false;
7408}
7409
7414static void
7415parser_lex_magic_comment_encoding(pm_parser_t *parser) {
7416 const uint8_t *cursor = parser->current.start + 1;
7417 const uint8_t *end = parser->current.end;
7418
7419 bool separator = false;
7420 while (true) {
7421 if (end - cursor <= 6) return;
7422 switch (cursor[6]) {
7423 case 'C': case 'c': cursor += 6; continue;
7424 case 'O': case 'o': cursor += 5; continue;
7425 case 'D': case 'd': cursor += 4; continue;
7426 case 'I': case 'i': cursor += 3; continue;
7427 case 'N': case 'n': cursor += 2; continue;
7428 case 'G': case 'g': cursor += 1; continue;
7429 case '=': case ':':
7430 separator = true;
7431 cursor += 6;
7432 break;
7433 default:
7434 cursor += 6;
7435 if (pm_char_is_whitespace(*cursor)) break;
7436 continue;
7437 }
7438 if (pm_strncasecmp(cursor - 6, (const uint8_t *) "coding", 6) == 0) break;
7439 separator = false;
7440 }
7441
7442 while (true) {
7443 do {
7444 if (++cursor >= end) return;
7445 } while (pm_char_is_whitespace(*cursor));
7446
7447 if (separator) break;
7448 if (*cursor != '=' && *cursor != ':') return;
7449
7450 separator = true;
7451 cursor++;
7452 }
7453
7454 /* The name of an encoding is made of ASCII letters, digits, - and _. */
7455 const uint8_t *value_start = cursor;
7456 while ((*cursor == '-' || *cursor == '_' || pm_encoding_ascii_alnum_p(*cursor)) && ++cursor < end);
7457
7458 if (!parser_lex_magic_comment_encoding_value(parser, value_start, cursor)) {
7459 // If we were unable to parse the encoding value, then we've got an
7460 // issue because we didn't understand the encoding that the user was
7461 // trying to use. In this case we'll keep using the default encoding but
7462 // add an error to the parser to indicate an unsuccessful parse.
7463 pm_parser_err(parser, U32(value_start - parser->start), U32(cursor - value_start), PM_ERR_INVALID_ENCODING_MAGIC_COMMENT);
7464 }
7465}
7466
7467typedef enum {
7468 PM_MAGIC_COMMENT_BOOLEAN_VALUE_TRUE,
7469 PM_MAGIC_COMMENT_BOOLEAN_VALUE_FALSE,
7470 PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID
7471} pm_magic_comment_boolean_value_t;
7472
7477static pm_magic_comment_boolean_value_t
7478parser_lex_magic_comment_boolean_value(const uint8_t *value_start, uint32_t value_length) {
7479 if (value_length == 4 && pm_strncasecmp(value_start, (const uint8_t *) "true", 4) == 0) {
7480 return PM_MAGIC_COMMENT_BOOLEAN_VALUE_TRUE;
7481 } else if (value_length == 5 && pm_strncasecmp(value_start, (const uint8_t *) "false", 5) == 0) {
7482 return PM_MAGIC_COMMENT_BOOLEAN_VALUE_FALSE;
7483 } else {
7484 return PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID;
7485 }
7486}
7487
7488static PRISM_INLINE bool
7489pm_char_is_magic_comment_key_delimiter(const uint8_t b) {
7490 return b == '\'' || b == '"' || b == ':' || b == ';';
7491}
7492
7498static PRISM_INLINE const uint8_t *
7499parser_lex_magic_comment_emacs_marker(pm_parser_t *parser, const uint8_t *cursor, const uint8_t *end) {
7500 // Scan for '*' as the middle character, since it is rarer than '-' in
7501 // typical comments and avoids repeated memchr calls for '-' that hit
7502 // dashes in words like "foo-bar".
7503 while ((cursor + 3 <= end) && (cursor = pm_memchr(cursor + 1, '*', (size_t) (end - cursor - 1), parser->encoding_changed, parser->encoding)) != NULL) {
7504 if (cursor[-1] == '-' && cursor + 1 < end && cursor[1] == '-') {
7505 return cursor - 1;
7506 }
7507 }
7508 return NULL;
7509}
7510
7518static bool
7519parser_magic_comment_key_p(const uint8_t *key, size_t key_length, const char *name) {
7520 PRISM_ASSERT(name[key_length] == '\0');
7521
7522 for (size_t index = 0; index < key_length; index++) {
7523 uint8_t byte = key[index];
7524 if (byte == '-') byte = '_';
7525 if ('A' <= byte && byte <= 'Z') byte |= 0x20;
7526 if (byte != (uint8_t) name[index]) return false;
7527 }
7528
7529 return true;
7530}
7531
7542static PRISM_INLINE bool
7543parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) {
7544 const uint8_t *start = parser->current.start + 1;
7545 const uint8_t *end = parser->current.end;
7546 if (end - start <= 7) return false;
7547
7548 const uint8_t *cursor;
7549 bool indicator = false;
7550
7551 if ((cursor = parser_lex_magic_comment_emacs_marker(parser, start, end)) != NULL) {
7552 start = cursor + 3;
7553
7554 if ((cursor = parser_lex_magic_comment_emacs_marker(parser, start, end)) != NULL) {
7555 end = cursor;
7556 indicator = true;
7557 } else {
7558 // If we have a start marker but not an end marker, then we cannot
7559 // have a magic comment.
7560 return false;
7561 }
7562 }
7563
7564 if (!indicator) {
7565 /* The encoding-comment line needs the full scan because its result
7566 * controls the fallback encoding scan. Other comments without a colon
7567 * cannot contain a magic directive, so reject them early. */
7568 if ((parser->current.start != parser->encoding_comment_start || parser->encoding_locked) &&
7569 pm_memchr(start, ':', (size_t) (end - start), parser->encoding_changed, parser->encoding) == NULL) {
7570 return false;
7571 }
7572
7573 start += pm_strspn_whitespace(start, end - start);
7574 }
7575
7576 cursor = start;
7577 while (cursor < end) {
7578 while (cursor < end && (pm_char_is_magic_comment_key_delimiter(*cursor) || pm_char_is_whitespace(*cursor))) cursor++;
7579
7580 const uint8_t *key_start = cursor;
7581 while (cursor < end && (!pm_char_is_magic_comment_key_delimiter(*cursor) && !pm_char_is_whitespace(*cursor))) cursor++;
7582
7583 const uint8_t *key_end = cursor;
7584 while (cursor < end && pm_char_is_whitespace(*cursor)) cursor++;
7585 if (cursor == end) break;
7586
7587 if (*cursor == ':') {
7588 cursor++;
7589 } else {
7590 if (!indicator) return false;
7591 continue;
7592 }
7593
7594 while (cursor < end && pm_char_is_whitespace(*cursor)) cursor++;
7595 if (cursor == end) break;
7596
7597 const uint8_t *value_start;
7598 const uint8_t *value_end;
7599
7600 if (*cursor == '"') {
7601 value_start = ++cursor;
7602 for (; cursor < end && *cursor != '"'; cursor++) {
7603 if (*cursor == '\\' && (cursor + 1 < end)) cursor++;
7604 }
7605 value_end = cursor;
7606 if (cursor < end && *cursor == '"') cursor++;
7607 } else {
7608 value_start = cursor;
7609 while (cursor < end && *cursor != '"' && *cursor != ';' && !pm_char_is_whitespace(*cursor)) cursor++;
7610 value_end = cursor;
7611 }
7612
7613 if (indicator) {
7614 while (cursor < end && (*cursor == ';' || pm_char_is_whitespace(*cursor))) cursor++;
7615 } else {
7616 while (cursor < end && pm_char_is_whitespace(*cursor)) cursor++;
7617 if (cursor != end) return false;
7618 }
7619
7620 /* Finally, we can start checking the key against the list of known
7621 * magic comment keys, and potentially change state based on that. */
7622 const size_t key_length = (size_t) (key_end - key_start);
7623 uint32_t value_length = (uint32_t) (value_end - value_start);
7624
7625 switch (key_length) {
7626 case 6:
7627 case 8:
7628 /* We only want to attempt to compare against encoding comments
7629 * if it is the first line in the file (or the second in the
7630 * case of a shebang). */
7631 if (parser->current.start == parser->encoding_comment_start && !parser->encoding_locked) {
7632 if (parser_magic_comment_key_p(key_start, key_length, key_length == 6 ? "coding" : "encoding")) {
7633 if (!parser_lex_magic_comment_encoding_value(parser, value_start, value_end)) {
7634 pm_parser_err(parser, U32(value_start - parser->start), value_length, PM_ERR_INVALID_ENCODING_MAGIC_COMMENT);
7635 }
7636 }
7637 }
7638 break;
7639 case 11:
7640 if (parser_magic_comment_key_p(key_start, key_length, "warn_indent")) {
7641 switch (parser_lex_magic_comment_boolean_value(value_start, value_length)) {
7642 case PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID:
7643 PM_PARSER_WARN_TOKEN_FORMAT(
7644 parser,
7645 &parser->current,
7646 PM_WARN_INVALID_MAGIC_COMMENT_VALUE,
7647 (int) key_length,
7648 "warn_indent",
7649 (int) value_length,
7650 (const char *) value_start
7651 );
7652 break;
7653 case PM_MAGIC_COMMENT_BOOLEAN_VALUE_FALSE:
7654 parser->warn_mismatched_indentation = false;
7655 break;
7656 case PM_MAGIC_COMMENT_BOOLEAN_VALUE_TRUE:
7657 parser->warn_mismatched_indentation = true;
7658 break;
7659 }
7660 }
7661 break;
7662 case 21:
7663 if (parser_magic_comment_key_p(key_start, key_length, "frozen_string_literal")) {
7664 /* We only want to handle frozen string literal comments if
7665 * it is before any semantic tokens have been seen. */
7666 if (semantic_token_seen) {
7667 pm_parser_warn_token(parser, &parser->current, PM_WARN_IGNORED_FROZEN_STRING_LITERAL);
7668 } else {
7669 switch (parser_lex_magic_comment_boolean_value(value_start, value_length)) {
7670 case PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID:
7671 PM_PARSER_WARN_TOKEN_FORMAT(
7672 parser,
7673 &parser->current,
7674 PM_WARN_INVALID_MAGIC_COMMENT_VALUE,
7675 (int) key_length,
7676 "frozen_string_literal",
7677 (int) value_length,
7678 (const char *) value_start
7679 );
7680 break;
7681 case PM_MAGIC_COMMENT_BOOLEAN_VALUE_FALSE:
7682 parser->frozen_string_literal = PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED;
7683 break;
7684 case PM_MAGIC_COMMENT_BOOLEAN_VALUE_TRUE:
7685 parser->frozen_string_literal = PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED;
7686 break;
7687 }
7688 }
7689 }
7690 break;
7691 case 24:
7692 if (parser_magic_comment_key_p(key_start, key_length, "shareable_constant_value")) {
7693 const uint8_t *cursor = parser->current.start;
7694 while ((cursor > parser->start) && ((cursor[-1] == ' ') || (cursor[-1] == '\t'))) cursor--;
7695
7696 if (!((cursor == parser->start) || (cursor[-1] == '\n'))) {
7697 pm_parser_warn_token(parser, &parser->current, PM_WARN_SHAREABLE_CONSTANT_VALUE_LINE);
7698 } else if (value_length == 4 && pm_strncasecmp(value_start, (const uint8_t *) "none", 4) == 0) {
7699 pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_NONE);
7700 } else if (value_length == 7 && pm_strncasecmp(value_start, (const uint8_t *) "literal", 7) == 0) {
7701 pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_LITERAL);
7702 } else if (value_length == 23 && pm_strncasecmp(value_start, (const uint8_t *) "experimental_everything", 23) == 0) {
7703 pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING);
7704 } else if (value_length == 17 && pm_strncasecmp(value_start, (const uint8_t *) "experimental_copy", 17) == 0) {
7705 pm_parser_scope_shareable_constant_set(parser, PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY);
7706 } else {
7707 PM_PARSER_WARN_TOKEN_FORMAT(
7708 parser,
7709 &parser->current,
7710 PM_WARN_INVALID_MAGIC_COMMENT_VALUE,
7711 (int) key_length,
7712 "shareable_constant_value",
7713 (int) value_length,
7714 (const char *) value_start
7715 );
7716 }
7717 }
7718 break;
7719 }
7720
7721 // Allocate a new magic comment node to append to the parser's list.
7722 pm_magic_comment_t *magic_comment = PM_ARENA_ALLOC(&parser->metadata_arena, pm_magic_comment_t);
7723 magic_comment->node.next = NULL;
7724 magic_comment->key = (pm_location_t) { .start = U32(key_start - parser->start), .length = U32(key_length) };
7725 magic_comment->value = (pm_location_t) { .start = U32(value_start - parser->start), .length = value_length };
7726 pm_list_append(&parser->magic_comment_list, (pm_list_node_t *) magic_comment);
7727 }
7728
7729 return true;
7730}
7731
7732/******************************************************************************/
7733/* Context manipulations */
7734/******************************************************************************/
7735
7736static const uint32_t context_terminators[] = {
7737 [PM_CONTEXT_NONE] = 0,
7738 [PM_CONTEXT_BEGIN] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7739 [PM_CONTEXT_BEGIN_ENSURE] = (1U << PM_TOKEN_KEYWORD_END),
7740 [PM_CONTEXT_BEGIN_ELSE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_END),
7741 [PM_CONTEXT_BEGIN_RESCUE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7742 [PM_CONTEXT_BLOCK_BRACES] = (1U << PM_TOKEN_BRACE_RIGHT),
7743 [PM_CONTEXT_BLOCK_KEYWORDS] = (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ENSURE),
7744 [PM_CONTEXT_BLOCK_ENSURE] = (1U << PM_TOKEN_KEYWORD_END),
7745 [PM_CONTEXT_BLOCK_ELSE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_END),
7746 [PM_CONTEXT_BLOCK_PARAMETERS] = (1U << PM_TOKEN_PIPE),
7747 [PM_CONTEXT_BLOCK_RESCUE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7748 [PM_CONTEXT_CASE_WHEN] = (1U << PM_TOKEN_KEYWORD_WHEN) | (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_ELSE),
7749 [PM_CONTEXT_CASE_IN] = (1U << PM_TOKEN_KEYWORD_IN) | (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_ELSE),
7750 [PM_CONTEXT_CLASS] = (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ENSURE),
7751 [PM_CONTEXT_CLASS_ENSURE] = (1U << PM_TOKEN_KEYWORD_END),
7752 [PM_CONTEXT_CLASS_ELSE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_END),
7753 [PM_CONTEXT_CLASS_RESCUE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7754 [PM_CONTEXT_DEF] = (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ENSURE),
7755 [PM_CONTEXT_DEF_ENSURE] = (1U << PM_TOKEN_KEYWORD_END),
7756 [PM_CONTEXT_DEF_ELSE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_END),
7757 [PM_CONTEXT_DEF_RESCUE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7758 [PM_CONTEXT_DEF_PARAMS] = (1U << PM_TOKEN_EOF),
7759 [PM_CONTEXT_DEFINED] = (1U << PM_TOKEN_EOF),
7760 [PM_CONTEXT_DEFAULT_PARAMS] = (1U << PM_TOKEN_COMMA) | (1U << PM_TOKEN_PARENTHESIS_RIGHT),
7761 [PM_CONTEXT_ELSE] = (1U << PM_TOKEN_KEYWORD_END),
7762 [PM_CONTEXT_ELSIF] = (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_ELSIF) | (1U << PM_TOKEN_KEYWORD_END),
7763 [PM_CONTEXT_EMBEXPR] = (1U << PM_TOKEN_EMBEXPR_END) | (1U << PM_TOKEN_HEREDOC_END),
7764 [PM_CONTEXT_FOR] = (1U << PM_TOKEN_KEYWORD_END),
7765 [PM_CONTEXT_FOR_INDEX] = (1U << PM_TOKEN_KEYWORD_IN),
7766 [PM_CONTEXT_IF] = (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_ELSIF) | (1U << PM_TOKEN_KEYWORD_END),
7767 [PM_CONTEXT_LAMBDA_BRACES] = (1U << PM_TOKEN_BRACE_RIGHT),
7768 [PM_CONTEXT_LAMBDA_DO_END] = (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ENSURE),
7769 [PM_CONTEXT_LAMBDA_ENSURE] = (1U << PM_TOKEN_KEYWORD_END),
7770 [PM_CONTEXT_LAMBDA_ELSE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_END),
7771 [PM_CONTEXT_LAMBDA_RESCUE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7772 [PM_CONTEXT_LOOP_PREDICATE] = (1U << PM_TOKEN_KEYWORD_DO) | (1U << PM_TOKEN_KEYWORD_THEN),
7773 [PM_CONTEXT_MAIN] = (1U << PM_TOKEN_EOF),
7774 [PM_CONTEXT_MODULE] = (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ENSURE),
7775 [PM_CONTEXT_MODULE_ENSURE] = (1U << PM_TOKEN_KEYWORD_END),
7776 [PM_CONTEXT_MODULE_ELSE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_END),
7777 [PM_CONTEXT_MODULE_RESCUE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7778 [PM_CONTEXT_MULTI_TARGET] = (1U << PM_TOKEN_EOF),
7779 [PM_CONTEXT_PARENS] = (1U << PM_TOKEN_PARENTHESIS_RIGHT),
7780 [PM_CONTEXT_POSTEXE] = (1U << PM_TOKEN_BRACE_RIGHT),
7781 [PM_CONTEXT_PREDICATE] = (1U << PM_TOKEN_KEYWORD_THEN) | (1U << PM_TOKEN_NEWLINE) | (1U << PM_TOKEN_SEMICOLON),
7782 [PM_CONTEXT_PREEXE] = (1U << PM_TOKEN_BRACE_RIGHT),
7783 [PM_CONTEXT_RESCUE_MODIFIER] = (1U << PM_TOKEN_EOF),
7784 [PM_CONTEXT_SCLASS] = (1U << PM_TOKEN_KEYWORD_END) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ENSURE),
7785 [PM_CONTEXT_SCLASS_ENSURE] = (1U << PM_TOKEN_KEYWORD_END),
7786 [PM_CONTEXT_SCLASS_ELSE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_END),
7787 [PM_CONTEXT_SCLASS_RESCUE] = (1U << PM_TOKEN_KEYWORD_ENSURE) | (1U << PM_TOKEN_KEYWORD_RESCUE) | (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7788 [PM_CONTEXT_TERNARY] = (1U << PM_TOKEN_EOF),
7789 [PM_CONTEXT_UNLESS] = (1U << PM_TOKEN_KEYWORD_ELSE) | (1U << PM_TOKEN_KEYWORD_END),
7790 [PM_CONTEXT_UNTIL] = (1U << PM_TOKEN_KEYWORD_END),
7791 [PM_CONTEXT_WHILE] = (1U << PM_TOKEN_KEYWORD_END),
7792};
7793
7794static PRISM_INLINE bool
7795context_terminator(pm_context_t context, pm_token_t *token) {
7796 return token->type < 32 && (context_terminators[context] & (1U << token->type));
7797}
7798
7803static pm_context_t
7804context_recoverable(const pm_parser_t *parser, pm_token_t *token) {
7805 pm_context_node_t *context_node = parser->current_context;
7806
7807 while (context_node != NULL) {
7808 if (context_terminator(context_node->context, token)) return context_node->context;
7809 context_node = context_node->prev;
7810 }
7811
7812 return PM_CONTEXT_NONE;
7813}
7814
7815PM_STATIC_ASSERT(__LINE__, PM_CONTEXT_MAXIMUM <= 64, "Expected every context to fit in the context mask.");
7816
7817static bool
7818context_push(pm_parser_t *parser, pm_context_t context) {
7819 pm_context_node_t *context_node = parser->free_contexts;
7820 if (context_node != NULL) {
7821 parser->free_contexts = context_node->prev;
7822 } else {
7823 context_node = PM_ARENA_ALLOC(&parser->scratch_arena, pm_context_node_t);
7824 }
7825
7826 pm_context_node_t *previous = parser->current_context;
7827 *context_node = (pm_context_node_t) {
7828 .context = context,
7829 .prev = previous,
7830 .mask = (previous == NULL ? 0 : previous->mask) | ((uint64_t) 1 << context)
7831 };
7832
7833 parser->current_context = context_node;
7834 return true;
7835}
7836
7837static void
7838context_pop(pm_parser_t *parser) {
7839 pm_context_node_t *context_node = parser->current_context;
7840 parser->current_context = context_node->prev;
7841 context_node->prev = parser->free_contexts;
7842 parser->free_contexts = context_node;
7843}
7844
7845static bool
7846context_p(const pm_parser_t *parser, pm_context_t context) {
7847 return parser->current_context != NULL && (parser->current_context->mask & ((uint64_t) 1 << context)) != 0;
7848}
7849
7850static bool
7851context_def_p(const pm_parser_t *parser) {
7852 pm_context_node_t *context_node = parser->current_context;
7853
7854 while (context_node != NULL) {
7855 switch (context_node->context) {
7856 case PM_CONTEXT_DEF:
7857 case PM_CONTEXT_DEF_PARAMS:
7858 case PM_CONTEXT_DEF_ENSURE:
7859 case PM_CONTEXT_DEF_RESCUE:
7860 case PM_CONTEXT_DEF_ELSE:
7861 return true;
7862 case PM_CONTEXT_CLASS:
7863 case PM_CONTEXT_CLASS_ENSURE:
7864 case PM_CONTEXT_CLASS_RESCUE:
7865 case PM_CONTEXT_CLASS_ELSE:
7866 case PM_CONTEXT_MODULE:
7867 case PM_CONTEXT_MODULE_ENSURE:
7868 case PM_CONTEXT_MODULE_RESCUE:
7869 case PM_CONTEXT_MODULE_ELSE:
7870 case PM_CONTEXT_SCLASS:
7871 case PM_CONTEXT_SCLASS_ENSURE:
7872 case PM_CONTEXT_SCLASS_RESCUE:
7873 case PM_CONTEXT_SCLASS_ELSE:
7874 return false;
7875 default:
7876 context_node = context_node->prev;
7877 }
7878 }
7879
7880 return false;
7881}
7882
7887static const char *
7888context_human(pm_context_t context) {
7889 switch (context) {
7890 case PM_CONTEXT_NONE:
7891 case PM_CONTEXT_MAXIMUM:
7892 PRISM_ASSERT(false && "unreachable");
7893 return "";
7894 case PM_CONTEXT_BEGIN: return "begin statement";
7895 case PM_CONTEXT_BLOCK_BRACES: return "'{'..'}' block";
7896 case PM_CONTEXT_BLOCK_KEYWORDS: return "'do'..'end' block";
7897 case PM_CONTEXT_BLOCK_PARAMETERS: return "'|'..'|' block parameter";
7898 case PM_CONTEXT_CASE_WHEN: return "'when' clause";
7899 case PM_CONTEXT_CASE_IN: return "'in' clause";
7900 case PM_CONTEXT_CLASS: return "class definition";
7901 case PM_CONTEXT_DEF: return "method definition";
7902 case PM_CONTEXT_DEF_PARAMS: return "method parameters";
7903 case PM_CONTEXT_DEFAULT_PARAMS: return "parameter default value";
7904 case PM_CONTEXT_DEFINED: return "'defined?' expression";
7905 case PM_CONTEXT_ELSE:
7906 case PM_CONTEXT_BEGIN_ELSE:
7907 case PM_CONTEXT_BLOCK_ELSE:
7908 case PM_CONTEXT_CLASS_ELSE:
7909 case PM_CONTEXT_DEF_ELSE:
7910 case PM_CONTEXT_LAMBDA_ELSE:
7911 case PM_CONTEXT_MODULE_ELSE:
7912 case PM_CONTEXT_SCLASS_ELSE: return "'else' clause";
7913 case PM_CONTEXT_ELSIF: return "'elsif' clause";
7914 case PM_CONTEXT_EMBEXPR: return "embedded expression";
7915 case PM_CONTEXT_BEGIN_ENSURE:
7916 case PM_CONTEXT_BLOCK_ENSURE:
7917 case PM_CONTEXT_CLASS_ENSURE:
7918 case PM_CONTEXT_DEF_ENSURE:
7919 case PM_CONTEXT_LAMBDA_ENSURE:
7920 case PM_CONTEXT_MODULE_ENSURE:
7921 case PM_CONTEXT_SCLASS_ENSURE: return "'ensure' clause";
7922 case PM_CONTEXT_FOR: return "for loop";
7923 case PM_CONTEXT_FOR_INDEX: return "for loop index";
7924 case PM_CONTEXT_IF: return "if statement";
7925 case PM_CONTEXT_LAMBDA_BRACES: return "'{'..'}' lambda block";
7926 case PM_CONTEXT_LAMBDA_DO_END: return "'do'..'end' lambda block";
7927 case PM_CONTEXT_LOOP_PREDICATE: return "loop predicate";
7928 case PM_CONTEXT_MAIN: return "top level context";
7929 case PM_CONTEXT_MODULE: return "module definition";
7930 case PM_CONTEXT_MULTI_TARGET: return "multiple targets";
7931 case PM_CONTEXT_PARENS: return "parentheses";
7932 case PM_CONTEXT_POSTEXE: return "'END' block";
7933 case PM_CONTEXT_PREDICATE: return "predicate";
7934 case PM_CONTEXT_PREEXE: return "'BEGIN' block";
7935 case PM_CONTEXT_BEGIN_RESCUE:
7936 case PM_CONTEXT_BLOCK_RESCUE:
7937 case PM_CONTEXT_CLASS_RESCUE:
7938 case PM_CONTEXT_DEF_RESCUE:
7939 case PM_CONTEXT_LAMBDA_RESCUE:
7940 case PM_CONTEXT_MODULE_RESCUE:
7941 case PM_CONTEXT_RESCUE_MODIFIER:
7942 case PM_CONTEXT_SCLASS_RESCUE: return "'rescue' clause";
7943 case PM_CONTEXT_SCLASS: return "singleton class definition";
7944 case PM_CONTEXT_TERNARY: return "ternary expression";
7945 case PM_CONTEXT_UNLESS: return "unless statement";
7946 case PM_CONTEXT_UNTIL: return "until statement";
7947 case PM_CONTEXT_WHILE: return "while statement";
7948 }
7949
7950 PRISM_ASSERT(false && "unreachable");
7951 return "";
7952}
7953
7954/******************************************************************************/
7955/* Specific token lexers */
7956/******************************************************************************/
7957
7958static PRISM_INLINE void
7959pm_strspn_number_validate(pm_parser_t *parser, const uint8_t *string, size_t length, const uint8_t *invalid) {
7960 if (invalid != NULL) {
7961 pm_diagnostic_id_t diag_id = (invalid == (string + length - 1)) ? PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING : PM_ERR_INVALID_NUMBER_UNDERSCORE_INNER;
7962 pm_parser_err(parser, U32(invalid - parser->start), 1, diag_id);
7963 }
7964}
7965
7966static size_t
7967pm_strspn_binary_number_validate(pm_parser_t *parser, const uint8_t *string) {
7968 const uint8_t *invalid = NULL;
7969 size_t length = pm_strspn_binary_number(string, parser->end - string, &invalid);
7970 pm_strspn_number_validate(parser, string, length, invalid);
7971 return length;
7972}
7973
7974static size_t
7975pm_strspn_octal_number_validate(pm_parser_t *parser, const uint8_t *string) {
7976 const uint8_t *invalid = NULL;
7977 size_t length = pm_strspn_octal_number(string, parser->end - string, &invalid);
7978 pm_strspn_number_validate(parser, string, length, invalid);
7979 return length;
7980}
7981
7982static size_t
7983pm_strspn_decimal_number_validate(pm_parser_t *parser, const uint8_t *string) {
7984 const uint8_t *invalid = NULL;
7985 size_t length = pm_strspn_decimal_number(string, parser->end - string, &invalid);
7986 pm_strspn_number_validate(parser, string, length, invalid);
7987 return length;
7988}
7989
7990static size_t
7991pm_strspn_hexadecimal_number_validate(pm_parser_t *parser, const uint8_t *string) {
7992 const uint8_t *invalid = NULL;
7993 size_t length = pm_strspn_hexadecimal_number(string, parser->end - string, &invalid);
7994 pm_strspn_number_validate(parser, string, length, invalid);
7995 return length;
7996}
7997
7998static pm_token_type_t
7999lex_optional_float_suffix(pm_parser_t *parser, bool* seen_e) {
8000 pm_token_type_t type = PM_TOKEN_INTEGER;
8001
8002 // Here we're going to attempt to parse the optional decimal portion of a
8003 // float. If it's not there, then it's okay and we'll just continue on.
8004 if (peek(parser) == '.') {
8005 if (pm_char_is_decimal_digit(peek_offset(parser, 1))) {
8006 parser->current.end += 2;
8007 parser->current.end += pm_strspn_decimal_number_validate(parser, parser->current.end);
8008 type = PM_TOKEN_FLOAT;
8009 } else {
8010 // If we had a . and then something else, then it's not a float
8011 // suffix on a number it's a method call or something else.
8012 return type;
8013 }
8014 }
8015
8016 // Here we're going to attempt to parse the optional exponent portion of a
8017 // float. If it's not there, it's okay and we'll just continue on.
8018 if ((peek(parser) == 'e') || (peek(parser) == 'E')) {
8019 if ((peek_offset(parser, 1) == '+') || (peek_offset(parser, 1) == '-')) {
8020 parser->current.end += 2;
8021
8022 if (pm_char_is_decimal_digit(peek(parser))) {
8023 parser->current.end++;
8024 parser->current.end += pm_strspn_decimal_number_validate(parser, parser->current.end);
8025 } else {
8026 pm_parser_err_current(parser, PM_ERR_INVALID_FLOAT_EXPONENT);
8027 }
8028 } else if (pm_char_is_decimal_digit(peek_offset(parser, 1))) {
8029 parser->current.end++;
8030 parser->current.end += pm_strspn_decimal_number_validate(parser, parser->current.end);
8031 } else {
8032 return type;
8033 }
8034
8035 *seen_e = true;
8036 type = PM_TOKEN_FLOAT;
8037 }
8038
8039 return type;
8040}
8041
8042static pm_token_type_t
8043lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) {
8044 pm_token_type_t type = PM_TOKEN_INTEGER;
8045 *seen_e = false;
8046
8047 if (peek_offset(parser, -1) == '0') {
8048 switch (*parser->current.end) {
8049 // 0d1111 is a decimal number
8050 case 'd':
8051 case 'D':
8052 parser->current.end++;
8053 if (pm_char_is_decimal_digit(peek(parser))) {
8054 parser->current.end += pm_strspn_decimal_number_validate(parser, parser->current.end);
8055 } else {
8056 match(parser, '_');
8057 pm_parser_err_current(parser, PM_ERR_INVALID_NUMBER_DECIMAL);
8058 }
8059
8060 break;
8061
8062 // 0b1111 is a binary number
8063 case 'b':
8064 case 'B':
8065 parser->current.end++;
8066 if (pm_char_is_binary_digit(peek(parser))) {
8067 parser->current.end += pm_strspn_binary_number_validate(parser, parser->current.end);
8068 } else {
8069 match(parser, '_');
8070 pm_parser_err_current(parser, PM_ERR_INVALID_NUMBER_BINARY);
8071 }
8072
8073 parser->integer.base = PM_INTEGER_BASE_FLAGS_BINARY;
8074 break;
8075
8076 // 0o1111 is an octal number
8077 case 'o':
8078 case 'O':
8079 parser->current.end++;
8080 if (pm_char_is_octal_digit(peek(parser))) {
8081 parser->current.end += pm_strspn_octal_number_validate(parser, parser->current.end);
8082 } else {
8083 match(parser, '_');
8084 pm_parser_err_current(parser, PM_ERR_INVALID_NUMBER_OCTAL);
8085 }
8086
8087 parser->integer.base = PM_INTEGER_BASE_FLAGS_OCTAL;
8088 break;
8089
8090 // 01111 is an octal number
8091 case '_':
8092 case '0':
8093 case '1':
8094 case '2':
8095 case '3':
8096 case '4':
8097 case '5':
8098 case '6':
8099 case '7':
8100 parser->current.end += pm_strspn_octal_number_validate(parser, parser->current.end);
8101 parser->integer.base = PM_INTEGER_BASE_FLAGS_OCTAL;
8102 break;
8103
8104 // 0x1111 is a hexadecimal number
8105 case 'x':
8106 case 'X':
8107 parser->current.end++;
8108 if (pm_char_is_hexadecimal_digit(peek(parser))) {
8109 parser->current.end += pm_strspn_hexadecimal_number_validate(parser, parser->current.end);
8110 } else {
8111 match(parser, '_');
8112 pm_parser_err_current(parser, PM_ERR_INVALID_NUMBER_HEXADECIMAL);
8113 }
8114
8115 parser->integer.base = PM_INTEGER_BASE_FLAGS_HEXADECIMAL;
8116 break;
8117
8118 // 0.xxx is a float
8119 case '.': {
8120 type = lex_optional_float_suffix(parser, seen_e);
8121 break;
8122 }
8123
8124 // 0exxx is a float
8125 case 'e':
8126 case 'E': {
8127 type = lex_optional_float_suffix(parser, seen_e);
8128 break;
8129 }
8130
8131 /* Anything else ends the literal, which is just 0. */
8132 default:
8133 parser->integer.value = 0;
8134 parser->integer.start = parser->current.start;
8135 break;
8136 }
8137 } else {
8138 // If it didn't start with a 0, then we'll lex as far as we can into a
8139 // decimal number. We compute the integer value inline to avoid
8140 // re-scanning the digits later in pm_integer_parse.
8141 {
8142 const uint8_t *cursor = parser->current.end;
8143 const uint8_t *end = parser->end;
8144 uint64_t value = (uint64_t) (cursor[-1] - '0');
8145
8146 size_t underscores = 0;
8147 bool prev_underscore = false;
8148 const uint8_t *invalid = NULL;
8149
8150 while (cursor < end) {
8151 uint8_t c = *cursor;
8152 if (c >= '0' && c <= '9') {
8153 value = value * 10 + (uint64_t) (c - '0');
8154 prev_underscore = false;
8155 cursor++;
8156 } else if (c == '_') {
8157 underscores++;
8158 if (prev_underscore && invalid == NULL) invalid = cursor;
8159 prev_underscore = true;
8160 cursor++;
8161 } else {
8162 break;
8163 }
8164 }
8165
8166 if (underscores > 0) {
8167 if (prev_underscore && invalid == NULL) invalid = cursor - 1;
8168 pm_strspn_number_validate(parser, parser->current.end, (size_t) (cursor - parser->current.end), invalid);
8169 }
8170
8171 /* This literal does not start with 0, so at most 19 digits means
8172 * it is below 10**19 and value has not wrapped. */
8173 size_t digits = (size_t) (cursor - parser->current.end) + 1 - underscores;
8174 if (digits <= 19) {
8175 parser->integer.value = value;
8176 parser->integer.start = parser->current.start;
8177 }
8178
8179 parser->current.end = cursor;
8180 }
8181
8182 // Afterward, we'll lex as far as we can into an optional float suffix.
8183 // Guard the function call: the vast majority of decimal numbers are
8184 // plain integers, so avoid the call when the next byte cannot start a
8185 // float suffix.
8186 {
8187 uint8_t next = peek(parser);
8188 if (next == '.' || next == 'e' || next == 'E') {
8189 type = lex_optional_float_suffix(parser, seen_e);
8190 }
8191 }
8192 }
8193
8194 // At this point we have a completed number, but we want to provide the user
8195 // with a good experience if they put an additional .xxx fractional
8196 // component on the end, so we'll check for that here.
8197 if (peek_offset(parser, 0) == '.' && pm_char_is_decimal_digit(peek_offset(parser, 1))) {
8198 const uint8_t *fraction_start = parser->current.end;
8199 const uint8_t *fraction_end = parser->current.end + 2;
8200 fraction_end += pm_strspn_decimal_digit(fraction_end, parser->end - fraction_end);
8201 pm_parser_err(parser, U32(fraction_start - parser->start), U32(fraction_end - fraction_start), PM_ERR_INVALID_NUMBER_FRACTION);
8202 }
8203
8204 return type;
8205}
8206
8207static pm_token_type_t
8208lex_numeric(pm_parser_t *parser) {
8209 pm_token_type_t type = PM_TOKEN_INTEGER;
8210 parser->integer.base = PM_INTEGER_BASE_FLAGS_DECIMAL;
8211
8212 if (parser->current.end < parser->end) {
8213 bool seen_e = false;
8214 type = lex_numeric_prefix(parser, &seen_e);
8215
8216 const uint8_t *end = parser->current.end;
8217 pm_token_type_t suffix_type = type;
8218
8219 if (type == PM_TOKEN_INTEGER) {
8220 if (match(parser, 'r')) {
8221 suffix_type = PM_TOKEN_INTEGER_RATIONAL;
8222
8223 if (match(parser, 'i')) {
8224 suffix_type = PM_TOKEN_INTEGER_RATIONAL_IMAGINARY;
8225 }
8226 } else if (match(parser, 'i')) {
8227 suffix_type = PM_TOKEN_INTEGER_IMAGINARY;
8228 }
8229 } else {
8230 if (!seen_e && match(parser, 'r')) {
8231 suffix_type = PM_TOKEN_FLOAT_RATIONAL;
8232
8233 if (match(parser, 'i')) {
8234 suffix_type = PM_TOKEN_FLOAT_RATIONAL_IMAGINARY;
8235 }
8236 } else if (match(parser, 'i')) {
8237 suffix_type = PM_TOKEN_FLOAT_IMAGINARY;
8238 }
8239 }
8240
8241 const uint8_t b = peek(parser);
8242 if (b != '\0' && (b >= 0x80 || ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')) || b == '_')) {
8243 parser->current.end = end;
8244 } else {
8245 type = suffix_type;
8246 }
8247 }
8248
8249 return type;
8250}
8251
8252static pm_token_type_t
8253lex_global_variable(pm_parser_t *parser) {
8254 if (parser->current.end >= parser->end) {
8255 pm_parser_err_token(parser, &parser->current, PM_ERR_GLOBAL_VARIABLE_BARE);
8256 return PM_TOKEN_GLOBAL_VARIABLE;
8257 }
8258
8259 // True if multiple characters are allowed after the declaration of the
8260 // global variable. Not true when it starts with "$-".
8261 bool allow_multiple = true;
8262
8263 switch (*parser->current.end) {
8264 case '~': // $~: match-data
8265 case '*': // $*: argv
8266 case '$': // $$: pid
8267 case '?': // $?: last status
8268 case '!': // $!: error string
8269 case '@': // $@: error position
8270 case '/': // $/: input record separator
8271 case '\\': // $\: output record separator
8272 case ';': // $;: field separator
8273 case ',': // $,: output field separator
8274 case '.': // $.: last read line number
8275 case '=': // $=: ignorecase
8276 case ':': // $:: load path
8277 case '<': // $<: reading filename
8278 case '>': // $>: default output handle
8279 case '\"': // $": already loaded files
8280 parser->current.end++;
8281 return PM_TOKEN_GLOBAL_VARIABLE;
8282
8283 case '&': // $&: last match
8284 case '`': // $`: string before last match
8285 case '\'': // $': string after last match
8286 case '+': // $+: string matches last paren.
8287 parser->current.end++;
8288 return lex_state_p(parser, PM_LEX_STATE_FNAME) ? PM_TOKEN_GLOBAL_VARIABLE : PM_TOKEN_BACK_REFERENCE;
8289
8290 case '0': {
8291 parser->current.end++;
8292 size_t width;
8293
8294 if ((width = char_is_identifier(parser, parser->current.end, parser->end - parser->current.end)) > 0) {
8295 do {
8296 parser->current.end += width;
8297 } while ((width = char_is_identifier(parser, parser->current.end, parser->end - parser->current.end)) > 0);
8298
8299 // $0 isn't allowed to be followed by anything.
8300 pm_diagnostic_id_t diag_id = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL;
8301 PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->current, diag_id);
8302 }
8303
8304 return PM_TOKEN_GLOBAL_VARIABLE;
8305 }
8306
8307 case '1':
8308 case '2':
8309 case '3':
8310 case '4':
8311 case '5':
8312 case '6':
8313 case '7':
8314 case '8':
8315 case '9':
8316 parser->current.end += pm_strspn_decimal_digit(parser->current.end, parser->end - parser->current.end);
8317 return lex_state_p(parser, PM_LEX_STATE_FNAME) ? PM_TOKEN_GLOBAL_VARIABLE : PM_TOKEN_NUMBERED_REFERENCE;
8318
8319 case '-':
8320 parser->current.end++;
8321 allow_multiple = false;
8323 default: {
8324 size_t width;
8325
8326 if ((width = char_is_identifier(parser, parser->current.end, parser->end - parser->current.end)) > 0) {
8327 do {
8328 parser->current.end += width;
8329 } while (allow_multiple && (width = char_is_identifier(parser, parser->current.end, parser->end - parser->current.end)) > 0);
8330 } else if (pm_char_is_whitespace(peek(parser))) {
8331 // If we get here, then we have a $ followed by whitespace,
8332 // which is not allowed.
8333 pm_parser_err_token(parser, &parser->current, PM_ERR_GLOBAL_VARIABLE_BARE);
8334 } else {
8335 // If we get here, then we have a $ followed by something that
8336 // isn't recognized as a global variable.
8337 pm_diagnostic_id_t diag_id = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL;
8338 size_t width = pm_encoding_char_width(parser->encoding, parser->current.end, parser->end - parser->current.end);
8339 PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), diag_id, (int) (PM_TOKEN_LENGTH(&parser->current) + U32(width)), (const char *) parser->current.start);
8340 }
8341
8342 return PM_TOKEN_GLOBAL_VARIABLE;
8343 }
8344 }
8345}
8346
8351typedef struct {
8352 /* The keyword itself. */
8353 const char *name;
8354
8355 /* The keyword's first, middle, and last bytes and its length, packed by
8356 * PM_KEYWORD_PACK. */
8357 uint32_t packed;
8358
8359 /* The state to transition to when the keyword is lexed. */
8360 pm_lex_state_t state;
8361
8362 /* The token type of the keyword. */
8363 pm_token_type_t type;
8364
8365 /* The token type of the keyword in modifier position, or PM_TOKEN_EOF if
8366 * the keyword has no modifier form. */
8367 pm_token_type_t modifier_type;
8368} pm_keyword_t;
8369
8373#define PM_KEYWORD_LENGTH_MAXIMUM 12
8374
8379#define PM_KEYWORD_PACK(first, middle, last, length) \
8380 ((uint32_t) (first) | ((uint32_t) (middle) << 8) | ((uint32_t) (last) << 16) | ((uint32_t) (length) << 24))
8381
8386#define PM_KEYWORD_SLOT_BITS 6
8387
8401#define PM_KEYWORD_SLOT(packed) ((uint32_t) ((uint32_t) (packed) * 0x75f796ddU) >> (32 - PM_KEYWORD_SLOT_BITS))
8402
8407#define PM_KEYWORD(name, first, middle, last, state, type, modifier_type) \
8408 [PM_KEYWORD_SLOT(PM_KEYWORD_PACK(first, middle, last, sizeof(name) - 1))] = { name, PM_KEYWORD_PACK(first, middle, last, sizeof(name) - 1), state, type, modifier_type }
8409
8415static const pm_keyword_t pm_keywords[1 << PM_KEYWORD_SLOT_BITS] = {
8416 PM_KEYWORD("do", 'd', 'o', 'o', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_DO, PM_TOKEN_EOF),
8417 PM_KEYWORD("if", 'i', 'f', 'f', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_IF, PM_TOKEN_KEYWORD_IF_MODIFIER),
8418 PM_KEYWORD("in", 'i', 'n', 'n', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_IN, PM_TOKEN_EOF),
8419 PM_KEYWORD("or", 'o', 'r', 'r', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_OR, PM_TOKEN_EOF),
8420 PM_KEYWORD("and", 'a', 'n', 'd', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_AND, PM_TOKEN_EOF),
8421 PM_KEYWORD("def", 'd', 'e', 'f', PM_LEX_STATE_FNAME, PM_TOKEN_KEYWORD_DEF, PM_TOKEN_EOF),
8422 PM_KEYWORD("end", 'e', 'n', 'd', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_END, PM_TOKEN_EOF),
8423 PM_KEYWORD("END", 'E', 'N', 'D', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_END_UPCASE, PM_TOKEN_EOF),
8424 PM_KEYWORD("for", 'f', 'o', 'r', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_FOR, PM_TOKEN_EOF),
8425 PM_KEYWORD("nil", 'n', 'i', 'l', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_NIL, PM_TOKEN_EOF),
8426 PM_KEYWORD("not", 'n', 'o', 't', PM_LEX_STATE_ARG, PM_TOKEN_KEYWORD_NOT, PM_TOKEN_EOF),
8427 PM_KEYWORD("case", 'c', 's', 'e', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_CASE, PM_TOKEN_EOF),
8428 PM_KEYWORD("else", 'e', 's', 'e', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_EOF),
8429 PM_KEYWORD("next", 'n', 'x', 't', PM_LEX_STATE_MID, PM_TOKEN_KEYWORD_NEXT, PM_TOKEN_EOF),
8430 PM_KEYWORD("redo", 'r', 'd', 'o', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_REDO, PM_TOKEN_EOF),
8431 PM_KEYWORD("self", 's', 'l', 'f', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_SELF, PM_TOKEN_EOF),
8432 PM_KEYWORD("then", 't', 'e', 'n', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_EOF),
8433 PM_KEYWORD("true", 't', 'u', 'e', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_TRUE, PM_TOKEN_EOF),
8434 PM_KEYWORD("when", 'w', 'e', 'n', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_WHEN, PM_TOKEN_EOF),
8435 PM_KEYWORD("alias", 'a', 'i', 's', PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM, PM_TOKEN_KEYWORD_ALIAS, PM_TOKEN_EOF),
8436 PM_KEYWORD("begin", 'b', 'g', 'n', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_BEGIN, PM_TOKEN_EOF),
8437 PM_KEYWORD("BEGIN", 'B', 'G', 'N', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_BEGIN_UPCASE, PM_TOKEN_EOF),
8438 PM_KEYWORD("break", 'b', 'e', 'k', PM_LEX_STATE_MID, PM_TOKEN_KEYWORD_BREAK, PM_TOKEN_EOF),
8439 PM_KEYWORD("class", 'c', 'a', 's', PM_LEX_STATE_CLASS, PM_TOKEN_KEYWORD_CLASS, PM_TOKEN_EOF),
8440 PM_KEYWORD("elsif", 'e', 's', 'f', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_ELSIF, PM_TOKEN_EOF),
8441 PM_KEYWORD("false", 'f', 'l', 'e', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_FALSE, PM_TOKEN_EOF),
8442 PM_KEYWORD("retry", 'r', 't', 'y', PM_LEX_STATE_END, PM_TOKEN_KEYWORD_RETRY, PM_TOKEN_EOF),
8443 PM_KEYWORD("super", 's', 'p', 'r', PM_LEX_STATE_ARG, PM_TOKEN_KEYWORD_SUPER, PM_TOKEN_EOF),
8444 PM_KEYWORD("undef", 'u', 'd', 'f', PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM, PM_TOKEN_KEYWORD_UNDEF, PM_TOKEN_EOF),
8445 PM_KEYWORD("until", 'u', 't', 'l', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_UNTIL, PM_TOKEN_KEYWORD_UNTIL_MODIFIER),
8446 PM_KEYWORD("while", 'w', 'i', 'e', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_WHILE, PM_TOKEN_KEYWORD_WHILE_MODIFIER),
8447 PM_KEYWORD("yield", 'y', 'e', 'd', PM_LEX_STATE_ARG, PM_TOKEN_KEYWORD_YIELD, PM_TOKEN_EOF),
8448 PM_KEYWORD("ensure", 'e', 'u', 'e', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_EOF),
8449 PM_KEYWORD("module", 'm', 'u', 'e', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_MODULE, PM_TOKEN_EOF),
8450 PM_KEYWORD("rescue", 'r', 'c', 'e', PM_LEX_STATE_MID, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_RESCUE_MODIFIER),
8451 PM_KEYWORD("return", 'r', 'u', 'n', PM_LEX_STATE_MID, PM_TOKEN_KEYWORD_RETURN, PM_TOKEN_EOF),
8452 PM_KEYWORD("unless", 'u', 'e', 's', PM_LEX_STATE_BEG, PM_TOKEN_KEYWORD_UNLESS, PM_TOKEN_KEYWORD_UNLESS_MODIFIER),
8453 PM_KEYWORD("defined?", 'd', 'n', '?', PM_LEX_STATE_ARG, PM_TOKEN_KEYWORD_DEFINED, PM_TOKEN_EOF),
8454 PM_KEYWORD("__LINE__", '_', 'N', '_', PM_LEX_STATE_END, PM_TOKEN_KEYWORD___LINE__, PM_TOKEN_EOF),
8455 PM_KEYWORD("__FILE__", '_', 'L', '_', PM_LEX_STATE_END, PM_TOKEN_KEYWORD___FILE__, PM_TOKEN_EOF),
8456 PM_KEYWORD("__ENCODING__", '_', 'D', '_', PM_LEX_STATE_END, PM_TOKEN_KEYWORD___ENCODING__, PM_TOKEN_EOF)
8457};
8458
8459#undef PM_KEYWORD
8460
8465static PRISM_INLINE const pm_keyword_t *
8466pm_keyword_find(const uint8_t *start, const uint8_t *end) {
8467 ptrdiff_t width = end - start;
8468 if (width < 2 || width > PM_KEYWORD_LENGTH_MAXIMUM) return NULL;
8469
8470 size_t uwidth = (size_t) width;
8471 uint32_t packed = PM_KEYWORD_PACK(start[0], start[uwidth >> 1], start[uwidth - 1], uwidth);
8472
8473 const pm_keyword_t *keyword = &pm_keywords[PM_KEYWORD_SLOT(packed)];
8474 return (keyword->packed == packed && memcmp(start + 1, keyword->name + 1, uwidth - 2) == 0) ? keyword : NULL;
8475}
8476
8481bool
8482pm_keyword_p(const uint8_t *start, const uint8_t *end) {
8483 return pm_keyword_find(start, end) != NULL;
8484}
8485
8486static pm_token_type_t
8487lex_identifier(pm_parser_t *parser, bool previous_command_start) {
8488 /* Lex as far as we can into the current identifier. */
8489 const uint8_t *end = parser->end;
8490 const uint8_t *current_start = parser->current.start;
8491 const uint8_t *current_end = parser->current.end;
8492 bool encoding_changed = parser->encoding_changed;
8493
8494 /* Fast path: scan ASCII identifier bytes using wide operations. Every
8495 * encoding that prism supports is ASCII-compatible, so this holds whether
8496 * or not the encoding has been changed. */
8497 current_end += scan_identifier_ascii(current_end, end);
8498
8499 /* Byte-at-a-time fallback for the tail and any multibyte characters. */
8500 size_t width;
8501 if (encoding_changed) {
8502 while ((width = char_is_identifier(parser, current_end, end - current_end)) > 0) {
8503 current_end += width;
8504 }
8505 } else {
8506 while ((width = char_is_identifier_utf8(current_end, end - current_end)) > 0) {
8507 current_end += width;
8508 }
8509 }
8510
8511 parser->current.end = current_end;
8512
8513 /* Whether the identifier ends in ! or ?, which makes it a method name
8514 * unless it spells a keyword (defined?). */
8515 bool method_name = false;
8516
8517 if (current_end < end) {
8518 if (((current_end + 1 >= end) || (current_end[1] != '=')) && (match(parser, '!') || match(parser, '?'))) {
8519 /* Extend the identifier by the ! or ?, so that the keyword lookup
8520 * below sees the whole name. */
8521 current_end++;
8522
8523 if (
8524 ((lex_state_p(parser, PM_LEX_STATE_LABEL | PM_LEX_STATE_ENDFN) && !previous_command_start) || lex_state_arg_p(parser)) &&
8525 (peek(parser) == ':') && (peek_offset(parser, 1) != ':')
8526 ) {
8527 /* If we are in a position where we can accept a : at the end of
8528 * an identifier, then we will optionally accept it. */
8529 lex_state_set(parser, PM_LEX_STATE_ARG | PM_LEX_STATE_LABELED);
8530 (void) match(parser, ':');
8531
8532 /* A label is a symbol lexed inline rather than through a lex
8533 * mode, so it clears the encoding here. */
8534 parser->explicit_encoding = NULL;
8535 return PM_TOKEN_LABEL;
8536 }
8537
8538 method_name = true;
8539 } else if (lex_state_p(parser, PM_LEX_STATE_FNAME) && peek_offset(parser, 1) != '~' && peek_offset(parser, 1) != '>' && (peek_offset(parser, 1) != '=' || peek_offset(parser, 2) == '>') && match(parser, '=')) {
8540 /* If we are in a position where we can accept a = at the end of an
8541 * identifier, then we will optionally accept it. */
8542 return PM_TOKEN_IDENTIFIER;
8543 } else if (
8544 ((lex_state_p(parser, PM_LEX_STATE_LABEL | PM_LEX_STATE_ENDFN) && !previous_command_start) || lex_state_arg_p(parser)) &&
8545 peek(parser) == ':' && peek_offset(parser, 1) != ':'
8546 ) {
8547 /* If we are in a position where we can accept a : at the end of an
8548 * identifier, then we will optionally accept it. */
8549 lex_state_set(parser, PM_LEX_STATE_ARG | PM_LEX_STATE_LABELED);
8550 (void) match(parser, ':');
8551
8552 /* A label is a symbol lexed inline rather than through a lex
8553 * mode, so it clears the encoding here. */
8554 parser->explicit_encoding = NULL;
8555 return PM_TOKEN_LABEL;
8556 }
8557 }
8558
8559 if (parser->lex_state != PM_LEX_STATE_DOT) {
8560 /* The lex state from before the keyword changes it, which decides
8561 * whether the keyword is in its modifier form and whether `do` is a
8562 * plain name below. */
8563 pm_lex_state_t previous_lex_state = parser->lex_state;
8564 const pm_keyword_t *keyword = pm_keyword_find(current_start, current_end);
8565
8566 if (keyword != NULL) {
8567 if (previous_lex_state & PM_LEX_STATE_FNAME) {
8568 lex_state_set(parser, PM_LEX_STATE_ENDFN);
8569 } else {
8570 lex_state_set(parser, keyword->state);
8571 if (keyword->state == PM_LEX_STATE_BEG) {
8572 parser->command_start = true;
8573 }
8574
8575 if ((keyword->modifier_type != PM_TOKEN_EOF) && !(previous_lex_state & (PM_LEX_STATE_BEG | PM_LEX_STATE_LABELED | PM_LEX_STATE_CLASS))) {
8576 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
8577 return keyword->modifier_type;
8578 }
8579 }
8580
8581 if (keyword->type == PM_TOKEN_KEYWORD_DO) {
8582 /* In FNAME position (a symbol like `:do` or a method name like
8583 * `def do`), `do` is a plain name rather than a block, loop, or
8584 * lambda opener, so none of the discrimination below applies
8585 * and the lambda nesting is left untouched. */
8586 if (previous_lex_state & PM_LEX_STATE_FNAME) {
8587 return PM_TOKEN_KEYWORD_DO;
8588 }
8589 if (parser->enclosure_nesting == parser->lambda_enclosure_nesting) {
8590 /* At the bare nesting level of a lambda literal (no
8591 * delimiter opened since `->`), a `do` opens the lambda
8592 * body. This is a distinct token so that a command in a
8593 * parameter default cannot consume it as its own block
8594 * (`-> a = foo do end` is `->(a = foo) do end`).
8595 *
8596 * Clear the nesting so that no token within the `do`/`end`
8597 * body is considered to be at the beginning of a lambda;
8598 * the parser restores the enclosing value once the lambda
8599 * has been fully parsed. */
8600 parser->lambda_enclosure_nesting = -1;
8601 return PM_TOKEN_KEYWORD_DO_LAMBDA;
8602 }
8603 if (pm_do_loop_stack_p(parser)) {
8604 return PM_TOKEN_KEYWORD_DO_LOOP;
8605 }
8606 if (!pm_accepts_block_stack_p(parser)) {
8607 return PM_TOKEN_KEYWORD_DO_BLOCK;
8608 }
8609 return PM_TOKEN_KEYWORD_DO;
8610 }
8611
8612 return keyword->type;
8613 }
8614 }
8615
8616 if (method_name) {
8617 return PM_TOKEN_METHOD_NAME;
8618 }
8619
8620 if (encoding_changed) {
8621 return pm_encoding_isupper_char(parser->encoding, current_start, end - current_start, parser->version) ? PM_TOKEN_CONSTANT : PM_TOKEN_IDENTIFIER;
8622 }
8623
8624 /* Identifiers usually start with an ASCII byte, for which the uppercase
8625 * check is a simple range comparison. This avoids the call into the
8626 * encoding module for every identifier. */
8627 if (*current_start < 0x80) {
8628 return (*current_start >= 'A' && *current_start <= 'Z') ? PM_TOKEN_CONSTANT : PM_TOKEN_IDENTIFIER;
8629 }
8630 return pm_encoding_utf_8_isupper_char(current_start, end - current_start, parser->version) ? PM_TOKEN_CONSTANT : PM_TOKEN_IDENTIFIER;
8631}
8632
8637static bool
8638current_token_starts_line(pm_parser_t *parser) {
8639 if (parser->current.start == parser->start) return true;
8640
8641 switch (parser->current.start[-1]) {
8642 case '\n':
8643 return true;
8644 case 0xbf:
8645 return (parser->current.start - parser->start == 3 && parser->start[0] == 0xef && parser->start[1] == 0xbb);
8646 default:
8647 return false;
8648 }
8649}
8650
8665static pm_token_type_t
8666lex_interpolation(pm_parser_t *parser, const uint8_t *pound) {
8667 // If there is no content following this #, then we're at the end of
8668 // the string and we can safely return string content.
8669 if (pound + 1 >= parser->end) {
8670 parser->current.end = pound + 1;
8671 return PM_TOKEN_STRING_CONTENT;
8672 }
8673
8674 // Now we'll check against the character that follows the #. If it
8675 // constitutes valid interplation, we'll handle that, otherwise we'll return
8676 // 0.
8677 switch (pound[1]) {
8678 case '@': {
8679 // In this case we may have hit an embedded instance or class variable.
8680 if (pound + 2 >= parser->end) {
8681 parser->current.end = pound + 1;
8682 return PM_TOKEN_STRING_CONTENT;
8683 }
8684
8685 // If we're looking at a @ and there's another @, then we'll skip past the
8686 // second @.
8687 const uint8_t *variable = pound + 2;
8688 if (*variable == '@' && pound + 3 < parser->end) variable++;
8689
8690 if (char_is_identifier_start(parser, variable, parser->end - variable)) {
8691 // At this point we're sure that we've either hit an embedded instance
8692 // or class variable. In this case we'll first need to check if we've
8693 // already consumed content.
8694 if (pound > parser->current.start) {
8695 parser->current.end = pound;
8696 return PM_TOKEN_STRING_CONTENT;
8697 }
8698
8699 // Otherwise we need to return the embedded variable token
8700 // and then switch to the embedded variable lex mode.
8701 lex_mode_push(parser, (pm_lex_mode_t) { .mode = PM_LEX_EMBVAR });
8702 parser->current.end = pound + 1;
8703 return PM_TOKEN_EMBVAR;
8704 }
8705
8706 // If we didn't get a valid interpolation, then this is just regular
8707 // string content. This is like if we get "#@-". In this case the caller
8708 // should keep lexing.
8709 parser->current.end = pound + 1;
8710 return 0;
8711 }
8712 case '$':
8713 // In this case we may have hit an embedded global variable. If there's
8714 // not enough room, then we'll just return string content.
8715 if (pound + 2 >= parser->end) {
8716 parser->current.end = pound + 1;
8717 return PM_TOKEN_STRING_CONTENT;
8718 }
8719
8720 // This is the character that we're going to check to see if it is the
8721 // start of an identifier that would indicate that this is a global
8722 // variable.
8723 const uint8_t *check = pound + 2;
8724
8725 if (pound[2] == '-') {
8726 if (pound + 3 >= parser->end) {
8727 parser->current.end = pound + 2;
8728 return PM_TOKEN_STRING_CONTENT;
8729 }
8730
8731 check++;
8732 }
8733
8734 // If the character that we're going to check is the start of an
8735 // identifier, or we don't have a - and the character is a decimal number
8736 // or a global name punctuation character, then we've hit an embedded
8737 // global variable.
8738 if (
8739 char_is_identifier_start(parser, check, parser->end - check) ||
8740 (pound[2] != '-' && (pm_char_is_decimal_digit(pound[2]) || char_is_global_name_punctuation(pound[2])))
8741 ) {
8742 // In this case we've hit an embedded global variable. First check to
8743 // see if we've already consumed content. If we have, then we need to
8744 // return that content as string content first.
8745 if (pound > parser->current.start) {
8746 parser->current.end = pound;
8747 return PM_TOKEN_STRING_CONTENT;
8748 }
8749
8750 // Otherwise, we need to return the embedded variable token and switch
8751 // to the embedded variable lex mode.
8752 lex_mode_push(parser, (pm_lex_mode_t) { .mode = PM_LEX_EMBVAR });
8753 parser->current.end = pound + 1;
8754 return PM_TOKEN_EMBVAR;
8755 }
8756
8757 // In this case we've hit a #$ that does not indicate a global variable.
8758 // In this case we'll continue lexing past it.
8759 parser->current.end = pound + 1;
8760 return 0;
8761 case '{':
8762 // In this case it's the start of an embedded expression. If we have
8763 // already consumed content, then we need to return that content as string
8764 // content first.
8765 if (pound > parser->current.start) {
8766 parser->current.end = pound;
8767 return PM_TOKEN_STRING_CONTENT;
8768 }
8769
8770 parser->enclosure_nesting++;
8771
8772 // Otherwise we'll skip past the #{ and begin lexing the embedded
8773 // expression.
8774 lex_mode_push(parser, (pm_lex_mode_t) { .mode = PM_LEX_EMBEXPR });
8775 parser->current.end = pound + 2;
8776 parser->command_start = true;
8777 pm_enclosure_frame_push(parser);
8778 return PM_TOKEN_EMBEXPR_BEGIN;
8779 default:
8780 // In this case we've hit a # that doesn't constitute interpolation. We'll
8781 // mark that by returning the not provided token type. This tells the
8782 // consumer to keep lexing forward.
8783 parser->current.end = pound + 1;
8784 return 0;
8785 }
8786}
8787
8788static const uint8_t PM_ESCAPE_FLAG_NONE = 0x0;
8789static const uint8_t PM_ESCAPE_FLAG_CONTROL = 0x1;
8790static const uint8_t PM_ESCAPE_FLAG_META = 0x2;
8791static const uint8_t PM_ESCAPE_FLAG_SINGLE = 0x4;
8792static const uint8_t PM_ESCAPE_FLAG_REGEXP = 0x8;
8793
8797static const bool ascii_printable_chars[] = {
8798 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,
8799 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
8800 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
8801 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
8802 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
8803 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1,
8804 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
8805 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0
8806};
8807
8808static PRISM_INLINE bool
8809char_is_ascii_printable(const uint8_t b) {
8810 return (b < 0x80) && ascii_printable_chars[b];
8811}
8812
8817static PRISM_INLINE uint8_t
8818escape_hexadecimal_digit(const uint8_t value) {
8819 return (uint8_t) ((value <= '9') ? (value - '0') : (value & 0x7) + 9);
8820}
8821
8827static PRISM_INLINE uint32_t
8828escape_unicode(pm_parser_t *parser, const uint8_t *string, size_t length, const pm_location_t *error_location, const uint8_t flags) {
8829 uint32_t value = 0;
8830 for (size_t index = 0; index < length; index++) {
8831 if (index != 0) value <<= 4;
8832 value |= escape_hexadecimal_digit(string[index]);
8833 }
8834
8835 // Here we're going to verify that the value is actually a valid Unicode
8836 // codepoint and not a surrogate pair.
8837 if (value >= 0xD800 && value <= 0xDFFF) {
8838 if (flags & PM_ESCAPE_FLAG_REGEXP) {
8839 // In regexp context, defer the error to regexp encoding
8840 // validation where we can produce a regexp-specific message.
8841 } else if (error_location != NULL) {
8842 pm_parser_err(parser, error_location->start, error_location->length, PM_ERR_ESCAPE_INVALID_UNICODE);
8843 } else {
8844 pm_parser_err(parser, U32(string - parser->start), U32(length), PM_ERR_ESCAPE_INVALID_UNICODE);
8845 }
8846 return 0xFFFD;
8847 }
8848
8849 return value;
8850}
8851
8855static PRISM_INLINE uint8_t
8856escape_byte(uint8_t value, const uint8_t flags) {
8857 if (flags & PM_ESCAPE_FLAG_CONTROL) value &= 0x9f;
8858 if (flags & PM_ESCAPE_FLAG_META) value |= 0x80;
8859 return value;
8860}
8861
8865static PRISM_INLINE void
8866escape_write_unicode(pm_parser_t *parser, pm_buffer_t *buffer, const uint8_t flags, const uint8_t *start, const uint8_t *end, uint32_t value) {
8867 // \u escape sequences in string-like structures implicitly change the
8868 // encoding to UTF-8 if they are >= 0x80 or if they are used in a character
8869 // literal.
8870 if (value >= 0x80 || flags & PM_ESCAPE_FLAG_SINGLE) {
8871 if (parser->explicit_encoding != NULL && parser->explicit_encoding != PM_ENCODING_UTF_8_ENTRY) {
8872 if (flags & PM_ESCAPE_FLAG_REGEXP) {
8873 // In regexp context, suppress this error — the regexp encoding
8874 // validation will produce a more specific error message.
8875 } else {
8876 PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(end - start), PM_ERR_MIXED_ENCODING, parser->explicit_encoding->name);
8877 }
8878 }
8879
8880 parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY;
8881 }
8882
8883 if (!(flags & PM_ESCAPE_FLAG_REGEXP) && !pm_buffer_append_unicode_codepoint(buffer, value)) {
8884 pm_parser_err(parser, U32(start - parser->start), U32(end - start), PM_ERR_ESCAPE_INVALID_UNICODE);
8885 pm_buffer_append_byte(buffer, 0xEF);
8886 pm_buffer_append_byte(buffer, 0xBF);
8887 pm_buffer_append_byte(buffer, 0xBD);
8888 }
8889}
8890
8897static PRISM_INLINE void
8898escape_write_byte_encoded(pm_parser_t *parser, pm_buffer_t *buffer, const uint8_t flags, uint8_t byte) {
8899 if (byte >= 0x80) {
8900 if (parser->explicit_encoding != NULL && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY && parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
8901 if (flags & PM_ESCAPE_FLAG_REGEXP) {
8902 // In regexp context, suppress this error — the regexp encoding
8903 // validation will produce a more specific error message.
8904 } else {
8905 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_MIXED_ENCODING, parser->encoding->name);
8906 }
8907 }
8908
8909 parser->explicit_encoding = parser->encoding;
8910 }
8911
8912 if (!(flags & PM_ESCAPE_FLAG_REGEXP)) pm_buffer_append_byte(buffer, byte);
8913}
8914
8930static PRISM_INLINE void
8931escape_write_byte(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags, uint8_t byte) {
8932 if (flags & PM_ESCAPE_FLAG_REGEXP) {
8933 pm_buffer_append_format(buffer, "\\x%02X", byte);
8934 }
8935
8936 escape_write_byte_encoded(parser, buffer, flags, byte);
8937}
8938
8948static PRISM_INLINE void
8949parser_flush_heredoc_end(pm_parser_t *parser) {
8950 PRISM_ASSERT(parser->heredoc_end <= parser->end);
8951 parser->next_start = parser->heredoc_end;
8952 parser->heredoc_end = NULL;
8953}
8954
8964static PRISM_INLINE void
8965escape_consume_newline(pm_parser_t *parser, uint32_t width) {
8966 if (parser->heredoc_end == NULL) {
8967 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current) + width);
8968 } else {
8969 parser_flush_heredoc_end(parser);
8970 }
8971}
8972
8977static PRISM_INLINE void
8978escape_write_invalid(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags) {
8979 escape_write_byte(parser, buffer, flags, escape_byte(0, flags));
8980}
8981
8986static PRISM_INLINE void
8987escape_skip_char(pm_parser_t *parser) {
8988 if (parser->current.end >= parser->end) return;
8989
8990 size_t eol_length = match_eol(parser);
8991 if (eol_length > 0) {
8992 escape_consume_newline(parser, U32(eol_length));
8993 parser->current.end += eol_length;
8994 return;
8995 }
8996
8997 size_t width = pm_encoding_char_width(parser->encoding, parser->current.end, parser->end - parser->current.end);
8998 parser->current.end += (width == 0 ? 1 : width);
8999}
9000
9004static PRISM_INLINE void
9005escape_write_escape_encoded(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags) {
9006 size_t width;
9007 if (parser->encoding_changed) {
9008 width = pm_encoding_char_width(parser->encoding, parser->current.end, parser->end - parser->current.end);
9009 } else {
9010 width = pm_encoding_utf_8_char_width(parser->current.end, parser->end - parser->current.end);
9011 }
9012
9013 if (width == 1) {
9014 if (*parser->current.end == '\n') escape_consume_newline(parser, 1);
9015 escape_write_byte(parser, buffer, flags, escape_byte(*parser->current.end++, flags));
9016 } else if (width > 1) {
9017 // Valid multibyte character. Just ignore the escape.
9018 pm_buffer_append_bytes(buffer, parser->current.end, width);
9019 parser->current.end += width;
9020 } else {
9021 // Assume the next character wasn't meant to be part of this escape
9022 // sequence since it is invalid. Add an error and move on.
9023 parser->current.end++;
9024 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL);
9025 }
9026}
9027
9033static void
9034escape_read_warn(pm_parser_t *parser, uint8_t flags, uint8_t flag, const char *type) {
9035#define FLAG(value) ((value & PM_ESCAPE_FLAG_CONTROL) ? "\\C-" : (value & PM_ESCAPE_FLAG_META) ? "\\M-" : "")
9036
9037 PM_PARSER_WARN_TOKEN_FORMAT(
9038 parser,
9039 &parser->current,
9040 PM_WARN_INVALID_CHARACTER,
9041 FLAG(flags),
9042 FLAG(flag),
9043 type
9044 );
9045
9046#undef FLAG
9047}
9048
9055static PRISM_INLINE uint8_t
9056escape_consume_modified_byte(pm_parser_t *parser, uint8_t byte, uint8_t flags, uint8_t flag) {
9057 size_t eol_length = match_eol(parser);
9058
9059 if (eol_length > 0) {
9060 escape_consume_newline(parser, U32(eol_length));
9061 parser->current.end += eol_length;
9062 byte = '\n';
9063 } else {
9064 parser->current.end++;
9065 }
9066
9067 switch (byte) {
9068 case '\n': escape_read_warn(parser, flags, flag, "\\n"); break;
9069 case '\t': escape_read_warn(parser, flags, flag, "\\t"); break;
9070 case '\v': escape_read_warn(parser, flags, flag, "\\v"); break;
9071 case '\f': escape_read_warn(parser, flags, flag, "\\f"); break;
9072 case '\r': escape_read_warn(parser, flags, flag, "\\r"); break;
9073 default: break;
9074 }
9075
9076 return byte;
9077}
9078
9094static void
9095escape_read(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags) {
9096 uint8_t peeked;
9097
9098 escape_read_next:
9099 switch (peeked = peek(parser)) {
9100 case '\\': {
9101 parser->current.end++;
9102 escape_write_byte(parser, buffer, flags, escape_byte('\\', flags));
9103 return;
9104 }
9105 case '\'': {
9106 parser->current.end++;
9107 escape_write_byte(parser, buffer, flags, escape_byte('\'', flags));
9108 return;
9109 }
9110 case 'a': {
9111 parser->current.end++;
9112 escape_write_byte(parser, buffer, flags, escape_byte('\a', flags));
9113 return;
9114 }
9115 case 'b': {
9116 parser->current.end++;
9117 escape_write_byte(parser, buffer, flags, escape_byte('\b', flags));
9118 return;
9119 }
9120 case 'e': {
9121 parser->current.end++;
9122 escape_write_byte(parser, buffer, flags, escape_byte('\033', flags));
9123 return;
9124 }
9125 case 'f': {
9126 parser->current.end++;
9127 escape_write_byte(parser, buffer, flags, escape_byte('\f', flags));
9128 return;
9129 }
9130 case 'n': {
9131 parser->current.end++;
9132 escape_write_byte(parser, buffer, flags, escape_byte('\n', flags));
9133 return;
9134 }
9135 case 'r': {
9136 parser->current.end++;
9137 escape_write_byte(parser, buffer, flags, escape_byte('\r', flags));
9138 return;
9139 }
9140 case 's': {
9141 parser->current.end++;
9142 escape_write_byte(parser, buffer, flags, escape_byte(' ', flags));
9143 return;
9144 }
9145 case 't': {
9146 parser->current.end++;
9147 escape_write_byte(parser, buffer, flags, escape_byte('\t', flags));
9148 return;
9149 }
9150 case 'v': {
9151 parser->current.end++;
9152 escape_write_byte(parser, buffer, flags, escape_byte('\v', flags));
9153 return;
9154 }
9155 case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': {
9156 uint8_t value = (uint8_t) (*parser->current.end - '0');
9157 parser->current.end++;
9158
9159 if (pm_char_is_octal_digit(peek(parser))) {
9160 value = ((uint8_t) (value << 3)) | ((uint8_t) (*parser->current.end - '0'));
9161 parser->current.end++;
9162
9163 if (pm_char_is_octal_digit(peek(parser))) {
9164 value = ((uint8_t) (value << 3)) | ((uint8_t) (*parser->current.end - '0'));
9165 parser->current.end++;
9166 }
9167 }
9168
9169 value = escape_byte(value, flags);
9170 escape_write_byte(parser, buffer, flags, value);
9171 return;
9172 }
9173 case 'x': {
9174 const uint8_t *start = parser->current.end - 1;
9175
9176 parser->current.end++;
9177 uint8_t byte = peek(parser);
9178
9179 if (pm_char_is_hexadecimal_digit(byte)) {
9180 uint8_t value = escape_hexadecimal_digit(byte);
9181 parser->current.end++;
9182
9183 byte = peek(parser);
9184 if (pm_char_is_hexadecimal_digit(byte)) {
9185 value = (uint8_t) ((value << 4) | escape_hexadecimal_digit(byte));
9186 parser->current.end++;
9187 }
9188
9189 value = escape_byte(value, flags);
9190 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9191 if (flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META)) {
9192 pm_buffer_append_format(buffer, "\\x%02X", value);
9193 } else {
9194 pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end - start));
9195 }
9196 }
9197
9198 escape_write_byte_encoded(parser, buffer, flags, value);
9199 } else {
9200 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_HEXADECIMAL);
9201
9202 /* A malformed hex escape in a regular expression stands for
9203 * nothing unless another escape wraps it. */
9204 if (!(flags & PM_ESCAPE_FLAG_REGEXP) || (flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META))) {
9205 escape_write_invalid(parser, buffer, flags);
9206 }
9207 }
9208
9209 return;
9210 }
9211 case 'u': {
9212 const uint8_t *start = parser->current.end - 1;
9213 parser->current.end++;
9214
9215 if (parser->current.end == parser->end) {
9216 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9217 /* The regular expression is validated even though it is
9218 * not terminated, which reports this escape. */
9219 pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end - start));
9220 } else {
9221 const uint8_t *start = parser->current.end - 2;
9222 PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start);
9223 }
9224 } else if (peek(parser) == '{') {
9225 const uint8_t *unicode_codepoints_start = parser->current.end - 2;
9226 parser->current.end++;
9227
9228 size_t whitespace;
9229 while (true) {
9230 if ((whitespace = pm_strspn_inline_whitespace(parser->current.end, parser->end - parser->current.end)) > 0) {
9231 parser->current.end += whitespace;
9232 } else if (peek(parser) == '\\' && peek_offset(parser, 1) == 'n') {
9233 // This is super hacky, but it gets us nicer error
9234 // messages because we can still pass it off to the
9235 // regular expression engine even if we hit an
9236 // unterminated regular expression.
9237 parser->current.end += 2;
9238 } else {
9239 break;
9240 }
9241 }
9242
9243 const uint8_t *extra_codepoints_start = NULL;
9244 int codepoints_count = 0;
9245
9246 while ((parser->current.end < parser->end) && (*parser->current.end != '}')) {
9247 const uint8_t *unicode_start = parser->current.end;
9248 size_t hexadecimal_length = pm_strspn_hexadecimal_digit(parser->current.end, parser->end - parser->current.end);
9249
9250 if (hexadecimal_length > 6) {
9251 /* A code point has at most 6 hexadecimal digits. In a
9252 * regular expression the escape may be in a comment,
9253 * so the code point is checked once its escapes are
9254 * read. */
9255 if (!(flags & PM_ESCAPE_FLAG_REGEXP)) {
9256 pm_parser_err(parser, U32(unicode_start - parser->start), U32(hexadecimal_length), PM_ERR_ESCAPE_INVALID_UNICODE_LONG);
9257 }
9258 } else if (hexadecimal_length == 0) {
9259 // there are not hexadecimal characters
9260
9261 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9262 // If this is a regular expression, we are going to
9263 // let the regular expression engine handle this
9264 // error instead of us because we don't know at this
9265 // point if we're inside a comment in /x mode.
9266 pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end - start));
9267 } else {
9268 pm_parser_err(parser, PM_TOKEN_END(parser, &parser->current), 0, PM_ERR_ESCAPE_INVALID_UNICODE);
9269 pm_parser_err(parser, PM_TOKEN_END(parser, &parser->current), 0, PM_ERR_ESCAPE_INVALID_UNICODE_TERM);
9270 }
9271
9272 return;
9273 }
9274
9275 parser->current.end += hexadecimal_length;
9276 codepoints_count++;
9277 if (flags & PM_ESCAPE_FLAG_SINGLE && codepoints_count == 2) {
9278 extra_codepoints_start = unicode_start;
9279 }
9280
9281 uint32_t value = escape_unicode(parser, unicode_start, hexadecimal_length, NULL, flags);
9282 escape_write_unicode(parser, buffer, flags, unicode_start, parser->current.end, value);
9283
9284 parser->current.end += pm_strspn_inline_whitespace(parser->current.end, parser->end - parser->current.end);
9285 }
9286
9287 // ?\u{nnnn} character literal should contain only one codepoint
9288 // and cannot be like ?\u{nnnn mmmm}.
9289 if (flags & PM_ESCAPE_FLAG_SINGLE && codepoints_count > 1) {
9290 pm_parser_err(parser, U32(extra_codepoints_start - parser->start), U32(parser->current.end - extra_codepoints_start), PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL);
9291 }
9292
9293 if (parser->current.end == parser->end) {
9294 /* The regular expression is validated even though it is
9295 * not terminated, which reports this escape. */
9296 if (!(flags & PM_ESCAPE_FLAG_REGEXP)) {
9297 PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->current.end - start), start);
9298 }
9299 } else if (peek(parser) == '}') {
9300 parser->current.end++;
9301 } else {
9302 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9303 // If this is a regular expression, we are going to let
9304 // the regular expression engine handle this error
9305 // instead of us because we don't know at this point if
9306 // we're inside a comment in /x mode.
9307 pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end - start));
9308 } else {
9309 pm_parser_err(parser, U32(unicode_codepoints_start - parser->start), U32(parser->current.end - unicode_codepoints_start), PM_ERR_ESCAPE_INVALID_UNICODE_TERM);
9310 }
9311 }
9312
9313 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9314 pm_buffer_append_bytes(buffer, unicode_codepoints_start, (size_t) (parser->current.end - unicode_codepoints_start));
9315 }
9316 } else {
9317 size_t length = pm_strspn_hexadecimal_digit(parser->current.end, MIN(parser->end - parser->current.end, 4));
9318
9319 if (length == 0) {
9320 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9321 pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end - start));
9322 } else {
9323 const uint8_t *start = parser->current.end - 2;
9324 PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start);
9325 }
9326 } else if (length == 4) {
9327 uint32_t value = escape_unicode(parser, parser->current.end, 4, NULL, flags);
9328
9329 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9330 pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end + 4 - start));
9331 }
9332
9333 escape_write_unicode(parser, buffer, flags, start, parser->current.end + 4, value);
9334 parser->current.end += 4;
9335 } else {
9336 parser->current.end += length;
9337
9338 if (flags & PM_ESCAPE_FLAG_REGEXP) {
9339 // If this is a regular expression, we are going to let
9340 // the regular expression engine handle this error
9341 // instead of us.
9342 pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end - start));
9343 } else {
9344 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_UNICODE);
9345 }
9346 }
9347 }
9348
9349 return;
9350 }
9351 case 'C': {
9352 parser->current.end++;
9353
9354 if (peek(parser) != '-') {
9355 escape_skip_char(parser);
9356 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL);
9357 escape_write_invalid(parser, buffer, flags);
9358 return;
9359 }
9360
9361 /* After its dash, a \C- escape is read the same as a \c escape,
9362 * starting by skipping the dash in place of the c. */
9364 }
9365 case 'c': {
9366 parser->current.end++;
9367
9368 if (flags & PM_ESCAPE_FLAG_CONTROL) {
9369 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT);
9370 escape_write_invalid(parser, buffer, flags);
9371 return;
9372 }
9373
9374 if (parser->current.end == parser->end) {
9375 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL);
9376 escape_write_invalid(parser, buffer, flags);
9377 return;
9378 }
9379
9380 uint8_t peeked = peek(parser);
9381 switch (peeked) {
9382 case '?': {
9383 parser->current.end++;
9384 escape_write_byte(parser, buffer, flags, escape_byte(0x7f, flags));
9385 return;
9386 }
9387 case '\\':
9388 parser->current.end++;
9389
9390 if (match(parser, 'u') || match(parser, 'U')) {
9391 pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER);
9392 escape_write_invalid(parser, buffer, flags);
9393 return;
9394 }
9395
9396 flags |= PM_ESCAPE_FLAG_CONTROL;
9397 goto escape_read_next;
9398 case ' ':
9399 parser->current.end++;
9400 escape_read_warn(parser, flags, PM_ESCAPE_FLAG_CONTROL, "\\s");
9401 escape_write_byte(parser, buffer, flags, escape_byte(peeked, flags | PM_ESCAPE_FLAG_CONTROL));
9402 return;
9403 default: {
9404 if (!char_is_ascii_printable(peeked)) {
9405 escape_skip_char(parser);
9406 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_CONTROL);
9407 escape_write_invalid(parser, buffer, flags);
9408 return;
9409 }
9410
9411 escape_write_byte(parser, buffer, flags, escape_byte(escape_consume_modified_byte(parser, peeked, flags, 0), flags | PM_ESCAPE_FLAG_CONTROL));
9412 return;
9413 }
9414 }
9415 }
9416 case 'M': {
9417 parser->current.end++;
9418
9419 if (flags & PM_ESCAPE_FLAG_META) {
9420 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META_REPEAT);
9421 escape_write_invalid(parser, buffer, flags);
9422 return;
9423 }
9424
9425 if (peek(parser) != '-') {
9426 escape_skip_char(parser);
9427 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META);
9428 escape_write_invalid(parser, buffer, flags);
9429 return;
9430 }
9431
9432 parser->current.end++;
9433 if (parser->current.end == parser->end) {
9434 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META);
9435 escape_write_invalid(parser, buffer, flags);
9436 return;
9437 }
9438
9439 uint8_t peeked = peek(parser);
9440 switch (peeked) {
9441 case '\\':
9442 parser->current.end++;
9443
9444 if (match(parser, 'u') || match(parser, 'U')) {
9445 pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER);
9446 escape_write_invalid(parser, buffer, flags);
9447 return;
9448 }
9449
9450 flags |= PM_ESCAPE_FLAG_META;
9451 goto escape_read_next;
9452 case ' ':
9453 parser->current.end++;
9454 escape_read_warn(parser, flags, PM_ESCAPE_FLAG_META, "\\s");
9455 escape_write_byte(parser, buffer, flags, escape_byte(peeked, flags | PM_ESCAPE_FLAG_META));
9456 return;
9457 default:
9458 if (!char_is_ascii_printable(peeked)) {
9459 escape_skip_char(parser);
9460 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META);
9461 escape_write_invalid(parser, buffer, flags);
9462 return;
9463 }
9464
9465 escape_write_byte(parser, buffer, flags, escape_byte(escape_consume_modified_byte(parser, peeked, flags & ((uint8_t) ~PM_ESCAPE_FLAG_CONTROL), PM_ESCAPE_FLAG_META), flags | PM_ESCAPE_FLAG_META));
9466 return;
9467 }
9468 }
9469 case '\r': {
9470 if (peek_offset(parser, 1) == '\n') {
9471 escape_consume_newline(parser, 2);
9472 parser->current.end += 2;
9473 escape_write_byte(parser, buffer, flags, escape_byte('\n', flags));
9474 return;
9475 }
9477 }
9478 default: {
9479 /* A meta or control escape may wrap an escape of any ASCII
9480 * character, including the control characters that it may not
9481 * wrap directly, but not one of a character that is not ASCII. */
9482 bool wrapped = (flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META)) != 0;
9483
9484 if (wrapped && parser->current.end < parser->end && peeked >= 0x80) {
9485 escape_skip_char(parser);
9486 pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_META);
9487 escape_write_invalid(parser, buffer, flags);
9488 return;
9489 }
9490
9491 if (parser->current.end < parser->end) {
9492 escape_write_escape_encoded(parser, buffer, flags);
9493 } else {
9494 pm_parser_err_current(parser, PM_ERR_INVALID_ESCAPE_CHARACTER);
9495 if (wrapped) escape_write_invalid(parser, buffer, flags);
9496 }
9497 return;
9498 }
9499 }
9500}
9501
9527static pm_token_type_t
9528lex_question_mark(pm_parser_t *parser) {
9529 if (lex_state_end_p(parser)) {
9530 lex_state_set(parser, PM_LEX_STATE_BEG);
9531 return PM_TOKEN_QUESTION_MARK;
9532 }
9533
9534 /*
9535 * A literal takes its encoding from its own contents. Literals that push a
9536 * lex mode clear this in lex_mode_push_*; a character literal is lexed
9537 * inline, so it clears the encoding here.
9538 */
9539 parser->explicit_encoding = NULL;
9540
9541 if (parser->current.end >= parser->end) {
9542 pm_parser_err_current(parser, PM_ERR_INCOMPLETE_QUESTION_MARK);
9543 pm_string_shared_init(&parser->current_string, parser->current.start + 1, parser->current.end);
9544 return PM_TOKEN_CHARACTER_LITERAL;
9545 }
9546
9547 if (pm_char_is_whitespace(*parser->current.end)) {
9548 lex_state_set(parser, PM_LEX_STATE_BEG);
9549 return PM_TOKEN_QUESTION_MARK;
9550 }
9551
9552 lex_state_set(parser, PM_LEX_STATE_BEG);
9553
9554 if (match(parser, '\\')) {
9555 lex_state_set(parser, PM_LEX_STATE_END);
9556
9557 pm_buffer_t buffer;
9558 pm_buffer_init(&buffer, 3);
9559
9560 escape_read(parser, &buffer, PM_ESCAPE_FLAG_SINGLE);
9561
9562 // Copy buffer data into the arena and free the heap buffer.
9563 void *arena_data = pm_arena_memdup(parser->arena, buffer.value, buffer.length, PRISM_ALIGNOF(uint8_t));
9564 pm_string_init(&parser->current_string, (const uint8_t *) arena_data, buffer.length);
9565 pm_buffer_cleanup(&buffer);
9566
9567 return PM_TOKEN_CHARACTER_LITERAL;
9568 } else {
9569 size_t encoding_width = pm_encoding_char_width(parser->encoding, parser->current.end, parser->end - parser->current.end);
9570
9571 /* A ? followed by two identifier characters, as in `a ?bc : d`, is
9572 * the ? of a ternary operator rather than a character literal. Any
9573 * character that is not ASCII is an identifier character. */
9574 if (
9575 !char_is_identifier(parser, parser->current.end, parser->end - parser->current.end) ||
9576 (
9577 (parser->current.end + encoding_width >= parser->end) ||
9578 !char_is_identifier(parser, parser->current.end + encoding_width, parser->end - (parser->current.end + encoding_width))
9579 )
9580 ) {
9581 lex_state_set(parser, PM_LEX_STATE_END);
9582 parser->current.end += encoding_width;
9583 pm_string_shared_init(&parser->current_string, parser->current.start + 1, parser->current.end);
9584 return PM_TOKEN_CHARACTER_LITERAL;
9585 }
9586 }
9587
9588 return PM_TOKEN_QUESTION_MARK;
9589}
9590
9595static pm_token_type_t
9596lex_at_variable(pm_parser_t *parser) {
9597 pm_token_type_t type = match(parser, '@') ? PM_TOKEN_CLASS_VARIABLE : PM_TOKEN_INSTANCE_VARIABLE;
9598 const uint8_t *end = parser->end;
9599
9600 size_t width;
9601 if ((width = char_is_identifier_start(parser, parser->current.end, end - parser->current.end)) > 0) {
9602 parser->current.end += width;
9603
9604 while ((width = char_is_identifier(parser, parser->current.end, end - parser->current.end)) > 0) {
9605 parser->current.end += width;
9606 }
9607 } else if (parser->current.end < end && pm_char_is_decimal_digit(*parser->current.end)) {
9608 pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE;
9609 if (parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3) {
9610 diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3 : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3;
9611 }
9612
9613 size_t width = pm_encoding_char_width(parser->encoding, parser->current.end, end - parser->current.end);
9614 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start);
9615 } else {
9616 pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_CLASS_VARIABLE_BARE : PM_ERR_INSTANCE_VARIABLE_BARE;
9617 pm_parser_err_token(parser, &parser->current, diag_id);
9618 }
9619
9620 // If we're lexing an embedded variable, then we need to pop back into the
9621 // parent lex context.
9622 if (parser->lex_modes.current->mode == PM_LEX_EMBVAR) {
9623 lex_mode_pop(parser);
9624 }
9625
9626 return type;
9627}
9628
9632static PRISM_INLINE void
9633parser_lex_callback(pm_parser_t *parser) {
9634 if (parser->lex_callback.callback) {
9635 parser->lex_callback.callback(parser, &parser->current, parser->lex_callback.data);
9636 }
9637}
9638
9643parser_comment(pm_parser_t *parser, pm_comment_type_t type) {
9644 pm_comment_t *comment = PM_ARENA_ALLOC(&parser->metadata_arena, pm_comment_t);
9645
9646 *comment = (pm_comment_t) {
9647 .type = type,
9648 .location = TOK2LOC(parser, &parser->current)
9649 };
9650
9651 return comment;
9652}
9653
9659static pm_token_type_t
9660lex_embdoc(pm_parser_t *parser) {
9661 // First, lex out the EMBDOC_BEGIN token.
9662 const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end);
9663
9664 if (newline == NULL) {
9665 parser->current.end = parser->end;
9666 } else {
9667 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(newline - parser->start + 1));
9668 parser->current.end = newline + 1;
9669 }
9670
9671 parser->current.type = PM_TOKEN_EMBDOC_BEGIN;
9672 parser_lex_callback(parser);
9673
9674 // Now, create a comment that is going to be attached to the parser.
9675 const uint8_t *comment_start = parser->current.start;
9676 pm_comment_t *comment = parser_comment(parser, PM_COMMENT_EMBDOC);
9677
9678 // Now, loop until we find the end of the embedded documentation or the end
9679 // of the file.
9680 while (parser->current.end < parser->end) {
9681 parser->current.start = parser->current.end;
9682
9683 // If we've hit the end of the embedded documentation then we'll return
9684 // that token here.
9685 if (
9686 (parser->end - parser->current.end >= 4) &&
9687 (memcmp(parser->current.end, "=end", 4) == 0) &&
9688 (
9689 (parser->current.end + 4 == parser->end) || // end of file
9690 pm_char_is_whitespace(parser->current.end[4]) || // whitespace
9691 (parser->current.end[4] == '\0') || // NUL or end of script
9692 (parser->current.end[4] == '\004') || // ^D
9693 (parser->current.end[4] == '\032') // ^Z
9694 )
9695 ) {
9696 const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end);
9697
9698 if (newline == NULL) {
9699 parser->current.end = parser->end;
9700 } else {
9701 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(newline - parser->start + 1));
9702 parser->current.end = newline + 1;
9703 }
9704
9705 parser->current.type = PM_TOKEN_EMBDOC_END;
9706 parser_lex_callback(parser);
9707
9708 comment->location.length = (uint32_t) (parser->current.end - comment_start);
9709 pm_list_append(&parser->comment_list, (pm_list_node_t *) comment);
9710
9711 return PM_TOKEN_EMBDOC_END;
9712 }
9713
9714 // Otherwise, we'll parse until the end of the line and return a line of
9715 // embedded documentation.
9716 const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end);
9717
9718 if (newline == NULL) {
9719 parser->current.end = parser->end;
9720 } else {
9721 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(newline - parser->start + 1));
9722 parser->current.end = newline + 1;
9723 }
9724
9725 parser->current.type = PM_TOKEN_EMBDOC_LINE;
9726 parser_lex_callback(parser);
9727 }
9728
9729 pm_parser_err_current(parser, PM_ERR_EMBDOC_TERM);
9730
9731 comment->location.length = (uint32_t) (parser->current.end - comment_start);
9732 pm_list_append(&parser->comment_list, (pm_list_node_t *) comment);
9733
9734 return PM_TOKEN_EOF;
9735}
9736
9742static PRISM_INLINE void
9743parser_lex_ignored_newline(pm_parser_t *parser) {
9744 parser->current.type = PM_TOKEN_IGNORED_NEWLINE;
9745 parser_lex_callback(parser);
9746}
9747
9751static bool
9752parser_end_of_line_p(const pm_parser_t *parser) {
9753 const uint8_t *cursor = parser->current.end;
9754
9755 while (cursor < parser->end && *cursor != '\n' && *cursor != '#') {
9756 if (!pm_char_is_inline_whitespace(*cursor++)) return false;
9757 }
9758
9759 return true;
9760}
9761
9780typedef struct {
9786
9791 const uint8_t *cursor;
9793
9797static PRISM_INLINE void
9798pm_token_buffer_push_byte(pm_token_buffer_t *token_buffer, uint8_t byte) {
9799 pm_buffer_append_byte(&token_buffer->buffer, byte);
9800}
9801
9805static PRISM_INLINE size_t
9806parser_char_width(const pm_parser_t *parser) {
9807 size_t width;
9808 if (parser->encoding_changed) {
9809 width = pm_encoding_char_width(parser->encoding, parser->current.end, parser->end - parser->current.end);
9810 } else {
9811 width = pm_encoding_utf_8_char_width(parser->current.end, parser->end - parser->current.end);
9812 }
9813
9814 // TODO: If the character is invalid in the given encoding, then we'll just
9815 // push one byte into the buffer. This should actually be an error.
9816 return (width == 0 ? 1 : width);
9817}
9818
9822static void
9823pm_token_buffer_push_escaped(pm_token_buffer_t *token_buffer, pm_parser_t *parser) {
9824 size_t width = parser_char_width(parser);
9825 pm_buffer_append_bytes(&token_buffer->buffer, parser->current.end, width);
9826 parser->current.end += width;
9827}
9828
9835static PRISM_INLINE void
9836pm_token_buffer_copy(pm_parser_t *parser, pm_token_buffer_t *token_buffer) {
9837 // Copy buffer data into the arena and free the heap buffer.
9838 size_t len = pm_buffer_length(&token_buffer->buffer);
9839 void *arena_data = pm_arena_memdup(parser->arena, pm_buffer_value(&token_buffer->buffer), len, PRISM_ALIGNOF(uint8_t));
9840 pm_string_init(&parser->current_string, (const uint8_t *) arena_data, len);
9841 pm_buffer_cleanup(&token_buffer->buffer);
9842}
9843
9853static void
9854pm_token_buffer_flush(pm_parser_t *parser, pm_token_buffer_t *token_buffer) {
9855 if (token_buffer->cursor == NULL) {
9856 pm_string_shared_init(&parser->current_string, parser->current.start, parser->current.end);
9857 } else {
9858 pm_buffer_append_bytes(&token_buffer->buffer, token_buffer->cursor, (size_t) (parser->current.end - token_buffer->cursor));
9859 pm_token_buffer_copy(parser, token_buffer);
9860 }
9861}
9862
9863#define PM_TOKEN_BUFFER_DEFAULT_SIZE 16
9864
9873static void
9874pm_token_buffer_escape(pm_parser_t *parser, pm_token_buffer_t *token_buffer) {
9875 const uint8_t *start;
9876 if (token_buffer->cursor == NULL) {
9877 pm_buffer_init(&token_buffer->buffer, PM_TOKEN_BUFFER_DEFAULT_SIZE);
9878 start = parser->current.start;
9879 } else {
9880 start = token_buffer->cursor;
9881 }
9882
9883 const uint8_t *end = parser->current.end - 1;
9884 PRISM_ASSERT(end >= start);
9885 pm_buffer_append_bytes(&token_buffer->buffer, start, (size_t) (end - start));
9886
9887 token_buffer->cursor = end;
9888}
9889
9890#undef PM_TOKEN_BUFFER_DEFAULT_SIZE
9891
9905
9917static PRISM_INLINE void
9918pm_regexp_token_buffer_resume(pm_parser_t *parser, pm_regexp_token_buffer_t *token_buffer, const uint8_t *cursor) {
9919 token_buffer->base.cursor = cursor;
9920
9921 uint32_t content = U32(pm_buffer_length(&token_buffer->base.buffer));
9922 uint32_t source = U32(cursor - parser->start);
9923 pm_regexp_offsets_record(parser, &token_buffer->offsets, content, source, U32(parser->current.start - parser->start));
9924}
9925
9934static PRISM_INLINE bool
9935pm_regexp_escape_kept_p(const pm_lex_mode_t *lex_mode, uint8_t peeked) {
9936 switch (peeked) {
9937 case '\r': case '\n': case 'c': case 'C': case 'M': case 'u': case 'x':
9938 return false;
9939 case '$': case ')': case '*': case '+': case '.': case '>':
9940 case '?': case ']': case '^': case '|': case '}':
9941 return true;
9942 default:
9943 return peeked < 0x80 && peeked != lex_mode->as.regexp.terminator;
9944 }
9945}
9946
9953static void
9954pm_regexp_lex_braced_unicode(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t terminator) {
9955 const uint8_t *start = parser->current.end - 1;
9956 const uint8_t *cursor = parser->current.end + 2;
9957 const uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE] = { '}', '\\', '\n', '\r', terminator, '\0' };
9958
9959 while (cursor < parser->end) {
9960 cursor = pm_strpbrk(parser, cursor, breakpoints, parser->end - cursor, PM_STRPBRK_REGEXP);
9961 if (cursor == NULL) {
9962 cursor = parser->end;
9963 break;
9964 }
9965
9966 uint8_t byte = *cursor;
9967 if (byte != '}' && (byte == terminator || byte == '\n' || (byte == '\r' && peek_at(parser, cursor + 1) == '\n'))) break;
9968 cursor++;
9969 if (byte == '}') break;
9970
9971 /* Leave multibyte characters and line endings for the next scan. */
9972 if (byte == '\\' && cursor < parser->end && *cursor < 0x80 && *cursor != '\n' && *cursor != '\r') cursor++;
9973 }
9974
9975 parser->current.end = cursor;
9976 pm_buffer_append_bytes(buffer, start, (size_t) (cursor - start));
9977}
9978
9983static void
9984pm_regexp_token_buffer_copy(pm_parser_t *parser, pm_regexp_token_buffer_t *token_buffer) {
9985 pm_token_buffer_copy(parser, &token_buffer->base);
9986 parser->current_regexp_offsets = token_buffer->offsets;
9987}
9988
9993static void
9994pm_regexp_token_buffer_flush(pm_parser_t *parser, pm_regexp_token_buffer_t *token_buffer) {
9995 pm_token_buffer_flush(parser, &token_buffer->base);
9996 parser->current_regexp_offsets = token_buffer->offsets;
9997}
9998
10003static PRISM_INLINE size_t
10004pm_heredoc_strspn_inline_whitespace(pm_parser_t *parser, const uint8_t **cursor, pm_heredoc_indent_t indent) {
10005 size_t whitespace = 0;
10006
10007 switch (indent) {
10008 case PM_HEREDOC_INDENT_NONE:
10009 // Do nothing, we can't match a terminator with
10010 // indentation and there's no need to calculate common
10011 // whitespace.
10012 break;
10013 case PM_HEREDOC_INDENT_DASH:
10014 // Skip past inline whitespace.
10015 *cursor += pm_strspn_inline_whitespace(*cursor, parser->end - *cursor);
10016 break;
10017 case PM_HEREDOC_INDENT_TILDE:
10018 // Skip past inline whitespace and calculate common
10019 // whitespace.
10020 while (*cursor < parser->end && pm_char_is_inline_whitespace(**cursor)) {
10021 if (**cursor == '\t') {
10022 whitespace = (whitespace / PM_TAB_WHITESPACE_SIZE + 1) * PM_TAB_WHITESPACE_SIZE;
10023 } else {
10024 whitespace++;
10025 }
10026 (*cursor)++;
10027 }
10028
10029 break;
10030 }
10031
10032 return whitespace;
10033}
10034
10039static uint8_t
10040pm_lex_percent_delimiter(pm_parser_t *parser) {
10041 size_t eol_length = match_eol(parser);
10042
10043 if (eol_length) {
10044 if (parser->heredoc_end) {
10045 // If we have already lexed a heredoc, then the newline has already
10046 // been added to the list. In this case we want to just flush the
10047 // heredoc end.
10048 parser_flush_heredoc_end(parser);
10049 } else {
10050 // Otherwise, we'll add the newline to the list of newlines.
10051 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current) + U32(eol_length));
10052 }
10053
10054 uint8_t delimiter = *parser->current.end;
10055
10056 // If our delimiter is \r\n, we want to treat it as if it's \n.
10057 // For example, %\r\nfoo\r\n should be "foo"
10058 if (eol_length == 2) {
10059 delimiter = *(parser->current.end + 1);
10060 }
10061
10062 parser->current.end += eol_length;
10063 return delimiter;
10064 }
10065
10066 return *parser->current.end++;
10067}
10068
10073#define LEX(token_type) parser->current.type = token_type; parser_lex_callback(parser); return
10074
10081static void
10082parser_lex(pm_parser_t *parser) {
10083 PRISM_ASSERT(parser->current.end <= parser->end);
10084 parser->previous = parser->current;
10085
10086 // This value mirrors cmd_state from CRuby.
10087 bool previous_command_start = parser->command_start;
10088 parser->command_start = false;
10089
10090 // This is used to communicate to the newline lexing function that we've
10091 // already seen a comment.
10092 bool lexed_comment = false;
10093
10094 // Here we cache the current value of the semantic token seen flag. This is
10095 // used to reset it in case we find a token that shouldn't flip this flag.
10096 unsigned int semantic_token_seen = parser->semantic_token_seen;
10097 parser->semantic_token_seen = true;
10098
10099 // We'll jump to this label when we are about to encounter an EOF.
10100 // If we still have lex_modes on the stack, we pop them so that cleanup
10101 // can happen. For example, we should still continue parsing after a heredoc
10102 // identifier, even if the heredoc body was syntax invalid.
10103 switch_lex_modes:
10104
10105 switch (parser->lex_modes.current->mode) {
10106 case PM_LEX_DEFAULT:
10107 case PM_LEX_EMBEXPR:
10108 case PM_LEX_EMBVAR:
10109
10110 // We have a specific named label here because we are going to jump back to
10111 // this location in the event that we have lexed a token that should not be
10112 // returned to the parser. This includes comments, ignored newlines, and
10113 // invalid tokens of some form.
10114 lex_next_token: {
10115 // If we have the special next_start pointer set, then we're going to jump
10116 // to that location and start lexing from there.
10117 if (parser->next_start != NULL) {
10118 parser->current.end = parser->next_start;
10119 parser->next_start = NULL;
10120 }
10121
10122 // This value mirrors space_seen from CRuby. It tracks whether or not
10123 // space has been eaten before the start of the next token.
10124 bool space_seen = false;
10125
10126 // First, we're going to skip past any whitespace at the front of the next
10127 // token. Skip runs of inline whitespace in bulk to avoid per-character
10128 // stores back to parser->current.end.
10129 bool chomping = true;
10130 while (chomping) {
10131 /* Skip the run of inline whitespace in bulk, then decide what
10132 * to do based on the first byte after it. Handling both in a
10133 * single pass avoids re-entering the scan when the run was
10134 * non-empty, which is the common case. */
10135 {
10136 static const uint8_t inline_whitespace[256] = {
10137 [' '] = 1, ['\t'] = 1, ['\f'] = 1, ['\v'] = 1
10138 };
10139 const uint8_t *scan = parser->current.end;
10140 while (scan < parser->end && inline_whitespace[*scan]) scan++;
10141 if (scan > parser->current.end) {
10142 parser->current.end = scan;
10143 space_seen = true;
10144 }
10145 if (scan >= parser->end) break;
10146 }
10147
10148 switch (*parser->current.end) {
10149 case '\r':
10150 if (match_eol_offset(parser, 1)) {
10151 chomping = false;
10152 } else {
10153 pm_parser_warn(parser, PM_TOKEN_END(parser, &parser->current), 1, PM_WARN_UNEXPECTED_CARRIAGE_RETURN);
10154 parser->current.end++;
10155 space_seen = true;
10156 }
10157 break;
10158 case '\\': {
10159 size_t eol_length = match_eol_offset(parser, 1);
10160 if (eol_length) {
10161 if (parser->heredoc_end) {
10162 parser->current.end = parser->heredoc_end;
10163 parser->heredoc_end = NULL;
10164 } else {
10165 parser->current.end += eol_length + 1;
10166 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current));
10167 space_seen = true;
10168 }
10169 } else if (pm_char_is_inline_whitespace(*parser->current.end)) {
10170 parser->current.end += 2;
10171 } else {
10172 chomping = false;
10173 }
10174
10175 break;
10176 }
10177 default:
10178 chomping = false;
10179 break;
10180 }
10181 }
10182
10183 // Next, we'll set to start of this token to be the current end.
10184 parser->current.start = parser->current.end;
10185
10186 // We'll check if we're at the end of the file. If we are, then we
10187 // need to return the EOF token.
10188 if (parser->current.end >= parser->end) {
10189 lex_eof:
10190 // We may be missing closing tokens. We should pop modes one by one
10191 // to do the appropriate cleanup like moving next_start for heredocs.
10192 // Only when no mode is remaining will we actually emit the EOF token.
10193 if (parser->lex_modes.current->mode != PM_LEX_DEFAULT) {
10194 lex_mode_pop(parser);
10195 goto switch_lex_modes;
10196 }
10197
10198 // If we hit EOF, but the EOF came immediately after a newline,
10199 // set the start of the token to the newline. This way any EOF
10200 // errors will be reported as happening on that line rather than
10201 // a line after. For example "foo(\n" should report an error
10202 // on line 1 even though EOF technically occurs on line 2.
10203 if (parser->current.start > parser->start && (*(parser->current.start - 1) == '\n')) {
10204 parser->current.start -= 1;
10205 }
10206 LEX(PM_TOKEN_EOF);
10207 }
10208
10209 // Finally, we'll check the current character to determine the next
10210 // token.
10211 switch (*parser->current.end++) {
10212 case '\0': // NUL or end of script
10213 case '\004': // ^D
10214 case '\032': // ^Z
10215 parser->current.end--;
10216 LEX(PM_TOKEN_EOF);
10217
10218 case '#': { // comments
10219 const uint8_t *ending = next_newline(parser->current.end, parser->end - parser->current.end);
10220 parser->current.end = ending == NULL ? parser->end : ending;
10221
10222 // If we found a comment while lexing, then we're going to
10223 // add it to the list of comments in the file and keep
10224 // lexing.
10225 pm_comment_t *comment = parser_comment(parser, PM_COMMENT_INLINE);
10226 pm_list_append(&parser->comment_list, (pm_list_node_t *) comment);
10227
10228 parser->current.type = PM_TOKEN_COMMENT;
10229 parser_lex_callback(parser);
10230
10231 // Here, parse the comment to see if it's a magic comment
10232 // and potentially change state on the parser.
10233 if (!parser_lex_magic_comment(parser, semantic_token_seen) && (parser->current.start == parser->encoding_comment_start)) {
10234 ptrdiff_t length = parser->current.end - parser->current.start;
10235
10236 // If we didn't find a magic comment within the first
10237 // pass and we're at the start of the file, then we need
10238 // to do another pass to potentially find other patterns
10239 // for encoding comments.
10240 if (length >= 10 && !parser->encoding_locked) {
10241 parser_lex_magic_comment_encoding(parser);
10242 }
10243 }
10244
10245 /* The comment does not include its terminating newline,
10246 * which lexes through the newline handling below as its
10247 * own token. A comment that ends the file has no newline,
10248 * so the newline handling runs without one to emit. */
10249 if (ending == NULL) {
10250 lexed_comment = true;
10251 } else {
10252 parser->current.start = ending;
10253 parser->current.end = ending + 1;
10254 }
10255 }
10257 case '\r':
10258 case '\n': {
10259 parser->semantic_token_seen = semantic_token_seen & 0x1;
10260 size_t eol_length = match_eol_at(parser, parser->current.end - 1);
10261
10262 if (eol_length) {
10263 // The only way you can have carriage returns in this
10264 // particular loop is if you have a carriage return
10265 // followed by a newline. In that case we'll just skip
10266 // over the carriage return and continue lexing, in
10267 // order to make it so that the newline token
10268 // encapsulates both the carriage return and the
10269 // newline. Note that we need to check that we haven't
10270 // already lexed a comment here because that falls
10271 // through into here as well.
10272 if (!lexed_comment) {
10273 parser->current.end += eol_length - 1; // skip CR
10274 }
10275
10276 if (parser->heredoc_end == NULL) {
10277 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current));
10278 }
10279 }
10280
10281 if (parser->heredoc_end) {
10282 parser_flush_heredoc_end(parser);
10283 }
10284
10285 // If this is an ignored newline, then we can continue lexing after
10286 // calling the callback with the ignored newline token.
10287 switch (lex_state_ignored_p(parser)) {
10288 case PM_IGNORED_NEWLINE_NONE:
10289 break;
10290 case PM_IGNORED_NEWLINE_PATTERN:
10291 if (parser->pattern_matching_newlines || parser->in_keyword_arg) {
10292 if (!lexed_comment) {
10293 parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR;
10294 parser_lex_callback(parser);
10295 }
10296
10297 lex_state_set(parser, PM_LEX_STATE_BEG);
10298 parser->command_start = true;
10299 parser->current.type = PM_TOKEN_NEWLINE;
10300 return;
10301 }
10303 case PM_IGNORED_NEWLINE_ALL:
10304 if (!lexed_comment) parser_lex_ignored_newline(parser);
10305 lexed_comment = false;
10306 goto lex_next_token;
10307 }
10308
10309 // Here we need to look ahead and see if there is a call operator
10310 // (either . or &.) that starts the next line. If there is, then this
10311 // is going to become an ignored newline and we're going to instead
10312 // return the call operator.
10313 const uint8_t *next_content = parser->next_start == NULL ? parser->current.end : parser->next_start;
10314 next_content += pm_strspn_inline_whitespace(next_content, parser->end - next_content);
10315
10316 if (next_content < parser->end) {
10317 // If we hit a comment after a newline, then we're going to check
10318 // if it's ignored or if it's followed by a method call ('.').
10319 // If it is, then we're going to call the
10320 // callback with an ignored newline and then continue lexing.
10321 // Otherwise we'll return a regular newline.
10322 if (next_content[0] == '#') {
10323 // Here we look for a "." or "&." following a "\n".
10324 const uint8_t *following = next_newline(next_content, parser->end - next_content);
10325
10326 while (following && (following + 1 < parser->end)) {
10327 following++;
10328 following += pm_strspn_inline_whitespace(following, parser->end - following);
10329
10330 // If this is not followed by a comment, then we can break out
10331 // of this loop.
10332 if (peek_at(parser, following) != '#') break;
10333
10334 // If there is a comment, then we need to find the end of the
10335 // comment and continue searching from there.
10336 following = next_newline(following, parser->end - following);
10337 }
10338
10339 // If the lex state was ignored, we will lex the
10340 // ignored newline.
10341 if (lex_state_ignored_p(parser)) {
10342 if (!lexed_comment) parser_lex_ignored_newline(parser);
10343 lexed_comment = false;
10344 goto lex_next_token;
10345 }
10346
10347 // If we hit a '.' or a '&.' we will lex the ignored
10348 // newline.
10349 if (following && (
10350 (peek_at(parser, following) == '.') ||
10351 (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '.')
10352 )) {
10353 if (!lexed_comment) parser_lex_ignored_newline(parser);
10354 lexed_comment = false;
10355 goto lex_next_token;
10356 }
10357
10358
10359 // If we are parsing as CRuby 4.0 or later and we
10360 // hit a '&&' or a '||' then we will lex the ignored
10361 // newline.
10362 if (
10363 (parser->version >= PM_OPTIONS_VERSION_CRUBY_4_0) &&
10364 following && (
10365 (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '&') ||
10366 (peek_at(parser, following) == '|' && peek_at(parser, following + 1) == '|') ||
10367 (
10368 peek_at(parser, following) == 'a' &&
10369 peek_at(parser, following + 1) == 'n' &&
10370 peek_at(parser, following + 2) == 'd' &&
10371 peek_at(parser, following + 3) != '!' &&
10372 peek_at(parser, following + 3) != '?' &&
10373 !char_is_identifier(parser, following + 3, parser->end - (following + 3))
10374 ) ||
10375 (
10376 peek_at(parser, following) == 'o' &&
10377 peek_at(parser, following + 1) == 'r' &&
10378 peek_at(parser, following + 2) != '!' &&
10379 peek_at(parser, following + 2) != '?' &&
10380 !char_is_identifier(parser, following + 2, parser->end - (following + 2))
10381 )
10382 )
10383 ) {
10384 if (!lexed_comment) parser_lex_ignored_newline(parser);
10385 lexed_comment = false;
10386 previous_command_start = false;
10387 goto lex_next_token;
10388 }
10389 }
10390
10391 // If we hit a . after a newline, then we're in a call chain and
10392 // we need to return the call operator.
10393 if (next_content[0] == '.') {
10394 /* A beginless range on the next line means this
10395 * newline terminates the statement rather than
10396 * continuing a method chain. */
10397 if (peek_at(parser, next_content + 1) == '.') {
10398 if (!lexed_comment) {
10399 parser->current.type = PM_TOKEN_NEWLINE_TERMINATOR;
10400 parser_lex_callback(parser);
10401 }
10402
10403 lex_state_set(parser, PM_LEX_STATE_BEG);
10404 parser->command_start = true;
10405 parser->current.type = PM_TOKEN_NEWLINE;
10406 return;
10407 }
10408
10409 if (!lexed_comment) parser_lex_ignored_newline(parser);
10410 lex_state_set(parser, PM_LEX_STATE_DOT);
10411 parser->current.start = next_content;
10412 parser->current.end = next_content + 1;
10413 parser->next_start = NULL;
10414 LEX(PM_TOKEN_DOT);
10415 }
10416
10417 // If we hit a &. after a newline, then we're in a call chain and
10418 // we need to return the call operator.
10419 if (peek_at(parser, next_content) == '&' && peek_at(parser, next_content + 1) == '.') {
10420 if (!lexed_comment) parser_lex_ignored_newline(parser);
10421 lex_state_set(parser, PM_LEX_STATE_DOT);
10422 parser->current.start = next_content;
10423 parser->current.end = next_content + 2;
10424 parser->next_start = NULL;
10425 LEX(PM_TOKEN_AMPERSAND_DOT);
10426 }
10427
10428 if (parser->version >= PM_OPTIONS_VERSION_CRUBY_4_0) {
10429 // If we hit an && then we are in a logical chain
10430 // and we need to return the logical operator.
10431 if (peek_at(parser, next_content) == '&' && peek_at(parser, next_content + 1) == '&') {
10432 if (!lexed_comment) parser_lex_ignored_newline(parser);
10433 lex_state_set(parser, PM_LEX_STATE_BEG);
10434 parser->current.start = next_content;
10435 parser->current.end = next_content + 2;
10436 parser->next_start = NULL;
10437 LEX(PM_TOKEN_AMPERSAND_AMPERSAND);
10438 }
10439
10440 // If we hit a || then we are in a logical chain and
10441 // we need to return the logical operator.
10442 if (peek_at(parser, next_content) == '|' && peek_at(parser, next_content + 1) == '|') {
10443 if (!lexed_comment) parser_lex_ignored_newline(parser);
10444 lex_state_set(parser, PM_LEX_STATE_BEG);
10445 parser->current.start = next_content;
10446 parser->current.end = next_content + 2;
10447 parser->next_start = NULL;
10448 LEX(PM_TOKEN_PIPE_PIPE);
10449 }
10450
10451 /* Let the identifier lexer distinguish a leading
10452 * logical keyword from a label such as `and:`. */
10453 if (
10454 peek_at(parser, next_content) == 'a' &&
10455 peek_at(parser, next_content + 1) == 'n' &&
10456 peek_at(parser, next_content + 2) == 'd' &&
10457 peek_at(parser, next_content + 3) != '!' &&
10458 peek_at(parser, next_content + 3) != '?' &&
10459 !char_is_identifier(parser, next_content + 3, parser->end - (next_content + 3))
10460 ) {
10461 if (!lexed_comment) parser_lex_ignored_newline(parser);
10462 lexed_comment = false;
10463 previous_command_start = false;
10464 goto lex_next_token;
10465 }
10466
10467 if (
10468 peek_at(parser, next_content) == 'o' &&
10469 peek_at(parser, next_content + 1) == 'r' &&
10470 peek_at(parser, next_content + 2) != '!' &&
10471 peek_at(parser, next_content + 2) != '?' &&
10472 !char_is_identifier(parser, next_content + 2, parser->end - (next_content + 2))
10473 ) {
10474 if (!lexed_comment) parser_lex_ignored_newline(parser);
10475 lexed_comment = false;
10476 previous_command_start = false;
10477 goto lex_next_token;
10478 }
10479 }
10480 }
10481
10482 // At this point we know this is a regular newline, and we can set the
10483 // necessary state and return the token.
10484 lex_state_set(parser, PM_LEX_STATE_BEG);
10485 parser->command_start = true;
10486 parser->current.type = PM_TOKEN_NEWLINE;
10487 if (!lexed_comment) parser_lex_callback(parser);
10488 return;
10489 }
10490
10491 // ,
10492 case ',':
10493 if ((parser->previous.type == PM_TOKEN_COMMA) && (parser->enclosure_nesting > 0)) {
10494 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARRAY_TERM, pm_token_str(parser->current.type));
10495 }
10496
10497 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
10498 LEX(PM_TOKEN_COMMA);
10499
10500 // (
10501 case '(': {
10502 /* A parenthesis scanned at the beginning of an expression
10503 * groups the expression it wraps, while one scanned in
10504 * argument position with a preceding space wraps a command
10505 * argument. Everything else opens an argument list. */
10506 pm_token_type_t type = PM_TOKEN_PARENTHESIS_LEFT;
10507
10508 if (lex_state_beg_p(parser)) {
10509 type = PM_TOKEN_PARENTHESIS_LEFT_GROUPING;
10510 } else if (space_seen && (lex_state_arg_p(parser) || parser->lex_state == (PM_LEX_STATE_END | PM_LEX_STATE_LABEL))) {
10511 type = PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES;
10512 }
10513
10514 parser->enclosure_nesting++;
10515 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
10516 pm_enclosure_frame_push(parser);
10517 LEX(type);
10518 }
10519
10520 // )
10521 case ')':
10522 parser->enclosure_nesting--;
10523 lex_state_set(parser, PM_LEX_STATE_ENDFN);
10524 pm_enclosure_frame_pop(parser);
10525 LEX(PM_TOKEN_PARENTHESIS_RIGHT);
10526
10527 // ;
10528 case ';':
10529 lex_state_set(parser, PM_LEX_STATE_BEG);
10530 parser->command_start = true;
10531 LEX(PM_TOKEN_SEMICOLON);
10532
10533 // [ [] []=
10534 case '[':
10535 parser->enclosure_nesting++;
10536 pm_token_type_t type = PM_TOKEN_BRACKET_LEFT;
10537
10538 if (lex_state_operator_p(parser)) {
10539 if (match(parser, ']')) {
10540 parser->enclosure_nesting--;
10541 lex_state_set(parser, PM_LEX_STATE_ARG);
10542 LEX(match(parser, '=') ? PM_TOKEN_BRACKET_LEFT_RIGHT_EQUAL : PM_TOKEN_BRACKET_LEFT_RIGHT);
10543 }
10544
10545 lex_state_set(parser, PM_LEX_STATE_ARG | PM_LEX_STATE_LABEL);
10546 LEX(type);
10547 }
10548
10549 if (lex_state_beg_p(parser) || (lex_state_arg_p(parser) && (space_seen || lex_state_p(parser, PM_LEX_STATE_LABELED)))) {
10550 type = PM_TOKEN_BRACKET_LEFT_ARRAY;
10551 }
10552
10553 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
10554 pm_enclosure_frame_push(parser);
10555 LEX(type);
10556
10557 // ]
10558 case ']':
10559 parser->enclosure_nesting--;
10560 lex_state_set(parser, PM_LEX_STATE_END);
10561 pm_enclosure_frame_pop(parser);
10562 LEX(PM_TOKEN_BRACKET_RIGHT);
10563
10564 // {
10565 case '{': {
10566 pm_token_type_t type = PM_TOKEN_BRACE_LEFT;
10567
10568 if (parser->enclosure_nesting == parser->lambda_enclosure_nesting) {
10569 /* This { begins a lambda */
10570 parser->command_start = true;
10571 lex_state_set(parser, PM_LEX_STATE_BEG);
10572 type = PM_TOKEN_LAMBDA_BEGIN;
10573 } else if (lex_state_p(parser, PM_LEX_STATE_LABELED)) {
10574 /* This { begins a hash literal */
10575 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
10576 type = PM_TOKEN_BRACE_LEFT_HASH;
10577 } else if (lex_state_p(parser, PM_LEX_STATE_ARG_ANY | PM_LEX_STATE_END | PM_LEX_STATE_ENDFN)) {
10578 /* This { begins a block */
10579 parser->command_start = true;
10580 lex_state_set(parser, PM_LEX_STATE_BEG);
10581 } else if (lex_state_p(parser, PM_LEX_STATE_ENDARG)) {
10582 /* This { begins a block following a parenthesized
10583 * command argument */
10584 parser->command_start = true;
10585 lex_state_set(parser, PM_LEX_STATE_BEG);
10586 type = PM_TOKEN_BRACE_LEFT_ARGUMENT;
10587 } else {
10588 /* This { begins a hash literal */
10589 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
10590 type = PM_TOKEN_BRACE_LEFT_HASH;
10591 }
10592
10593 parser->enclosure_nesting++;
10594 parser->brace_nesting++;
10595 pm_enclosure_frame_push(parser);
10596
10597 LEX(type);
10598 }
10599
10600 // }
10601 case '}':
10602 parser->enclosure_nesting--;
10603 pm_enclosure_frame_pop(parser);
10604
10605 if ((parser->lex_modes.current->mode == PM_LEX_EMBEXPR) && (parser->brace_nesting == 0)) {
10606 lex_mode_pop(parser);
10607
10608 /* The literal resumes after its interpolation, and the
10609 * encoding that literals inside the interpolation
10610 * fixed is not the encoding of its next part. */
10611 parser->explicit_encoding = NULL;
10612 LEX(PM_TOKEN_EMBEXPR_END);
10613 }
10614
10615 parser->brace_nesting--;
10616 lex_state_set(parser, PM_LEX_STATE_END);
10617 LEX(PM_TOKEN_BRACE_RIGHT);
10618
10619 // * ** **= *=
10620 case '*': {
10621 if (match(parser, '*')) {
10622 if (match(parser, '=')) {
10623 lex_state_set(parser, PM_LEX_STATE_BEG);
10624 LEX(PM_TOKEN_STAR_STAR_EQUAL);
10625 }
10626
10627 pm_token_type_t type = PM_TOKEN_STAR_STAR;
10628
10629 if (lex_state_spcarg_p(parser, space_seen)) {
10630 pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_PREFIX_STAR_STAR);
10631 type = PM_TOKEN_USTAR_STAR;
10632 } else if (lex_state_beg_p(parser)) {
10633 type = PM_TOKEN_USTAR_STAR;
10634 } else if (ambiguous_operator_p(parser, space_seen)) {
10635 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "**", "argument prefix");
10636 }
10637
10638 if (lex_state_operator_p(parser)) {
10639 lex_state_set(parser, PM_LEX_STATE_ARG);
10640 } else {
10641 lex_state_set(parser, PM_LEX_STATE_BEG);
10642 }
10643
10644 LEX(type);
10645 }
10646
10647 if (match(parser, '=')) {
10648 lex_state_set(parser, PM_LEX_STATE_BEG);
10649 LEX(PM_TOKEN_STAR_EQUAL);
10650 }
10651
10652 pm_token_type_t type = PM_TOKEN_STAR;
10653
10654 if (lex_state_spcarg_p(parser, space_seen)) {
10655 pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_PREFIX_STAR);
10656 type = PM_TOKEN_USTAR;
10657 } else if (lex_state_beg_p(parser)) {
10658 type = PM_TOKEN_USTAR;
10659 } else if (ambiguous_operator_p(parser, space_seen)) {
10660 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "*", "argument prefix");
10661 }
10662
10663 if (lex_state_operator_p(parser)) {
10664 lex_state_set(parser, PM_LEX_STATE_ARG);
10665 } else {
10666 lex_state_set(parser, PM_LEX_STATE_BEG);
10667 }
10668
10669 LEX(type);
10670 }
10671
10672 // ! != !~ !@
10673 case '!':
10674 if (lex_state_operator_p(parser)) {
10675 lex_state_set(parser, PM_LEX_STATE_ARG);
10676 if (match(parser, '@')) {
10677 LEX(PM_TOKEN_BANG);
10678 }
10679 } else {
10680 lex_state_set(parser, PM_LEX_STATE_BEG);
10681 }
10682
10683 if (match(parser, '=')) {
10684 LEX(PM_TOKEN_BANG_EQUAL);
10685 }
10686
10687 if (match(parser, '~')) {
10688 LEX(PM_TOKEN_BANG_TILDE);
10689 }
10690
10691 LEX(PM_TOKEN_BANG);
10692
10693 // = => =~ == === =begin
10694 case '=':
10695 if (
10696 current_token_starts_line(parser) &&
10697 (parser->current.end + 5 <= parser->end) &&
10698 memcmp(parser->current.end, "begin", 5) == 0 &&
10699 (
10700 pm_char_is_whitespace(peek_offset(parser, 5)) ||
10701 (peek_offset(parser, 5) == '\0') ||
10702 (peek_offset(parser, 5) == '\004') || /* Ctrl-D */
10703 (peek_offset(parser, 5) == '\032') /* Ctrl-Z */
10704 )
10705 ) {
10706 pm_token_type_t type = lex_embdoc(parser);
10707 if (type == PM_TOKEN_EOF) {
10708 parser->current.start = parser->current.end;
10709 goto lex_eof;
10710 }
10711
10712 goto lex_next_token;
10713 }
10714
10715 if (lex_state_operator_p(parser)) {
10716 lex_state_set(parser, PM_LEX_STATE_ARG);
10717 } else {
10718 lex_state_set(parser, PM_LEX_STATE_BEG);
10719 }
10720
10721 if (match(parser, '>')) {
10722 LEX(PM_TOKEN_EQUAL_GREATER);
10723 }
10724
10725 if (match(parser, '~')) {
10726 LEX(PM_TOKEN_EQUAL_TILDE);
10727 }
10728
10729 if (match(parser, '=')) {
10730 LEX(match(parser, '=') ? PM_TOKEN_EQUAL_EQUAL_EQUAL : PM_TOKEN_EQUAL_EQUAL);
10731 }
10732
10733 LEX(PM_TOKEN_EQUAL);
10734
10735 // < << <<= <= <=>
10736 case '<':
10737 if (match(parser, '<')) {
10738 if (
10739 !lex_state_p(parser, PM_LEX_STATE_DOT | PM_LEX_STATE_CLASS) &&
10740 !lex_state_end_p(parser) &&
10741 (!lex_state_p(parser, PM_LEX_STATE_ARG_ANY) || lex_state_p(parser, PM_LEX_STATE_LABELED) || space_seen)
10742 ) {
10743 const uint8_t *end = parser->current.end;
10744
10745 pm_heredoc_quote_t quote = PM_HEREDOC_QUOTE_NONE;
10746 pm_heredoc_indent_t indent = PM_HEREDOC_INDENT_NONE;
10747
10748 if (match(parser, '-')) {
10749 indent = PM_HEREDOC_INDENT_DASH;
10750 }
10751 else if (match(parser, '~')) {
10752 indent = PM_HEREDOC_INDENT_TILDE;
10753 }
10754
10755 if (match(parser, '`')) {
10756 quote = PM_HEREDOC_QUOTE_BACKTICK;
10757 }
10758 else if (match(parser, '"')) {
10759 quote = PM_HEREDOC_QUOTE_DOUBLE;
10760 }
10761 else if (match(parser, '\'')) {
10762 quote = PM_HEREDOC_QUOTE_SINGLE;
10763 }
10764
10765 const uint8_t *ident_start = parser->current.end;
10766 size_t width = 0;
10767
10768 if (parser->current.end >= parser->end) {
10769 parser->current.end = end;
10770 } else if (quote == PM_HEREDOC_QUOTE_NONE && (width = char_is_identifier(parser, parser->current.end, parser->end - parser->current.end)) == 0) {
10771 parser->current.end = end;
10772 } else {
10773 if (quote == PM_HEREDOC_QUOTE_NONE) {
10774 parser->current.end += width;
10775
10776 while ((width = char_is_identifier(parser, parser->current.end, parser->end - parser->current.end))) {
10777 parser->current.end += width;
10778 }
10779 } else {
10780 // If we have quotes, then we're going to go until we find the
10781 // end quote.
10782 while ((parser->current.end < parser->end) && quote != (pm_heredoc_quote_t) (*parser->current.end)) {
10783 if (*parser->current.end == '\r' || *parser->current.end == '\n') break;
10784 parser->current.end++;
10785 }
10786 }
10787
10788 size_t ident_length = (size_t) (parser->current.end - ident_start);
10789 bool ident_error = false;
10790
10791 if (quote != PM_HEREDOC_QUOTE_NONE && !match(parser, (uint8_t) quote)) {
10792 pm_parser_err(parser, U32(ident_start - parser->start), U32(ident_length), PM_ERR_HEREDOC_IDENTIFIER);
10793 ident_error = true;
10794 }
10795
10796 parser->explicit_encoding = NULL;
10797 lex_mode_push(parser, (pm_lex_mode_t) {
10798 .mode = PM_LEX_HEREDOC,
10799 .as.heredoc = {
10800 .base = {
10801 .ident_start = ident_start,
10802 .ident_length = ident_length,
10803 .quote = quote,
10804 .indent = indent
10805 },
10806 .next_start = parser->current.end,
10807 .common_whitespace = NULL,
10808 .line_continuation = false
10809 }
10810 });
10811
10812 if (parser->heredoc_end == NULL) {
10813 const uint8_t *body_start = next_newline(parser->current.end, parser->end - parser->current.end);
10814
10815 if (body_start == NULL) {
10816 // If there is no newline after the heredoc identifier, then
10817 // this is not a valid heredoc declaration. In this case we
10818 // will add an error, but we will still return a heredoc
10819 // start.
10820 if (!ident_error) pm_parser_err_heredoc_term(parser, ident_start, ident_length);
10821 body_start = parser->end;
10822 } else {
10823 // Otherwise, we want to indicate that the body of the
10824 // heredoc starts on the character after the next newline.
10825 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(body_start - parser->start + 1));
10826 body_start++;
10827 }
10828
10829 parser->next_start = body_start;
10830 } else {
10831 parser->next_start = parser->heredoc_end;
10832 }
10833
10834 LEX(PM_TOKEN_HEREDOC_START);
10835 }
10836 }
10837
10838 if (match(parser, '=')) {
10839 lex_state_set(parser, PM_LEX_STATE_BEG);
10840 LEX(PM_TOKEN_LESS_LESS_EQUAL);
10841 }
10842
10843 if (ambiguous_operator_p(parser, space_seen)) {
10844 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "<<", "here document");
10845 }
10846
10847 if (lex_state_operator_p(parser)) {
10848 lex_state_set(parser, PM_LEX_STATE_ARG);
10849 } else {
10850 if (lex_state_p(parser, PM_LEX_STATE_CLASS)) parser->command_start = true;
10851 lex_state_set(parser, PM_LEX_STATE_BEG);
10852 }
10853
10854 LEX(PM_TOKEN_LESS_LESS);
10855 }
10856
10857 if (lex_state_operator_p(parser)) {
10858 lex_state_set(parser, PM_LEX_STATE_ARG);
10859 } else {
10860 if (lex_state_p(parser, PM_LEX_STATE_CLASS)) parser->command_start = true;
10861 lex_state_set(parser, PM_LEX_STATE_BEG);
10862 }
10863
10864 if (match(parser, '=')) {
10865 if (match(parser, '>')) {
10866 LEX(PM_TOKEN_LESS_EQUAL_GREATER);
10867 }
10868
10869 LEX(PM_TOKEN_LESS_EQUAL);
10870 }
10871
10872 LEX(PM_TOKEN_LESS);
10873
10874 // > >> >>= >=
10875 case '>':
10876 if (match(parser, '>')) {
10877 if (lex_state_operator_p(parser)) {
10878 lex_state_set(parser, PM_LEX_STATE_ARG);
10879 } else {
10880 lex_state_set(parser, PM_LEX_STATE_BEG);
10881 }
10882 LEX(match(parser, '=') ? PM_TOKEN_GREATER_GREATER_EQUAL : PM_TOKEN_GREATER_GREATER);
10883 }
10884
10885 if (lex_state_operator_p(parser)) {
10886 lex_state_set(parser, PM_LEX_STATE_ARG);
10887 } else {
10888 lex_state_set(parser, PM_LEX_STATE_BEG);
10889 }
10890
10891 LEX(match(parser, '=') ? PM_TOKEN_GREATER_EQUAL : PM_TOKEN_GREATER);
10892
10893 // double-quoted string literal
10894 case '"': {
10895 bool label_allowed = (lex_state_p(parser, PM_LEX_STATE_LABEL | PM_LEX_STATE_ENDFN) && !previous_command_start) || lex_state_arg_p(parser);
10896 lex_mode_push_string(parser, true, label_allowed, '\0', '"');
10897 LEX(PM_TOKEN_STRING_BEGIN);
10898 }
10899
10900 // xstring literal
10901 case '`': {
10902 if (lex_state_p(parser, PM_LEX_STATE_FNAME)) {
10903 lex_state_set(parser, PM_LEX_STATE_ENDFN);
10904 LEX(PM_TOKEN_BACKTICK);
10905 }
10906
10907 if (lex_state_p(parser, PM_LEX_STATE_DOT)) {
10908 if (previous_command_start) {
10909 lex_state_set(parser, PM_LEX_STATE_CMDARG);
10910 } else {
10911 lex_state_set(parser, PM_LEX_STATE_ARG);
10912 }
10913
10914 LEX(PM_TOKEN_BACKTICK);
10915 }
10916
10917 lex_mode_push_string(parser, true, false, '\0', '`');
10918 LEX(PM_TOKEN_XSTRING_BEGIN);
10919 }
10920
10921 // single-quoted string literal
10922 case '\'': {
10923 bool label_allowed = (lex_state_p(parser, PM_LEX_STATE_LABEL | PM_LEX_STATE_ENDFN) && !previous_command_start) || lex_state_arg_p(parser);
10924 lex_mode_push_string(parser, false, label_allowed, '\0', '\'');
10925 LEX(PM_TOKEN_STRING_BEGIN);
10926 }
10927
10928 // ? character literal
10929 case '?':
10930 LEX(lex_question_mark(parser));
10931
10932 // & && &&= &=
10933 case '&': {
10934 if (match(parser, '&')) {
10935 lex_state_set(parser, PM_LEX_STATE_BEG);
10936
10937 if (match(parser, '=')) {
10938 LEX(PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL);
10939 }
10940
10941 LEX(PM_TOKEN_AMPERSAND_AMPERSAND);
10942 }
10943
10944 if (match(parser, '=')) {
10945 lex_state_set(parser, PM_LEX_STATE_BEG);
10946 LEX(PM_TOKEN_AMPERSAND_EQUAL);
10947 }
10948
10949 if (match(parser, '.')) {
10950 lex_state_set(parser, PM_LEX_STATE_DOT);
10951 LEX(PM_TOKEN_AMPERSAND_DOT);
10952 }
10953
10954 pm_token_type_t type = PM_TOKEN_AMPERSAND;
10955 if (lex_state_spcarg_p(parser, space_seen)) {
10956 if ((peek(parser) != ':') || (peek_offset(parser, 1) == '\0')) {
10957 pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_PREFIX_AMPERSAND);
10958 } else {
10959 const uint8_t delim = peek_offset(parser, 1);
10960
10961 if ((delim != '\'') && (delim != '"') && !char_is_identifier(parser, parser->current.end + 1, parser->end - (parser->current.end + 1))) {
10962 pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_PREFIX_AMPERSAND);
10963 }
10964 }
10965
10966 type = PM_TOKEN_UAMPERSAND;
10967 } else if (lex_state_beg_p(parser)) {
10968 type = PM_TOKEN_UAMPERSAND;
10969 } else if (ambiguous_operator_p(parser, space_seen)) {
10970 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "&", "argument prefix");
10971 }
10972
10973 if (lex_state_operator_p(parser)) {
10974 lex_state_set(parser, PM_LEX_STATE_ARG);
10975 } else {
10976 lex_state_set(parser, PM_LEX_STATE_BEG);
10977 }
10978
10979 LEX(type);
10980 }
10981
10982 // | || ||= |=
10983 case '|':
10984 if (match(parser, '|')) {
10985 if (match(parser, '=')) {
10986 lex_state_set(parser, PM_LEX_STATE_BEG);
10987 LEX(PM_TOKEN_PIPE_PIPE_EQUAL);
10988 }
10989
10990 if (lex_state_p(parser, PM_LEX_STATE_BEG)) {
10991 parser->current.end--;
10992 LEX(PM_TOKEN_PIPE);
10993 }
10994
10995 lex_state_set(parser, PM_LEX_STATE_BEG);
10996 LEX(PM_TOKEN_PIPE_PIPE);
10997 }
10998
10999 if (match(parser, '=')) {
11000 lex_state_set(parser, PM_LEX_STATE_BEG);
11001 LEX(PM_TOKEN_PIPE_EQUAL);
11002 }
11003
11004 if (lex_state_operator_p(parser)) {
11005 lex_state_set(parser, PM_LEX_STATE_ARG);
11006 } else {
11007 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
11008 }
11009
11010 LEX(PM_TOKEN_PIPE);
11011
11012 // + += +@
11013 case '+': {
11014 if (lex_state_operator_p(parser)) {
11015 lex_state_set(parser, PM_LEX_STATE_ARG);
11016
11017 if (match(parser, '@')) {
11018 LEX(PM_TOKEN_UPLUS);
11019 }
11020
11021 LEX(PM_TOKEN_PLUS);
11022 }
11023
11024 if (match(parser, '=')) {
11025 lex_state_set(parser, PM_LEX_STATE_BEG);
11026 LEX(PM_TOKEN_PLUS_EQUAL);
11027 }
11028
11029 if (
11030 lex_state_beg_p(parser) ||
11031 (lex_state_spcarg_p(parser, space_seen) ? (pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS), true) : false)
11032 ) {
11033 lex_state_set(parser, PM_LEX_STATE_BEG);
11034
11035 if (pm_char_is_decimal_digit(peek(parser))) {
11036 parser->current.end++;
11037 pm_token_type_t type = lex_numeric(parser);
11038 lex_state_set(parser, PM_LEX_STATE_END);
11039 LEX(type);
11040 }
11041
11042 LEX(PM_TOKEN_UPLUS);
11043 }
11044
11045 if (ambiguous_operator_p(parser, space_seen)) {
11046 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "+", "unary operator");
11047 }
11048
11049 lex_state_set(parser, PM_LEX_STATE_BEG);
11050 LEX(PM_TOKEN_PLUS);
11051 }
11052
11053 // - -= -@
11054 case '-': {
11055 if (lex_state_operator_p(parser)) {
11056 lex_state_set(parser, PM_LEX_STATE_ARG);
11057
11058 if (match(parser, '@')) {
11059 LEX(PM_TOKEN_UMINUS);
11060 }
11061
11062 LEX(PM_TOKEN_MINUS);
11063 }
11064
11065 if (match(parser, '=')) {
11066 lex_state_set(parser, PM_LEX_STATE_BEG);
11067 LEX(PM_TOKEN_MINUS_EQUAL);
11068 }
11069
11070 if (match(parser, '>')) {
11071 lex_state_set(parser, PM_LEX_STATE_ENDFN);
11072 LEX(PM_TOKEN_MINUS_GREATER);
11073 }
11074
11075 bool spcarg = lex_state_spcarg_p(parser, space_seen);
11076 bool is_beg = lex_state_beg_p(parser);
11077 if (!is_beg && spcarg) {
11078 pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS);
11079 }
11080
11081 if (is_beg || spcarg) {
11082 lex_state_set(parser, PM_LEX_STATE_BEG);
11083 LEX(pm_char_is_decimal_digit(peek(parser)) ? PM_TOKEN_UMINUS_NUM : PM_TOKEN_UMINUS);
11084 }
11085
11086 if (ambiguous_operator_p(parser, space_seen)) {
11087 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "-", "unary operator");
11088 }
11089
11090 lex_state_set(parser, PM_LEX_STATE_BEG);
11091 LEX(PM_TOKEN_MINUS);
11092 }
11093
11094 // . .. ...
11095 case '.': {
11096 bool beg_p = lex_state_beg_p(parser);
11097
11098 if (match(parser, '.')) {
11099 if (match(parser, '.')) {
11100 // If we're _not_ inside a range within default parameters
11101 if (!context_p(parser, PM_CONTEXT_DEFAULT_PARAMS) && context_p(parser, PM_CONTEXT_DEF_PARAMS)) {
11102 if (lex_state_p(parser, PM_LEX_STATE_END)) {
11103 lex_state_set(parser, PM_LEX_STATE_BEG);
11104 } else {
11105 lex_state_set(parser, PM_LEX_STATE_ENDARG);
11106 }
11107 LEX(PM_TOKEN_UDOT_DOT_DOT);
11108 }
11109
11110 if (parser->enclosure_nesting == 0 && parser_end_of_line_p(parser)) {
11111 pm_parser_warn_token(parser, &parser->current, PM_WARN_DOT_DOT_DOT_EOL);
11112 }
11113
11114 lex_state_set(parser, PM_LEX_STATE_BEG);
11115 LEX(beg_p ? PM_TOKEN_UDOT_DOT_DOT : PM_TOKEN_DOT_DOT_DOT);
11116 }
11117
11118 lex_state_set(parser, PM_LEX_STATE_BEG);
11119 LEX(beg_p ? PM_TOKEN_UDOT_DOT : PM_TOKEN_DOT_DOT);
11120 }
11121
11122 lex_state_set(parser, PM_LEX_STATE_DOT);
11123 LEX(PM_TOKEN_DOT);
11124 }
11125
11126 // integer
11127 case '0':
11128 case '1':
11129 case '2':
11130 case '3':
11131 case '4':
11132 case '5':
11133 case '6':
11134 case '7':
11135 case '8':
11136 case '9': {
11137 pm_token_type_t type = lex_numeric(parser);
11138 lex_state_set(parser, PM_LEX_STATE_END);
11139 LEX(type);
11140 }
11141
11142 // :: symbol
11143 case ':':
11144 if (match(parser, ':')) {
11145 if (lex_state_beg_p(parser) || lex_state_p(parser, PM_LEX_STATE_CLASS) || (lex_state_p(parser, PM_LEX_STATE_ARG_ANY) && space_seen)) {
11146 lex_state_set(parser, PM_LEX_STATE_BEG);
11147 LEX(PM_TOKEN_UCOLON_COLON);
11148 }
11149
11150 lex_state_set(parser, PM_LEX_STATE_DOT);
11151 LEX(PM_TOKEN_COLON_COLON);
11152 }
11153
11154 if (lex_state_end_p(parser) || pm_char_is_whitespace(peek(parser)) || peek(parser) == '#') {
11155 lex_state_set(parser, PM_LEX_STATE_BEG);
11156 LEX(PM_TOKEN_COLON);
11157 }
11158
11159 if (peek(parser) == '"' || peek(parser) == '\'') {
11160 lex_mode_push_string(parser, peek(parser) == '"', false, '\0', *parser->current.end);
11161 parser->current.end++;
11162 } else {
11163 /*
11164 * A quoted symbol clears its encoding by pushing a lex
11165 * mode above. A bare symbol is lexed inline, so it
11166 * clears the encoding here.
11167 */
11168 parser->explicit_encoding = NULL;
11169 }
11170
11171 lex_state_set(parser, PM_LEX_STATE_FNAME);
11172 LEX(PM_TOKEN_SYMBOL_BEGIN);
11173
11174 // / /=
11175 case '/':
11176 if (lex_state_beg_p(parser)) {
11177 lex_mode_push_regexp(parser, '\0', '/');
11178 LEX(PM_TOKEN_REGEXP_BEGIN);
11179 }
11180
11181 if (match(parser, '=')) {
11182 lex_state_set(parser, PM_LEX_STATE_BEG);
11183 LEX(PM_TOKEN_SLASH_EQUAL);
11184 }
11185
11186 if (lex_state_spcarg_p(parser, space_seen)) {
11187 // https://bugs.ruby-lang.org/issues/21994
11188 if (parser->version <= PM_OPTIONS_VERSION_CRUBY_4_0) {
11189 pm_parser_warn_token(parser, &parser->current, PM_WARN_AMBIGUOUS_SLASH);
11190 }
11191 lex_mode_push_regexp(parser, '\0', '/');
11192 LEX(PM_TOKEN_REGEXP_BEGIN);
11193 }
11194
11195 if (ambiguous_operator_p(parser, space_seen)) {
11196 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "/", "regexp literal");
11197 }
11198
11199 if (lex_state_operator_p(parser)) {
11200 lex_state_set(parser, PM_LEX_STATE_ARG);
11201 } else {
11202 lex_state_set(parser, PM_LEX_STATE_BEG);
11203 }
11204
11205 LEX(PM_TOKEN_SLASH);
11206
11207 // ^ ^=
11208 case '^':
11209 if (lex_state_operator_p(parser)) {
11210 lex_state_set(parser, PM_LEX_STATE_ARG);
11211 } else {
11212 lex_state_set(parser, PM_LEX_STATE_BEG);
11213 }
11214 LEX(match(parser, '=') ? PM_TOKEN_CARET_EQUAL : PM_TOKEN_CARET);
11215
11216 // ~ ~@
11217 case '~':
11218 if (lex_state_operator_p(parser)) {
11219 (void) match(parser, '@');
11220 lex_state_set(parser, PM_LEX_STATE_ARG);
11221 } else {
11222 lex_state_set(parser, PM_LEX_STATE_BEG);
11223 }
11224
11225 LEX(PM_TOKEN_TILDE);
11226
11227 // % %= %i %I %q %Q %w %W
11228 case '%': {
11229 // If there is no subsequent character then we have an
11230 // invalid token. We're going to say it's the percent
11231 // operator because we don't want to move into the string
11232 // lex mode unnecessarily.
11233 if ((lex_state_beg_p(parser) || lex_state_arg_p(parser)) && (parser->current.end >= parser->end)) {
11234 pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT_EOF);
11235 LEX(PM_TOKEN_PERCENT);
11236 }
11237
11238 if (!lex_state_beg_p(parser) && match(parser, '=')) {
11239 lex_state_set(parser, PM_LEX_STATE_BEG);
11240 LEX(PM_TOKEN_PERCENT_EQUAL);
11241 } else if (
11242 lex_state_beg_p(parser) ||
11243 (lex_state_p(parser, PM_LEX_STATE_FITEM) && (peek(parser) == 's')) ||
11244 lex_state_spcarg_p(parser, space_seen)
11245 ) {
11246 /* A % followed by a character that is not ASCII is an
11247 * error. If an ASCII letter or digit follows it, that
11248 * character is the type of the literal, as in %q(a).
11249 * Any other character is the delimiter of a string. */
11250 if (*parser->current.end >= 0x80) {
11251 pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT);
11252 goto lex_next_token;
11253 }
11254
11255 if (!pm_encoding_ascii_alnum_p(*parser->current.end)) {
11256 const uint8_t delimiter = pm_lex_percent_delimiter(parser);
11257 lex_mode_push_string(parser, true, false, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter));
11258 LEX(PM_TOKEN_STRING_BEGIN);
11259 }
11260
11261 // Delimiters for %-literals cannot be alphanumeric. We
11262 // validate that here.
11263 uint8_t delimiter = peek_offset(parser, 1);
11264 if (delimiter >= 0x80 || pm_encoding_ascii_alnum_p(delimiter)) {
11265 pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT);
11266 goto lex_next_token;
11267 }
11268
11269 switch (peek(parser)) {
11270 case 'i': {
11271 parser->current.end++;
11272
11273 if (parser->current.end < parser->end) {
11274 lex_mode_push_list(parser, false, pm_lex_percent_delimiter(parser));
11275 } else {
11276 lex_mode_push_list_eof(parser);
11277 }
11278
11279 LEX(PM_TOKEN_PERCENT_LOWER_I);
11280 }
11281 case 'I': {
11282 parser->current.end++;
11283
11284 if (parser->current.end < parser->end) {
11285 lex_mode_push_list(parser, true, pm_lex_percent_delimiter(parser));
11286 } else {
11287 lex_mode_push_list_eof(parser);
11288 }
11289
11290 LEX(PM_TOKEN_PERCENT_UPPER_I);
11291 }
11292 case 'r': {
11293 parser->current.end++;
11294
11295 if (parser->current.end < parser->end) {
11296 const uint8_t delimiter = pm_lex_percent_delimiter(parser);
11297 lex_mode_push_regexp(parser, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter));
11298 } else {
11299 lex_mode_push_regexp(parser, '\0', '\0');
11300 }
11301
11302 LEX(PM_TOKEN_REGEXP_BEGIN);
11303 }
11304 case 'q': {
11305 parser->current.end++;
11306
11307 if (parser->current.end < parser->end) {
11308 const uint8_t delimiter = pm_lex_percent_delimiter(parser);
11309 lex_mode_push_string(parser, false, false, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter));
11310 } else {
11311 lex_mode_push_string_eof(parser);
11312 }
11313
11314 LEX(PM_TOKEN_STRING_BEGIN);
11315 }
11316 case 'Q': {
11317 parser->current.end++;
11318
11319 if (parser->current.end < parser->end) {
11320 const uint8_t delimiter = pm_lex_percent_delimiter(parser);
11321 lex_mode_push_string(parser, true, false, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter));
11322 } else {
11323 lex_mode_push_string_eof(parser);
11324 }
11325
11326 LEX(PM_TOKEN_STRING_BEGIN);
11327 }
11328 case 's': {
11329 parser->current.end++;
11330
11331 if (parser->current.end < parser->end) {
11332 const uint8_t delimiter = pm_lex_percent_delimiter(parser);
11333 lex_mode_push_string(parser, false, false, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter));
11334 lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM);
11335 } else {
11336 lex_mode_push_string_eof(parser);
11337 }
11338
11339 LEX(PM_TOKEN_SYMBOL_BEGIN);
11340 }
11341 case 'w': {
11342 parser->current.end++;
11343
11344 if (parser->current.end < parser->end) {
11345 lex_mode_push_list(parser, false, pm_lex_percent_delimiter(parser));
11346 } else {
11347 lex_mode_push_list_eof(parser);
11348 }
11349
11350 LEX(PM_TOKEN_PERCENT_LOWER_W);
11351 }
11352 case 'W': {
11353 parser->current.end++;
11354
11355 if (parser->current.end < parser->end) {
11356 lex_mode_push_list(parser, true, pm_lex_percent_delimiter(parser));
11357 } else {
11358 lex_mode_push_list_eof(parser);
11359 }
11360
11361 LEX(PM_TOKEN_PERCENT_UPPER_W);
11362 }
11363 case 'x': {
11364 parser->current.end++;
11365
11366 if (parser->current.end < parser->end) {
11367 const uint8_t delimiter = pm_lex_percent_delimiter(parser);
11368 lex_mode_push_string(parser, true, false, lex_mode_incrementor(delimiter), lex_mode_terminator(delimiter));
11369 } else {
11370 lex_mode_push_string_eof(parser);
11371 }
11372
11373 LEX(PM_TOKEN_PERCENT_LOWER_X);
11374 }
11375 default:
11376 // If we get to this point, then we have a % that is completely
11377 // unparsable. In this case we'll just drop it from the parser
11378 // and skip past it and hope that the next token is something
11379 // that we can parse.
11380 pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT);
11381 goto lex_next_token;
11382 }
11383 }
11384
11385 if (ambiguous_operator_p(parser, space_seen)) {
11386 PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "%", "string literal");
11387 }
11388
11389 lex_state_set(parser, lex_state_operator_p(parser) ? PM_LEX_STATE_ARG : PM_LEX_STATE_BEG);
11390 LEX(PM_TOKEN_PERCENT);
11391 }
11392
11393 // global variable
11394 case '$': {
11395 pm_token_type_t type = lex_global_variable(parser);
11396
11397 // If we're lexing an embedded variable, then we need to pop back into
11398 // the parent lex context.
11399 if (parser->lex_modes.current->mode == PM_LEX_EMBVAR) {
11400 lex_mode_pop(parser);
11401 }
11402
11403 lex_state_set(parser, PM_LEX_STATE_END);
11404 LEX(type);
11405 }
11406
11407 // instance variable, class variable
11408 case '@':
11409 lex_state_set(parser, parser->lex_state & PM_LEX_STATE_FNAME ? PM_LEX_STATE_ENDFN : PM_LEX_STATE_END);
11410 LEX(lex_at_variable(parser));
11411
11412 default: {
11413 if (*parser->current.start != '_') {
11414 size_t width = char_is_identifier_start(parser, parser->current.start, parser->end - parser->current.start);
11415
11416 // If this isn't the beginning of an identifier, then
11417 // it's an invalid token as we've exhausted all of the
11418 // other options. We'll skip past it and return the next
11419 // token after adding an appropriate error message.
11420 if (!width) {
11421 if (*parser->current.start >= 0x80) {
11422 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *parser->current.start);
11423 } else if (*parser->current.start == '\\') {
11424 switch (peek_at(parser, parser->current.start + 1)) {
11425 case ' ':
11426 parser->current.end++;
11427 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped space");
11428 break;
11429 case '\f':
11430 parser->current.end++;
11431 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped form feed");
11432 break;
11433 case '\t':
11434 parser->current.end++;
11435 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped horizontal tab");
11436 break;
11437 case '\v':
11438 parser->current.end++;
11439 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped vertical tab");
11440 break;
11441 case '\r':
11442 if (peek_at(parser, parser->current.start + 2) != '\n') {
11443 parser->current.end++;
11444 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped carriage return");
11445 break;
11446 }
11448 default:
11449 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "backslash");
11450 break;
11451 }
11452 } else if (char_is_ascii_printable(*parser->current.start)) {
11453 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_PRINTABLE_CHARACTER, *parser->current.start);
11454 } else {
11455 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_CHARACTER, *parser->current.start);
11456 }
11457
11458 goto lex_next_token;
11459 }
11460
11461 parser->current.end = parser->current.start + width;
11462 }
11463
11464 pm_token_type_t type = lex_identifier(parser, previous_command_start);
11465
11466 // If we've hit a __END__ and it was at the start of the
11467 // line or the start of the file and it is followed by
11468 // either a \n or a \r\n, then this is the last token of the
11469 // file.
11470 if (
11471 ((parser->current.end - parser->current.start) == 7) &&
11472 current_token_starts_line(parser) &&
11473 (memcmp(parser->current.start, "__END__", 7) == 0) &&
11474 (parser->current.end == parser->end || match_eol(parser))
11475 ) {
11476 // Since we know we're about to add an __END__ comment,
11477 // we know we need to add all of the newlines to get the
11478 // correct column information for it.
11479 const uint8_t *cursor = parser->current.end;
11480 while ((cursor = next_newline(cursor, parser->end - cursor)) != NULL) {
11481 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(++cursor - parser->start));
11482 }
11483
11484 parser->current.end = parser->end;
11485 parser->current.type = PM_TOKEN___END__;
11486 parser_lex_callback(parser);
11487
11488 parser->data_loc.start = PM_TOKEN_START(parser, &parser->current);
11489 parser->data_loc.length = PM_TOKEN_LENGTH(&parser->current);
11490
11491 parser->current.start = parser->current.end;
11492 goto lex_eof;
11493 }
11494
11495 pm_lex_state_t last_state = parser->lex_state;
11496
11497 if (type == PM_TOKEN_IDENTIFIER || type == PM_TOKEN_CONSTANT || type == PM_TOKEN_METHOD_NAME) {
11498 if (lex_state_p(parser, PM_LEX_STATE_BEG_ANY | PM_LEX_STATE_ARG_ANY | PM_LEX_STATE_DOT)) {
11499 if (previous_command_start) {
11500 lex_state_set(parser, PM_LEX_STATE_CMDARG);
11501 } else {
11502 lex_state_set(parser, PM_LEX_STATE_ARG);
11503 }
11504 } else if (parser->lex_state == PM_LEX_STATE_FNAME) {
11505 lex_state_set(parser, PM_LEX_STATE_ENDFN);
11506 } else {
11507 lex_state_set(parser, PM_LEX_STATE_END);
11508 }
11509 }
11510
11511 if (
11512 !(last_state & (PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME)) &&
11513 (type == PM_TOKEN_IDENTIFIER) &&
11514 ((pm_parser_local_depth(parser, &parser->current) != -1) ||
11515 pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current)))
11516 ) {
11517 lex_state_set(parser, PM_LEX_STATE_END | PM_LEX_STATE_LABEL);
11518 }
11519
11520 LEX(type);
11521 }
11522 }
11523 }
11524 case PM_LEX_LIST: {
11525 if (parser->next_start != NULL) {
11526 parser->current.end = parser->next_start;
11527 parser->next_start = NULL;
11528 }
11529
11530 // First we'll set the beginning of the token.
11531 parser->current.start = parser->current.end;
11532
11533 pm_lex_mode_t *lex_mode = parser->lex_modes.current;
11534
11535 // If there's any whitespace at the start of the list, then we're
11536 // going to trim it off the beginning and create a new token.
11537 size_t whitespace;
11538
11539 if (lex_mode->as.list.terminator != '\n' && parser->heredoc_end) {
11540 whitespace = pm_strspn_inline_whitespace(parser->current.end, parser->end - parser->current.end);
11541 if (peek_offset(parser, (ptrdiff_t)whitespace) == '\n') {
11542 whitespace += 1;
11543 }
11544 } else if (lex_mode->as.list.terminator == '\n') {
11545 // When the list delimiter is a newline (e.g. `%w` followed by a
11546 // newline), the newline is the terminator rather than a word
11547 // separator. We only trim inline whitespace here so that the
11548 // terminating newline is left for the terminator handling below.
11549 whitespace = pm_strspn_inline_whitespace(parser->current.end, parser->end - parser->current.end);
11550 } else {
11551 whitespace = pm_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current));
11552 }
11553
11554 if (whitespace > 0) {
11555 parser->current.end += whitespace;
11556 if (peek_offset(parser, -1) == '\n') {
11557 // mutates next_start
11558 parser_flush_heredoc_end(parser);
11559 }
11560
11561 lex_mode->as.list.started = true;
11562 lex_mode->as.list.separated = true;
11563 LEX(PM_TOKEN_WORDS_SEP);
11564 }
11565
11566 // We'll check if we're at the end of the file. If we are, then we
11567 // need to return the EOF token.
11568 if (parser->current.end >= parser->end) {
11569 LEX(PM_TOKEN_EOF);
11570 }
11571
11572 /* A word separator delimits the opener from the first word (or
11573 * from the terminator when the list is empty). The parser accepts
11574 * the words without it, so when the list does not start with
11575 * whitespace the implicit separator goes to the lex callback alone
11576 * and lexing continues on to the token the parser receives. */
11577 if (!lex_mode->as.list.started) {
11578 lex_mode->as.list.started = true;
11579 lex_mode->as.list.separated = true;
11580 parser->current.type = PM_TOKEN_WORDS_SEP_IMPLICIT;
11581 parser_lex_callback(parser);
11582 }
11583
11584 // Here we'll get a list of the places where strpbrk should break,
11585 // and then find the first one.
11586 const uint8_t *breakpoints = lex_mode->as.list.breakpoints;
11587 const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
11588
11589 // If we haven't found an escape yet, then this buffer will be
11590 // unallocated since we can refer directly to the source string.
11591 pm_token_buffer_t token_buffer = { 0 };
11592
11593 while (breakpoint != NULL) {
11594 // If we hit whitespace, then we must have received content by
11595 // now, so we can return an element of the list. A whitespace
11596 // character that is also the terminator (e.g. a newline
11597 // delimiter) is handled by the terminator check below, not here.
11598 if (pm_char_is_whitespace(*breakpoint) && *breakpoint != lex_mode->as.list.terminator) {
11599 parser->current.end = breakpoint;
11600 pm_token_buffer_flush(parser, &token_buffer);
11601 lex_mode->as.list.separated = false;
11602 LEX(PM_TOKEN_STRING_CONTENT);
11603 }
11604
11605 // If we hit the terminator, we need to check which token to
11606 // return.
11607 if (*breakpoint == lex_mode->as.list.terminator) {
11608 // If this terminator doesn't actually close the list, then
11609 // we need to continue on past it.
11610 if (lex_mode->as.list.nesting > 0) {
11611 parser->current.end = breakpoint + 1;
11612 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
11613 lex_mode->as.list.nesting--;
11614 continue;
11615 }
11616
11617 // If we've hit the terminator and we've already skipped
11618 // past content, then we can return a list node.
11619 if (breakpoint > parser->current.start) {
11620 parser->current.end = breakpoint;
11621 pm_token_buffer_flush(parser, &token_buffer);
11622 lex_mode->as.list.separated = false;
11623 LEX(PM_TOKEN_STRING_CONTENT);
11624 }
11625
11626 /* A word separator delimits the last word from the
11627 * terminator. The parser accepts the terminator without
11628 * it, so when the list does not end with whitespace the
11629 * implicit separator goes to the lex callback alone and
11630 * lexing continues on to the terminator. */
11631 if (!lex_mode->as.list.separated) {
11632 lex_mode->as.list.separated = true;
11633 parser->current.end = breakpoint;
11634 parser->current.type = PM_TOKEN_WORDS_SEP_IMPLICIT;
11635 parser_lex_callback(parser);
11636 }
11637
11638 // Otherwise, switch back to the default state and return
11639 // the end of the list.
11640 parser->current.end = breakpoint + 1;
11641
11642 // If the terminator is a newline (i.e. the list delimiter
11643 // was a newline), then we need to record it so that line
11644 // numbers after the list remain accurate.
11645 if (*breakpoint == '\n') {
11646 if (parser->heredoc_end == NULL) {
11647 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current));
11648 } else {
11649 parser_flush_heredoc_end(parser);
11650 }
11651 }
11652
11653 lex_mode_pop(parser);
11654 lex_state_set(parser, PM_LEX_STATE_END);
11655 LEX(PM_TOKEN_STRING_END);
11656 }
11657
11658 // If we hit a null byte, skip directly past it.
11659 if (*breakpoint == '\0') {
11660 breakpoint = pm_strpbrk(parser, breakpoint + 1, breakpoints, parser->end - (breakpoint + 1), PM_STRPBRK_STRING);
11661 continue;
11662 }
11663
11664 // If we hit escapes, then we need to treat the next token
11665 // literally. In this case we'll skip past the next character
11666 // and find the next breakpoint.
11667 if (*breakpoint == '\\') {
11668 parser->current.end = breakpoint + 1;
11669
11670 // If we've hit the end of the file, then break out of the
11671 // loop by setting the breakpoint to NULL.
11672 if (parser->current.end == parser->end) {
11673 breakpoint = NULL;
11674 continue;
11675 }
11676
11677 pm_token_buffer_escape(parser, &token_buffer);
11678 uint8_t peeked = peek(parser);
11679
11680 switch (peeked) {
11681 case ' ':
11682 case '\f':
11683 case '\t':
11684 case '\v':
11685 case '\\':
11686 pm_token_buffer_push_byte(&token_buffer, peeked);
11687 parser->current.end++;
11688 break;
11689 case '\r':
11690 parser->current.end++;
11691 if (peek(parser) != '\n') {
11692 pm_token_buffer_push_byte(&token_buffer, '\r');
11693 break;
11694 }
11696 case '\n':
11697 pm_token_buffer_push_byte(&token_buffer, '\n');
11698
11699 if (parser->heredoc_end) {
11700 // ... if we are on the same line as a heredoc,
11701 // flush the heredoc and continue parsing after
11702 // heredoc_end.
11703 parser_flush_heredoc_end(parser);
11704 pm_token_buffer_copy(parser, &token_buffer);
11705 LEX(PM_TOKEN_STRING_CONTENT);
11706 } else {
11707 // ... else track the newline.
11708 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current) + 1);
11709 }
11710
11711 parser->current.end++;
11712 break;
11713 default:
11714 if (peeked == lex_mode->as.list.incrementor || peeked == lex_mode->as.list.terminator) {
11715 pm_token_buffer_push_byte(&token_buffer, peeked);
11716 parser->current.end++;
11717 } else if (lex_mode->as.list.interpolation) {
11718 escape_read(parser, &token_buffer.buffer, PM_ESCAPE_FLAG_NONE);
11719
11720 /* The escape consumed a newline with a heredoc body pending. */
11721 if (parser->next_start != NULL) {
11722 pm_token_buffer_copy(parser, &token_buffer);
11723 lex_mode->as.list.separated = false;
11724 LEX(PM_TOKEN_STRING_CONTENT);
11725 }
11726 } else {
11727 pm_token_buffer_push_byte(&token_buffer, '\\');
11728 pm_token_buffer_push_escaped(&token_buffer, parser);
11729 }
11730
11731 break;
11732 }
11733
11734 token_buffer.cursor = parser->current.end;
11735 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
11736 continue;
11737 }
11738
11739 // If we hit a #, then we will attempt to lex interpolation.
11740 if (*breakpoint == '#') {
11741 pm_token_type_t type = lex_interpolation(parser, breakpoint);
11742
11743 if (!type) {
11744 // If we haven't returned at this point then we had something
11745 // that looked like an interpolated class or instance variable
11746 // like "#@" but wasn't actually. In this case we'll just skip
11747 // to the next breakpoint.
11748 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
11749 continue;
11750 }
11751
11752 if (type == PM_TOKEN_STRING_CONTENT) {
11753 pm_token_buffer_flush(parser, &token_buffer);
11754 }
11755
11756 lex_mode->as.list.separated = false;
11757 LEX(type);
11758 }
11759
11760 // If we've hit the incrementor, then we need to skip past it
11761 // and find the next breakpoint.
11762 PRISM_ASSERT(*breakpoint == lex_mode->as.list.incrementor);
11763 parser->current.end = breakpoint + 1;
11764 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
11765 lex_mode->as.list.nesting++;
11766 continue;
11767 }
11768
11769 /* With no breakpoint left, the rest of the file is content, so this
11770 * token runs to the end of the file. That keeps the last element of
11771 * a list that is not terminated in one piece. */
11772 parser->current.end = parser->end;
11773 pm_token_buffer_flush(parser, &token_buffer);
11774 LEX(PM_TOKEN_STRING_CONTENT);
11775 }
11776 case PM_LEX_REGEXP: {
11777 // First, we'll set to start of this token to be the current end.
11778 if (parser->next_start == NULL) {
11779 parser->current.start = parser->current.end;
11780 } else {
11781 parser->current.start = parser->next_start;
11782 parser->current.end = parser->next_start;
11783 parser->next_start = NULL;
11784 }
11785
11786 // We'll check if we're at the end of the file. If we are, then we
11787 // need to return the EOF token.
11788 if (parser->current.end >= parser->end) {
11789 LEX(PM_TOKEN_EOF);
11790 }
11791
11792 // Get a reference to the current mode.
11793 pm_lex_mode_t *lex_mode = parser->lex_modes.current;
11794
11795 // These are the places where we need to split up the content of the
11796 // regular expression. We'll use strpbrk to find the first of these
11797 // characters.
11798 const uint8_t *breakpoints = lex_mode->as.regexp.breakpoints;
11799 const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
11800 pm_regexp_token_buffer_t token_buffer = { 0 };
11801
11802 while (breakpoint != NULL) {
11803 uint8_t term = lex_mode->as.regexp.terminator;
11804 bool is_terminator = (*breakpoint == term);
11805
11806 // If the terminator is newline, we need to consider \r\n _also_ a newline
11807 // For example: `%\nfoo\r\n`
11808 // The string should be "foo", not "foo\r"
11809 if (*breakpoint == '\r' && peek_at(parser, breakpoint + 1) == '\n') {
11810 if (term == '\n') {
11811 is_terminator = true;
11812 }
11813
11814 // If the terminator is a CR, but we see a CRLF, we need to
11815 // treat the CRLF as a newline, meaning this is _not_ the
11816 // terminator
11817 if (term == '\r') {
11818 is_terminator = false;
11819 }
11820 }
11821
11822 // If we hit the terminator, we need to determine what kind of
11823 // token to return.
11824 if (is_terminator) {
11825 if (lex_mode->as.regexp.nesting > 0) {
11826 parser->current.end = breakpoint + 1;
11827 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
11828 lex_mode->as.regexp.nesting--;
11829 continue;
11830 }
11831
11832 // Here we've hit the terminator. If we have already consumed
11833 // content then we need to return that content as string content
11834 // first.
11835 if (breakpoint > parser->current.start) {
11836 parser->current.end = breakpoint;
11837 pm_regexp_token_buffer_flush(parser, &token_buffer);
11838 LEX(PM_TOKEN_STRING_CONTENT);
11839 }
11840
11841 // Check here if we need to track the newline.
11842 size_t eol_length = match_eol_at(parser, breakpoint);
11843 if (eol_length) {
11844 parser->current.end = breakpoint + eol_length;
11845
11846 // Track the newline if we're not in a heredoc that
11847 // would have already have added the newline to the
11848 // list.
11849 if (parser->heredoc_end == NULL) {
11850 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current));
11851 } else {
11852 parser_flush_heredoc_end(parser);
11853 }
11854 } else {
11855 parser->current.end = breakpoint + 1;
11856 }
11857
11858 /* Since we've hit the terminator of the regular expression,
11859 * we now need to parse the options. */
11860 if (parser->next_start == NULL) {
11861 parser->current.end += pm_strspn_regexp_option(parser->current.end, parser->end - parser->current.end);
11862 } else {
11863 /* A newline terminator with a heredoc body pending
11864 * leaves the options on the line after that body. The
11865 * token then starts on the newline ending the body, so
11866 * that it is still a newline followed by the options. */
11867 size_t options = pm_strspn_regexp_option(parser->next_start, parser->end - parser->next_start);
11868
11869 if (options > 0) {
11870 parser->current.start = parser->next_start - 1;
11871 parser->current.end = parser->next_start + options;
11872 parser->next_start = NULL;
11873 }
11874 }
11875
11876 lex_mode_pop(parser);
11877 lex_state_set(parser, PM_LEX_STATE_END);
11878 LEX(PM_TOKEN_REGEXP_END);
11879 }
11880
11881 // If we've hit the incrementor, then we need to skip past it
11882 // and find the next breakpoint.
11883 if (*breakpoint && *breakpoint == lex_mode->as.regexp.incrementor) {
11884 parser->current.end = breakpoint + 1;
11885 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
11886 lex_mode->as.regexp.nesting++;
11887 continue;
11888 }
11889
11890 switch (*breakpoint) {
11891 case '\0':
11892 // If we hit a null byte, skip directly past it.
11893 parser->current.end = breakpoint + 1;
11894 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
11895 break;
11896 case '\r':
11897 if (peek_at(parser, breakpoint + 1) != '\n') {
11898 parser->current.end = breakpoint + 1;
11899 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
11900 break;
11901 }
11902
11903 breakpoint++;
11904 parser->current.end = breakpoint;
11905 pm_token_buffer_escape(parser, &token_buffer.base);
11906 pm_regexp_token_buffer_resume(parser, &token_buffer, breakpoint);
11907
11909 case '\n':
11910 // If we've hit a newline, then we need to track that in
11911 // the list of newlines.
11912 if (parser->heredoc_end == NULL) {
11913 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(breakpoint - parser->start + 1));
11914 parser->current.end = breakpoint + 1;
11915 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
11916 break;
11917 }
11918
11919 parser->current.end = breakpoint + 1;
11920 parser_flush_heredoc_end(parser);
11921 pm_regexp_token_buffer_flush(parser, &token_buffer);
11922 LEX(PM_TOKEN_STRING_CONTENT);
11923 case '\\': {
11924 // If we hit escapes, then we need to treat the next
11925 // token literally. In this case we'll skip past the
11926 // next character and find the next breakpoint.
11927 parser->current.end = breakpoint + 1;
11928
11929 /* A backslash at the end of the file escapes nothing,
11930 * so it is left out of the content, and the regular
11931 * expression is validated without it. */
11932 if (parser->current.end == parser->end) {
11933 pm_token_buffer_escape(parser, &token_buffer.base);
11934 pm_regexp_token_buffer_resume(parser, &token_buffer, parser->end);
11935 breakpoint = NULL;
11936 break;
11937 }
11938
11939 /* An escape that the content keeps as it is in the
11940 * source needs no copy of the content, so the search
11941 * continues past it. */
11942 uint8_t peeked = peek(parser);
11943
11944 if (pm_regexp_escape_kept_p(lex_mode, peeked)) {
11945 parser->current.end++;
11946 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
11947 break;
11948 }
11949
11950 pm_token_buffer_escape(parser, &token_buffer.base);
11951
11952 switch (peeked) {
11953 case '\r':
11954 parser->current.end++;
11955 if (peek(parser) != '\n') {
11956 if (lex_mode->as.regexp.terminator != '\r') {
11957 pm_token_buffer_push_byte(&token_buffer.base, '\\');
11958 }
11959 pm_token_buffer_push_byte(&token_buffer.base, '\r');
11960 break;
11961 }
11963 case '\n':
11964 if (parser->heredoc_end) {
11965 // ... if we are on the same line as a heredoc,
11966 // flush the heredoc and continue parsing after
11967 // heredoc_end.
11968 parser_flush_heredoc_end(parser);
11969 pm_regexp_token_buffer_copy(parser, &token_buffer);
11970 LEX(PM_TOKEN_STRING_CONTENT);
11971 } else {
11972 // ... else track the newline.
11973 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current) + 1);
11974 }
11975
11976 parser->current.end++;
11977 break;
11978 case 'c':
11979 case 'C':
11980 case 'M':
11981 case 'u':
11982 case 'x':
11983 if (peeked == 'u' && peek_offset(parser, 1) == '{') {
11984 pm_regexp_lex_braced_unicode(parser, &token_buffer.base.buffer, lex_mode->as.regexp.terminator);
11985 } else {
11986 escape_read(parser, &token_buffer.base.buffer, PM_ESCAPE_FLAG_REGEXP);
11987 }
11988
11989 /* The escape consumed a newline with a heredoc body pending. */
11990 if (parser->next_start != NULL) {
11991 pm_regexp_token_buffer_copy(parser, &token_buffer);
11992 LEX(PM_TOKEN_STRING_CONTENT);
11993 }
11994
11995 break;
11996 default:
11997 /* The backslash before the terminator is left
11998 * out. So is one before a character that is not
11999 * ASCII, which the search below reads as content
12000 * like any other. */
12001 if (peeked == lex_mode->as.regexp.terminator) {
12002 pm_token_buffer_push_byte(&token_buffer.base, peeked);
12003 parser->current.end++;
12004 }
12005
12006 break;
12007 }
12008
12009 pm_regexp_token_buffer_resume(parser, &token_buffer, parser->current.end);
12010 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
12011 break;
12012 }
12013 case '#': {
12014 // If we hit a #, then we will attempt to lex
12015 // interpolation.
12016 pm_token_type_t type = lex_interpolation(parser, breakpoint);
12017
12018 if (!type) {
12019 // If we haven't returned at this point then we had
12020 // something that looked like an interpolated class or
12021 // instance variable like "#@" but wasn't actually. In
12022 // this case we'll just skip to the next breakpoint.
12023 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_REGEXP);
12024 break;
12025 }
12026
12027 if (type == PM_TOKEN_STRING_CONTENT) {
12028 pm_regexp_token_buffer_flush(parser, &token_buffer);
12029 }
12030
12031 LEX(type);
12032 }
12033 default:
12034 PRISM_ASSERT(false && "unreachable");
12035 break;
12036 }
12037 }
12038
12039 /* With no breakpoint left, the rest of the file is content, so this
12040 * token runs to the end of the file. That keeps the content of a
12041 * regular expression that is not terminated in one piece. */
12042 parser->current.end = parser->end;
12043 pm_regexp_token_buffer_flush(parser, &token_buffer);
12044 LEX(PM_TOKEN_STRING_CONTENT);
12045 }
12046 case PM_LEX_STRING: {
12047 // First, we'll set to start of this token to be the current end.
12048 if (parser->next_start == NULL) {
12049 parser->current.start = parser->current.end;
12050 } else {
12051 parser->current.start = parser->next_start;
12052 parser->current.end = parser->next_start;
12053 parser->next_start = NULL;
12054 }
12055
12056 // We'll check if we're at the end of the file. If we are, then we need to
12057 // return the EOF token.
12058 if (parser->current.end >= parser->end) {
12059 LEX(PM_TOKEN_EOF);
12060 }
12061
12062 // These are the places where we need to split up the content of the
12063 // string. We'll use strpbrk to find the first of these characters.
12064 pm_lex_mode_t *lex_mode = parser->lex_modes.current;
12065 const uint8_t *breakpoints = lex_mode->as.string.breakpoints;
12066 const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12067
12068 // If we haven't found an escape yet, then this buffer will be
12069 // unallocated since we can refer directly to the source string.
12070 pm_token_buffer_t token_buffer = { 0 };
12071
12072 while (breakpoint != NULL) {
12073 // If we hit the incrementor, then we'll increment then nesting and
12074 // continue lexing.
12075 if (lex_mode->as.string.incrementor != '\0' && *breakpoint == lex_mode->as.string.incrementor) {
12076 lex_mode->as.string.nesting++;
12077 parser->current.end = breakpoint + 1;
12078 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12079 continue;
12080 }
12081
12082 uint8_t term = lex_mode->as.string.terminator;
12083 bool is_terminator = (*breakpoint == term);
12084
12085 // If the terminator is newline, we need to consider \r\n _also_ a newline
12086 // For example: `%r\nfoo\r\n`
12087 // The string should be /foo/, not /foo\r/
12088 if (*breakpoint == '\r' && peek_at(parser, breakpoint + 1) == '\n') {
12089 if (term == '\n') {
12090 is_terminator = true;
12091 }
12092
12093 // If the terminator is a CR, but we see a CRLF, we need to
12094 // treat the CRLF as a newline, meaning this is _not_ the
12095 // terminator
12096 if (term == '\r') {
12097 is_terminator = false;
12098 }
12099 }
12100
12101 // Note that we have to check the terminator here first because we could
12102 // potentially be parsing a % string that has a # character as the
12103 // terminator.
12104 if (is_terminator) {
12105 // If this terminator doesn't actually close the string, then we need
12106 // to continue on past it.
12107 if (lex_mode->as.string.nesting > 0) {
12108 parser->current.end = breakpoint + 1;
12109 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12110 lex_mode->as.string.nesting--;
12111 continue;
12112 }
12113
12114 // Here we've hit the terminator. If we have already consumed content
12115 // then we need to return that content as string content first.
12116 if (breakpoint > parser->current.start) {
12117 parser->current.end = breakpoint;
12118 pm_token_buffer_flush(parser, &token_buffer);
12119 LEX(PM_TOKEN_STRING_CONTENT);
12120 }
12121
12122 // Otherwise we need to switch back to the parent lex mode and
12123 // return the end of the string.
12124 size_t eol_length = match_eol_at(parser, breakpoint);
12125 if (eol_length) {
12126 parser->current.end = breakpoint + eol_length;
12127
12128 // Track the newline if we're not in a heredoc that
12129 // would have already have added the newline to the
12130 // list.
12131 if (parser->heredoc_end == NULL) {
12132 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current));
12133 }
12134 } else {
12135 parser->current.end = breakpoint + 1;
12136 }
12137
12138 if (lex_mode->as.string.label_allowed && (peek(parser) == ':') && (peek_offset(parser, 1) != ':')) {
12139 parser->current.end++;
12140 lex_state_set(parser, PM_LEX_STATE_ARG | PM_LEX_STATE_LABELED);
12141 lex_mode_pop(parser);
12142 LEX(PM_TOKEN_LABEL_END);
12143 }
12144
12145 // When the delimiter itself is a newline, we won't
12146 // get a chance to flush heredocs in the usual places since
12147 // the newline is already consumed.
12148 if (term == '\n' && parser->heredoc_end) {
12149 parser_flush_heredoc_end(parser);
12150 }
12151
12152 lex_state_set(parser, PM_LEX_STATE_END);
12153 lex_mode_pop(parser);
12154 LEX(PM_TOKEN_STRING_END);
12155 }
12156
12157 switch (*breakpoint) {
12158 case '\0':
12159 // Skip directly past the null character.
12160 parser->current.end = breakpoint + 1;
12161 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12162 break;
12163 case '\r':
12164 if (peek_at(parser, breakpoint + 1) != '\n') {
12165 parser->current.end = breakpoint + 1;
12166 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12167 break;
12168 }
12169
12170 // If we hit a \r\n sequence, then we need to treat it
12171 // as a newline.
12172 breakpoint++;
12173 parser->current.end = breakpoint;
12174 pm_token_buffer_escape(parser, &token_buffer);
12175 token_buffer.cursor = breakpoint;
12176
12178 case '\n':
12179 // When we hit a newline, we need to flush any potential
12180 // heredocs. Note that this has to happen after we check
12181 // for the terminator in case the terminator is a
12182 // newline character.
12183 if (parser->heredoc_end == NULL) {
12184 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(breakpoint - parser->start + 1));
12185 parser->current.end = breakpoint + 1;
12186 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12187 break;
12188 }
12189
12190 parser->current.end = breakpoint + 1;
12191 parser_flush_heredoc_end(parser);
12192 pm_token_buffer_flush(parser, &token_buffer);
12193 LEX(PM_TOKEN_STRING_CONTENT);
12194 case '\\': {
12195 // Here we hit escapes.
12196 parser->current.end = breakpoint + 1;
12197
12198 // If we've hit the end of the file, then break out of
12199 // the loop by setting the breakpoint to NULL.
12200 if (parser->current.end == parser->end) {
12201 breakpoint = NULL;
12202 continue;
12203 }
12204
12205 pm_token_buffer_escape(parser, &token_buffer);
12206 uint8_t peeked = peek(parser);
12207
12208 switch (peeked) {
12209 case '\\':
12210 pm_token_buffer_push_byte(&token_buffer, '\\');
12211 parser->current.end++;
12212 break;
12213 case '\r':
12214 parser->current.end++;
12215 if (peek(parser) != '\n') {
12216 if (!lex_mode->as.string.interpolation) {
12217 pm_token_buffer_push_byte(&token_buffer, '\\');
12218 }
12219 pm_token_buffer_push_byte(&token_buffer, '\r');
12220 break;
12221 }
12223 case '\n':
12224 if (!lex_mode->as.string.interpolation) {
12225 pm_token_buffer_push_byte(&token_buffer, '\\');
12226 pm_token_buffer_push_byte(&token_buffer, '\n');
12227 }
12228
12229 if (parser->heredoc_end) {
12230 // ... if we are on the same line as a heredoc,
12231 // flush the heredoc and continue parsing after
12232 // heredoc_end.
12233 parser_flush_heredoc_end(parser);
12234 pm_token_buffer_copy(parser, &token_buffer);
12235 LEX(PM_TOKEN_STRING_CONTENT);
12236 } else {
12237 // ... else track the newline.
12238 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, PM_TOKEN_END(parser, &parser->current) + 1);
12239 }
12240
12241 parser->current.end++;
12242 break;
12243 default:
12244 if (lex_mode->as.string.incrementor != '\0' && peeked == lex_mode->as.string.incrementor) {
12245 pm_token_buffer_push_byte(&token_buffer, peeked);
12246 parser->current.end++;
12247 } else if (lex_mode->as.string.terminator != '\0' && peeked == lex_mode->as.string.terminator) {
12248 pm_token_buffer_push_byte(&token_buffer, peeked);
12249 parser->current.end++;
12250 } else if (lex_mode->as.string.interpolation) {
12251 escape_read(parser, &token_buffer.buffer, PM_ESCAPE_FLAG_NONE);
12252
12253 /* The escape consumed a newline with a heredoc body pending. */
12254 if (parser->next_start != NULL) {
12255 pm_token_buffer_copy(parser, &token_buffer);
12256 LEX(PM_TOKEN_STRING_CONTENT);
12257 }
12258 } else {
12259 pm_token_buffer_push_byte(&token_buffer, '\\');
12260 pm_token_buffer_push_escaped(&token_buffer, parser);
12261 }
12262
12263 break;
12264 }
12265
12266 token_buffer.cursor = parser->current.end;
12267 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12268 break;
12269 }
12270 case '#': {
12271 pm_token_type_t type = lex_interpolation(parser, breakpoint);
12272
12273 if (!type) {
12274 // If we haven't returned at this point then we had something that
12275 // looked like an interpolated class or instance variable like "#@"
12276 // but wasn't actually. In this case we'll just skip to the next
12277 // breakpoint.
12278 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, PM_STRPBRK_STRING);
12279 break;
12280 }
12281
12282 if (type == PM_TOKEN_STRING_CONTENT) {
12283 pm_token_buffer_flush(parser, &token_buffer);
12284 }
12285
12286 LEX(type);
12287 }
12288 default:
12289 PRISM_ASSERT(false && "unreachable");
12290 }
12291 }
12292
12293 /* With no breakpoint left, the string is not terminated and the rest
12294 * of the file is content, so this token runs to the end of the file.
12295 * That keeps the content in one piece. */
12296 lex_mode->as.string.joined_at_eof = parser->current.end > parser->current.start && parser->current.end < parser->end;
12297 parser->current.end = parser->end;
12298 pm_token_buffer_flush(parser, &token_buffer);
12299 LEX(PM_TOKEN_STRING_CONTENT);
12300 }
12301 case PM_LEX_HEREDOC: {
12302 // First, we'll set to start of this token.
12303 if (parser->next_start == NULL) {
12304 parser->current.start = parser->current.end;
12305 } else {
12306 parser->current.start = parser->next_start;
12307 parser->current.end = parser->next_start;
12308 parser->heredoc_end = NULL;
12309 parser->next_start = NULL;
12310 }
12311
12312 // Now let's grab the information about the identifier off of the
12313 // current lex mode.
12314 pm_lex_mode_t *lex_mode = parser->lex_modes.current;
12315 pm_heredoc_lex_mode_t *heredoc_lex_mode = &lex_mode->as.heredoc.base;
12316
12317 bool line_continuation = lex_mode->as.heredoc.line_continuation;
12318 lex_mode->as.heredoc.line_continuation = false;
12319
12320 // We'll check if we're at the end of the file. If we are, then we
12321 // will add an error (because we weren't able to find the
12322 // terminator) but still continue parsing so that content after the
12323 // declaration of the heredoc can be parsed.
12324 if (parser->current.end >= parser->end) {
12325 pm_parser_err_heredoc_term(parser, heredoc_lex_mode->ident_start, heredoc_lex_mode->ident_length);
12326 parser->next_start = lex_mode->as.heredoc.next_start;
12327 parser->heredoc_end = parser->current.end;
12328 lex_state_set(parser, PM_LEX_STATE_END);
12329 lex_mode_pop(parser);
12330 LEX(PM_TOKEN_HEREDOC_END);
12331 }
12332
12333 const uint8_t *ident_start = heredoc_lex_mode->ident_start;
12334 size_t ident_length = heredoc_lex_mode->ident_length;
12335
12336 // If we are immediately following a newline and we have hit the
12337 // terminator, then we need to return the ending of the heredoc.
12338 if (current_token_starts_line(parser)) {
12339 const uint8_t *start = parser->current.start;
12340
12341 if (!line_continuation && (start + ident_length <= parser->end)) {
12342 const uint8_t *newline = next_newline(start, parser->end - start);
12343 const uint8_t *ident_end = newline;
12344 const uint8_t *terminator_end = newline;
12345
12346 if (newline == NULL) {
12347 terminator_end = parser->end;
12348 ident_end = parser->end;
12349 } else {
12350 terminator_end++;
12351 if (newline[-1] == '\r') {
12352 ident_end--; // Remove \r
12353 }
12354 }
12355
12356 /*
12357 * The line on which we are looking for the terminator runs
12358 * from `start` to `ident_end`. If that line is shorter than
12359 * the identifier, the terminator cannot fit and we skip the
12360 * match. This also keeps `terminator_start` from pointing
12361 * before `start`.
12362 */
12363 if ((size_t) (ident_end - start) >= ident_length) {
12364 const uint8_t *terminator_start = ident_end - ident_length;
12365 const uint8_t *cursor = start;
12366
12367 if (heredoc_lex_mode->indent == PM_HEREDOC_INDENT_DASH || heredoc_lex_mode->indent == PM_HEREDOC_INDENT_TILDE) {
12368 while (cursor < terminator_start && pm_char_is_inline_whitespace(*cursor)) {
12369 cursor++;
12370 }
12371 }
12372
12373 if (
12374 (cursor == terminator_start) &&
12375 (memcmp(terminator_start, ident_start, ident_length) == 0)
12376 ) {
12377 if (newline != NULL) {
12378 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(newline - parser->start + 1));
12379 }
12380
12381 parser->current.end = terminator_end;
12382 parser->next_start = lex_mode->as.heredoc.next_start;
12383 parser->heredoc_end = parser->current.end;
12384
12385 lex_state_set(parser, PM_LEX_STATE_END);
12386 lex_mode_pop(parser);
12387 LEX(PM_TOKEN_HEREDOC_END);
12388 }
12389 }
12390 }
12391
12392 size_t whitespace = pm_heredoc_strspn_inline_whitespace(parser, &start, heredoc_lex_mode->indent);
12393 if (
12394 heredoc_lex_mode->indent == PM_HEREDOC_INDENT_TILDE &&
12395 lex_mode->as.heredoc.common_whitespace != NULL &&
12396 (*lex_mode->as.heredoc.common_whitespace > whitespace) &&
12397 peek_at(parser, start) != '\n'
12398 ) {
12399 *lex_mode->as.heredoc.common_whitespace = whitespace;
12400 }
12401 }
12402
12403 // Otherwise we'll be parsing string content. These are the places
12404 // where we need to split up the content of the heredoc. We'll use
12405 // strpbrk to find the first of these characters.
12406 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE] = "\r\n\\#";
12407
12408 pm_heredoc_quote_t quote = heredoc_lex_mode->quote;
12409 /* Single-quoted heredocs preserve bytes without validating them. */
12410 pm_strpbrk_mode_t mode = (quote == PM_HEREDOC_QUOTE_SINGLE) ? PM_STRPBRK_RAW : PM_STRPBRK_STRING;
12411 if (quote == PM_HEREDOC_QUOTE_SINGLE) {
12412 breakpoints[3] = '\0';
12413 }
12414
12415 const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, mode);
12416 pm_token_buffer_t token_buffer = { 0 };
12417 bool was_line_continuation = false;
12418
12419 while (breakpoint != NULL) {
12420 switch (*breakpoint) {
12421 case '\0':
12422 // Skip directly past the null character.
12423 parser->current.end = breakpoint + 1;
12424 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, mode);
12425 break;
12426 case '\r':
12427 parser->current.end = breakpoint + 1;
12428
12429 if (peek_at(parser, breakpoint + 1) != '\n') {
12430 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, mode);
12431 break;
12432 }
12433
12434 // If we hit a \r\n sequence, then we want to replace it
12435 // with a single \n character in the final string.
12436 breakpoint++;
12437 pm_token_buffer_escape(parser, &token_buffer);
12438 token_buffer.cursor = breakpoint;
12439
12441 case '\n': {
12442 if (parser->heredoc_end != NULL && (parser->heredoc_end > breakpoint)) {
12443 parser_flush_heredoc_end(parser);
12444 parser->current.end = breakpoint + 1;
12445 pm_token_buffer_flush(parser, &token_buffer);
12446 /* A continued line cannot hold this heredoc's terminator. */
12447 lex_mode->as.heredoc.line_continuation = was_line_continuation;
12448 LEX(PM_TOKEN_STRING_CONTENT);
12449 }
12450
12451 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(breakpoint - parser->start + 1));
12452
12453 // If we have a - or ~ heredoc, then we can match after
12454 // some leading whitespace.
12455 const uint8_t *start = breakpoint + 1;
12456
12457 if (!was_line_continuation && (start + ident_length <= parser->end)) {
12458 // We want to match the terminator starting from the end of the line in case
12459 // there is whitespace in the ident such as <<-' DOC' or <<~' DOC'.
12460 const uint8_t *newline = next_newline(start, parser->end - start);
12461
12462 if (newline == NULL) {
12463 newline = parser->end;
12464 } else if (newline[-1] == '\r') {
12465 newline--; // Remove \r
12466 }
12467
12468 /*
12469 * The line on which we are looking for the
12470 * terminator runs from `start` to `newline`. If
12471 * that line is shorter than the identifier the
12472 * terminator cannot fit and we skip the match.
12473 * This also keeps `terminator_start` from pointing
12474 * before `start`.
12475 */
12476 if ((size_t) (newline - start) >= ident_length) {
12477 // Start of a possible terminator.
12478 const uint8_t *terminator_start = newline - ident_length;
12479
12480 // Cursor to check for the leading whitespace. We skip the
12481 // leading whitespace if we have a - or ~ heredoc.
12482 const uint8_t *cursor = start;
12483
12484 if (heredoc_lex_mode->indent == PM_HEREDOC_INDENT_DASH || heredoc_lex_mode->indent == PM_HEREDOC_INDENT_TILDE) {
12485 while (cursor < terminator_start && pm_char_is_inline_whitespace(*cursor)) {
12486 cursor++;
12487 }
12488 }
12489
12490 if (
12491 cursor == terminator_start &&
12492 (memcmp(terminator_start, ident_start, ident_length) == 0)
12493 ) {
12494 parser->current.end = breakpoint + 1;
12495 pm_token_buffer_flush(parser, &token_buffer);
12496 LEX(PM_TOKEN_STRING_CONTENT);
12497 }
12498 }
12499 }
12500
12501 size_t whitespace = pm_heredoc_strspn_inline_whitespace(parser, &start, lex_mode->as.heredoc.base.indent);
12502
12503 // If we have hit a newline that is followed by a valid
12504 // terminator, then we need to return the content of the
12505 // heredoc here as string content. Then, the next time a
12506 // token is lexed, it will match again and return the
12507 // end of the heredoc.
12508 if (lex_mode->as.heredoc.base.indent == PM_HEREDOC_INDENT_TILDE) {
12509 if ((lex_mode->as.heredoc.common_whitespace != NULL) && (*lex_mode->as.heredoc.common_whitespace > whitespace) && peek_at(parser, start) != '\n') {
12510 *lex_mode->as.heredoc.common_whitespace = whitespace;
12511 }
12512
12513 parser->current.end = breakpoint + 1;
12514 pm_token_buffer_flush(parser, &token_buffer);
12515 LEX(PM_TOKEN_STRING_CONTENT);
12516 }
12517
12518 // Otherwise we hit a newline and it wasn't followed by
12519 // a terminator, so we can continue parsing.
12520 parser->current.end = breakpoint + 1;
12521 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, mode);
12522 break;
12523 }
12524 case '\\': {
12525 // If we hit an escape, then we need to skip past
12526 // however many characters the escape takes up. However
12527 // it's important that if \n or \r\n are escaped, we
12528 // stop looping before the newline and not after the
12529 // newline so that we can still potentially find the
12530 // terminator of the heredoc.
12531 parser->current.end = breakpoint + 1;
12532
12533 // If we've hit the end of the file, then break out of
12534 // the loop by setting the breakpoint to NULL.
12535 if (parser->current.end == parser->end) {
12536 breakpoint = NULL;
12537 continue;
12538 }
12539
12540 pm_token_buffer_escape(parser, &token_buffer);
12541 uint8_t peeked = peek(parser);
12542
12543 if (quote == PM_HEREDOC_QUOTE_SINGLE) {
12544 switch (peeked) {
12545 case '\r':
12546 parser->current.end++;
12547 if (peek(parser) != '\n') {
12548 pm_token_buffer_push_byte(&token_buffer, '\\');
12549 pm_token_buffer_push_byte(&token_buffer, '\r');
12550 break;
12551 }
12553 case '\n':
12554 pm_token_buffer_push_byte(&token_buffer, '\\');
12555 pm_token_buffer_push_byte(&token_buffer, '\n');
12556 token_buffer.cursor = parser->current.end + 1;
12557 breakpoint = parser->current.end;
12558 continue;
12559 default:
12560 pm_token_buffer_push_byte(&token_buffer, '\\');
12561 pm_token_buffer_push_escaped(&token_buffer, parser);
12562 break;
12563 }
12564 } else {
12565 switch (peeked) {
12566 case '\r':
12567 parser->current.end++;
12568 if (peek(parser) != '\n') {
12569 pm_token_buffer_push_byte(&token_buffer, '\r');
12570 break;
12571 }
12573 case '\n':
12574 // If we are in a tilde here, we should
12575 // break out of the loop and return the
12576 // string content.
12577 if (heredoc_lex_mode->indent == PM_HEREDOC_INDENT_TILDE) {
12578 const uint8_t *end = parser->current.end;
12579
12580 if (parser->heredoc_end == NULL) {
12581 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(end - parser->start + 1));
12582 } else {
12583 parser_flush_heredoc_end(parser);
12584 }
12585
12586 // Here we want the buffer to only
12587 // include up to the backslash.
12588 parser->current.end = breakpoint;
12589 pm_token_buffer_flush(parser, &token_buffer);
12590
12591 // Now we can advance the end of the
12592 // token past the newline.
12593 parser->current.end = end + 1;
12594 lex_mode->as.heredoc.line_continuation = true;
12595 LEX(PM_TOKEN_STRING_CONTENT);
12596 }
12597
12598 was_line_continuation = true;
12599 token_buffer.cursor = parser->current.end + 1;
12600 breakpoint = parser->current.end;
12601 continue;
12602 default:
12603 escape_read(parser, &token_buffer.buffer, PM_ESCAPE_FLAG_NONE);
12604
12605 /* The escape consumed a newline with a heredoc
12606 * body pending, and the line it continues onto
12607 * cannot hold this heredoc's terminator. */
12608 if (parser->next_start != NULL) {
12609 /* Dedenting applies at the start of each
12610 * token, and the continued line is not
12611 * dedented, so it stays in this token. */
12612 if (heredoc_lex_mode->indent == PM_HEREDOC_INDENT_TILDE) {
12613 parser->current.end = parser->next_start;
12614 parser->next_start = NULL;
12615 break;
12616 }
12617
12618 pm_token_buffer_copy(parser, &token_buffer);
12619 lex_mode->as.heredoc.line_continuation = true;
12620 LEX(PM_TOKEN_STRING_CONTENT);
12621 }
12622
12623 break;
12624 }
12625 }
12626
12627 token_buffer.cursor = parser->current.end;
12628 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, mode);
12629 break;
12630 }
12631 case '#': {
12632 pm_token_type_t type = lex_interpolation(parser, breakpoint);
12633
12634 if (!type) {
12635 // If we haven't returned at this point then we had
12636 // something that looked like an interpolated class
12637 // or instance variable like "#@" but wasn't
12638 // actually. In this case we'll just skip to the
12639 // next breakpoint.
12640 breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, mode);
12641 break;
12642 }
12643
12644 if (type == PM_TOKEN_STRING_CONTENT) {
12645 pm_token_buffer_flush(parser, &token_buffer);
12646 }
12647
12648 LEX(type);
12649 }
12650 default:
12651 PRISM_ASSERT(false && "unreachable");
12652 }
12653
12654 was_line_continuation = false;
12655 }
12656
12657 if (parser->current.end > parser->current.start) {
12658 parser->current.end = parser->end;
12659 pm_token_buffer_flush(parser, &token_buffer);
12660 LEX(PM_TOKEN_STRING_CONTENT);
12661 }
12662
12663 // If we've hit the end of the string, then this is an unterminated
12664 // heredoc. In that case we'll return a string content token.
12665 parser->current.end = parser->end;
12666 pm_token_buffer_flush(parser, &token_buffer);
12667 LEX(PM_TOKEN_STRING_CONTENT);
12668 }
12669 }
12670
12671 PRISM_ASSERT(false && "unreachable");
12672}
12673
12674#undef LEX
12675
12676/******************************************************************************/
12677/* Parse functions */
12678/******************************************************************************/
12679
12688typedef enum {
12689 PM_BINDING_POWER_UNSET = 0, // used to indicate this token cannot be used as an infix operator
12690 PM_BINDING_POWER_STATEMENT = 2,
12691 PM_BINDING_POWER_MODIFIER_RESCUE = 4, // rescue
12692 PM_BINDING_POWER_MODIFIER = 6, // if unless until while
12693 PM_BINDING_POWER_COMPOSITION = 8, // and or
12694 PM_BINDING_POWER_NOT = 10, // not
12695 PM_BINDING_POWER_MATCH = 12, // => in
12696 PM_BINDING_POWER_DEFINED = 14, // defined?
12697 PM_BINDING_POWER_MULTI_ASSIGNMENT = 16, // =
12698 PM_BINDING_POWER_ASSIGNMENT = 18, // = += -= *= /= %= &= |= ^= &&= ||= <<= >>= **=
12699 PM_BINDING_POWER_TERNARY = 20, // ?:
12700 PM_BINDING_POWER_RANGE = 22, // .. ...
12701 PM_BINDING_POWER_LOGICAL_OR = 24, // ||
12702 PM_BINDING_POWER_LOGICAL_AND = 26, // &&
12703 PM_BINDING_POWER_EQUALITY = 28, // <=> == === != =~ !~
12704 PM_BINDING_POWER_COMPARISON = 30, // > >= < <=
12705 PM_BINDING_POWER_BITWISE_OR = 32, // | ^
12706 PM_BINDING_POWER_BITWISE_AND = 34, // &
12707 PM_BINDING_POWER_SHIFT = 36, // << >>
12708 PM_BINDING_POWER_TERM = 38, // + -
12709 PM_BINDING_POWER_FACTOR = 40, // * / %
12710 PM_BINDING_POWER_UMINUS = 42, // -@
12711 PM_BINDING_POWER_EXPONENT = 44, // **
12712 PM_BINDING_POWER_UNARY = 46, // ! ~ +@
12713 PM_BINDING_POWER_INDEX = 48, // [] []=
12714 PM_BINDING_POWER_CALL = 50, // :: .
12715 PM_BINDING_POWER_MAX = 52
12716} pm_binding_power_t;
12717
12722typedef struct {
12724 pm_binding_power_t left;
12725
12727 pm_binding_power_t right;
12728
12731
12738
12739#define BINDING_POWER_ASSIGNMENT { PM_BINDING_POWER_UNARY, PM_BINDING_POWER_ASSIGNMENT, true, false }
12740#define LEFT_ASSOCIATIVE(precedence) { precedence, precedence + 1, true, false }
12741#define RIGHT_ASSOCIATIVE(precedence) { precedence, precedence, true, false }
12742#define NON_ASSOCIATIVE(precedence) { precedence, precedence + 1, true, true }
12743#define RIGHT_ASSOCIATIVE_UNARY(precedence) { precedence, precedence, false, false }
12744
12745pm_binding_powers_t pm_binding_powers[PM_TOKEN_MAXIMUM] = {
12746 // rescue
12747 [PM_TOKEN_KEYWORD_RESCUE_MODIFIER] = { PM_BINDING_POWER_MODIFIER_RESCUE, PM_BINDING_POWER_COMPOSITION, true, false },
12748
12749 // if unless until while
12750 [PM_TOKEN_KEYWORD_IF_MODIFIER] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_MODIFIER),
12751 [PM_TOKEN_KEYWORD_UNLESS_MODIFIER] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_MODIFIER),
12752 [PM_TOKEN_KEYWORD_UNTIL_MODIFIER] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_MODIFIER),
12753 [PM_TOKEN_KEYWORD_WHILE_MODIFIER] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_MODIFIER),
12754
12755 // and or
12756 [PM_TOKEN_KEYWORD_AND] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_COMPOSITION),
12757 [PM_TOKEN_KEYWORD_OR] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_COMPOSITION),
12758
12759 // => in
12760 [PM_TOKEN_EQUAL_GREATER] = NON_ASSOCIATIVE(PM_BINDING_POWER_MATCH),
12761 [PM_TOKEN_KEYWORD_IN] = NON_ASSOCIATIVE(PM_BINDING_POWER_MATCH),
12762
12763 // &&= &= ^= = >>= <<= -= %= |= ||= += /= *= **=
12764 [PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL] = BINDING_POWER_ASSIGNMENT,
12765 [PM_TOKEN_AMPERSAND_EQUAL] = BINDING_POWER_ASSIGNMENT,
12766 [PM_TOKEN_CARET_EQUAL] = BINDING_POWER_ASSIGNMENT,
12767 [PM_TOKEN_EQUAL] = BINDING_POWER_ASSIGNMENT,
12768 [PM_TOKEN_GREATER_GREATER_EQUAL] = BINDING_POWER_ASSIGNMENT,
12769 [PM_TOKEN_LESS_LESS_EQUAL] = BINDING_POWER_ASSIGNMENT,
12770 [PM_TOKEN_MINUS_EQUAL] = BINDING_POWER_ASSIGNMENT,
12771 [PM_TOKEN_PERCENT_EQUAL] = BINDING_POWER_ASSIGNMENT,
12772 [PM_TOKEN_PIPE_EQUAL] = BINDING_POWER_ASSIGNMENT,
12773 [PM_TOKEN_PIPE_PIPE_EQUAL] = BINDING_POWER_ASSIGNMENT,
12774 [PM_TOKEN_PLUS_EQUAL] = BINDING_POWER_ASSIGNMENT,
12775 [PM_TOKEN_SLASH_EQUAL] = BINDING_POWER_ASSIGNMENT,
12776 [PM_TOKEN_STAR_EQUAL] = BINDING_POWER_ASSIGNMENT,
12777 [PM_TOKEN_STAR_STAR_EQUAL] = BINDING_POWER_ASSIGNMENT,
12778
12779 // ?:
12780 [PM_TOKEN_QUESTION_MARK] = RIGHT_ASSOCIATIVE(PM_BINDING_POWER_TERNARY),
12781
12782 // .. ...
12783 [PM_TOKEN_DOT_DOT] = NON_ASSOCIATIVE(PM_BINDING_POWER_RANGE),
12784 [PM_TOKEN_DOT_DOT_DOT] = NON_ASSOCIATIVE(PM_BINDING_POWER_RANGE),
12785 [PM_TOKEN_UDOT_DOT] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_LOGICAL_OR),
12786 [PM_TOKEN_UDOT_DOT_DOT] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_LOGICAL_OR),
12787
12788 // ||
12789 [PM_TOKEN_PIPE_PIPE] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_LOGICAL_OR),
12790
12791 // &&
12792 [PM_TOKEN_AMPERSAND_AMPERSAND] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_LOGICAL_AND),
12793
12794 // != !~ == === =~ <=>
12795 [PM_TOKEN_BANG_EQUAL] = NON_ASSOCIATIVE(PM_BINDING_POWER_EQUALITY),
12796 [PM_TOKEN_BANG_TILDE] = NON_ASSOCIATIVE(PM_BINDING_POWER_EQUALITY),
12797 [PM_TOKEN_EQUAL_EQUAL] = NON_ASSOCIATIVE(PM_BINDING_POWER_EQUALITY),
12798 [PM_TOKEN_EQUAL_EQUAL_EQUAL] = NON_ASSOCIATIVE(PM_BINDING_POWER_EQUALITY),
12799 [PM_TOKEN_EQUAL_TILDE] = NON_ASSOCIATIVE(PM_BINDING_POWER_EQUALITY),
12800 [PM_TOKEN_LESS_EQUAL_GREATER] = NON_ASSOCIATIVE(PM_BINDING_POWER_EQUALITY),
12801
12802 // > >= < <=
12803 [PM_TOKEN_GREATER] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_COMPARISON),
12804 [PM_TOKEN_GREATER_EQUAL] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_COMPARISON),
12805 [PM_TOKEN_LESS] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_COMPARISON),
12806 [PM_TOKEN_LESS_EQUAL] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_COMPARISON),
12807
12808 // ^ |
12809 [PM_TOKEN_CARET] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_BITWISE_OR),
12810 [PM_TOKEN_PIPE] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_BITWISE_OR),
12811
12812 // &
12813 [PM_TOKEN_AMPERSAND] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_BITWISE_AND),
12814
12815 // >> <<
12816 [PM_TOKEN_GREATER_GREATER] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_SHIFT),
12817 [PM_TOKEN_LESS_LESS] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_SHIFT),
12818
12819 // - +
12820 [PM_TOKEN_MINUS] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_TERM),
12821 [PM_TOKEN_PLUS] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_TERM),
12822
12823 // % / *
12824 [PM_TOKEN_PERCENT] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_FACTOR),
12825 [PM_TOKEN_SLASH] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_FACTOR),
12826 [PM_TOKEN_STAR] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_FACTOR),
12827 [PM_TOKEN_USTAR] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_FACTOR),
12828
12829 // -@
12830 [PM_TOKEN_UMINUS] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_UMINUS),
12831 [PM_TOKEN_UMINUS_NUM] = { PM_BINDING_POWER_UMINUS, PM_BINDING_POWER_MAX, false, false },
12832
12833 // **
12834 [PM_TOKEN_STAR_STAR] = RIGHT_ASSOCIATIVE(PM_BINDING_POWER_EXPONENT),
12835 [PM_TOKEN_USTAR_STAR] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_UNARY),
12836
12837 // ! ~ +@
12838 [PM_TOKEN_BANG] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_UNARY),
12839 [PM_TOKEN_TILDE] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_UNARY),
12840 [PM_TOKEN_UPLUS] = RIGHT_ASSOCIATIVE_UNARY(PM_BINDING_POWER_UNARY),
12841
12842 // [
12843 [PM_TOKEN_BRACKET_LEFT] = LEFT_ASSOCIATIVE(PM_BINDING_POWER_INDEX),
12844
12845 // :: . &.
12846 [PM_TOKEN_COLON_COLON] = RIGHT_ASSOCIATIVE(PM_BINDING_POWER_CALL),
12847 [PM_TOKEN_DOT] = RIGHT_ASSOCIATIVE(PM_BINDING_POWER_CALL),
12848 [PM_TOKEN_AMPERSAND_DOT] = RIGHT_ASSOCIATIVE(PM_BINDING_POWER_CALL)
12849};
12850
12851#undef BINDING_POWER_ASSIGNMENT
12852#undef LEFT_ASSOCIATIVE
12853#undef RIGHT_ASSOCIATIVE
12854#undef RIGHT_ASSOCIATIVE_UNARY
12855
12859static PRISM_INLINE bool
12860match1(const pm_parser_t *parser, pm_token_type_t type) {
12861 return parser->current.type == type;
12862}
12863
12867static PRISM_INLINE bool
12868match2(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2) {
12869 return match1(parser, type1) || match1(parser, type2);
12870}
12871
12875static PRISM_INLINE bool
12876match3(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_token_type_t type3) {
12877 return match1(parser, type1) || match1(parser, type2) || match1(parser, type3);
12878}
12879
12883static PRISM_INLINE bool
12884match4(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_token_type_t type3, pm_token_type_t type4) {
12885 return match1(parser, type1) || match1(parser, type2) || match1(parser, type3) || match1(parser, type4);
12886}
12887
12891static PRISM_INLINE bool
12892match5(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_token_type_t type3, pm_token_type_t type4, pm_token_type_t type5) {
12893 return match1(parser, type1) || match1(parser, type2) || match1(parser, type3) || match1(parser, type4) || match1(parser, type5);
12894}
12895
12899static PRISM_INLINE bool
12900match6(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_token_type_t type3, pm_token_type_t type4, pm_token_type_t type5, pm_token_type_t type6) {
12901 return match1(parser, type1) || match1(parser, type2) || match1(parser, type3) || match1(parser, type4) || match1(parser, type5) || match1(parser, type6);
12902}
12903
12907static PRISM_INLINE bool
12908match8(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_token_type_t type3, pm_token_type_t type4, pm_token_type_t type5, pm_token_type_t type6, pm_token_type_t type7, pm_token_type_t type8) {
12909 return match1(parser, type1) || match1(parser, type2) || match1(parser, type3) || match1(parser, type4) || match1(parser, type5) || match1(parser, type6) || match1(parser, type7) || match1(parser, type8);
12910}
12911
12918static bool
12919accept1(pm_parser_t *parser, pm_token_type_t type) {
12920 if (match1(parser, type)) {
12921 parser_lex(parser);
12922 return true;
12923 }
12924 return false;
12925}
12926
12931static PRISM_INLINE bool
12932accept2(pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2) {
12933 if (match2(parser, type1, type2)) {
12934 parser_lex(parser);
12935 return true;
12936 }
12937 return false;
12938}
12939
12951static void
12952expect1(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t diag_id) {
12953 if (accept1(parser, type)) return;
12954
12955 const uint8_t *location = parser->previous.end;
12956 pm_parser_err(parser, U32(location - parser->start), 0, diag_id);
12957
12958 parser->previous.start = location;
12959 parser->previous.type = 0;
12960}
12961
12966static void
12967expect2(pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_diagnostic_id_t diag_id) {
12968 if (accept2(parser, type1, type2)) return;
12969
12970 const uint8_t *location = parser->previous.end;
12971 pm_parser_err(parser, U32(location - parser->start), 0, diag_id);
12972
12973 parser->previous.start = location;
12974 parser->previous.type = 0;
12975}
12976
12981static void
12982expect1_heredoc_term(pm_parser_t *parser, const uint8_t *ident_start, size_t ident_length) {
12983 if (match1(parser, PM_TOKEN_HEREDOC_END)) {
12984 lex_state_set(parser, PM_LEX_STATE_END);
12985 parser_lex(parser);
12986 } else {
12987 pm_parser_err_heredoc_term(parser, ident_start, ident_length);
12988 parser->previous.start = parser->previous.end;
12989 parser->previous.type = 0;
12990 }
12991}
12992
12999static void
13000expect1_opening(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t diag_id, const pm_token_t *opening) {
13001 if (accept1(parser, type)) return;
13002
13003 const uint8_t *start = opening->start;
13004 pm_parser_err(parser, U32(start - parser->start), U32(opening->end - start), diag_id);
13005
13006 parser->previous.start = parser->previous.end;
13007 parser->previous.type = 0;
13008}
13009
13011#define PM_PARSE_ACCEPTS_COMMAND_CALL ((uint8_t) 0x1)
13012#define PM_PARSE_ACCEPTS_LABEL ((uint8_t) 0x2)
13013#define PM_PARSE_ACCEPTS_DO_BLOCK ((uint8_t) 0x4)
13014#define PM_PARSE_IN_ENDLESS_DEF ((uint8_t) 0x8)
13015
13025#define PM_PARSE_ACCEPTS_STATEMENT ((uint8_t) 0x10)
13026
13038#define PM_PARSE_ACCEPTS_COMMAND_DEF ((uint8_t) 0x20)
13039
13040static pm_node_t *
13041parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth);
13042
13043static pm_node_t *
13044parse_heredoc(pm_parser_t *parser, uint16_t depth);
13045
13046/* Resource failure must preserve the same token/context ownership as ordinary
13047 * syntax recovery. In particular, do not consume an enclosing terminator or
13048 * leave an unparsed heredoc's lexer mode pending. */
13049static pm_node_t *
13050parse_resource_recovery(pm_parser_t *parser, uint16_t depth) {
13051 if (depth >= PRISM_DEPTH_MAXIMUM) {
13052 pm_parser_err_current(parser, PM_ERR_NESTING_TOO_DEEP);
13053 } else {
13054 pm_parser_err_stack_exhausted(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current));
13055 }
13056
13057 if (context_recoverable(parser, &parser->current)) {
13058 parser->recovering = true;
13059 }
13060
13061 if (match1(parser, PM_TOKEN_HEREDOC_START)) {
13062 pm_token_t opening = parser->current;
13063 PRISM_ASSERT(parser->lex_modes.current->mode == PM_LEX_HEREDOC);
13064
13065 /* Discard the heredoc without recursing. */
13066 parser->lex_modes.current->as.heredoc.base.quote = PM_HEREDOC_QUOTE_SINGLE;
13067
13068 do {
13069 parser_lex(parser);
13070 } while (!match2(parser, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF));
13071
13072 if (match1(parser, PM_TOKEN_HEREDOC_END)) {
13073 lex_state_set(parser, PM_LEX_STATE_END);
13074 parser_lex(parser);
13075 }
13076
13077 parser->previous = opening;
13078 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &opening), PM_TOKEN_LENGTH(&opening)));
13079 }
13080
13081 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_END(parser, &parser->previous), 0));
13082}
13083
13084/* Discard an unexpected token without abandoning lexer state owned by it.
13085 * A heredoc opener has already pushed its lex mode and scheduled a source
13086 * jump, so its body must be consumed before resuming on the declaration line. */
13087static void
13088parse_discard_token(pm_parser_t *parser, uint16_t depth) {
13089 if (match1(parser, PM_TOKEN_HEREDOC_START)) {
13090 (void) parse_heredoc(parser, depth);
13091 } else {
13092 parser_lex(parser);
13093 }
13094}
13095
13100static pm_node_t *
13101parse_value_expression(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
13102 pm_node_t *node = parse_expression(parser, binding_power, flags, diag_id, depth);
13103 pm_assert_value_expression(parser, node);
13104 return node;
13105}
13106
13125static PRISM_INLINE bool
13126token_begins_expression_p(pm_token_type_t type) {
13127 switch (type) {
13128 case PM_TOKEN_EQUAL_GREATER:
13129 case PM_TOKEN_KEYWORD_IN:
13130 // We need to special case this because it is a binary operator that
13131 // should not be marked as beginning an expression.
13132 return false;
13133 case PM_TOKEN_BRACE_RIGHT:
13134 case PM_TOKEN_BRACKET_RIGHT:
13135 case PM_TOKEN_COLON:
13136 case PM_TOKEN_COMMA:
13137 case PM_TOKEN_EMBEXPR_END:
13138 case PM_TOKEN_EOF:
13139 case PM_TOKEN_HEREDOC_END:
13140 case PM_TOKEN_LAMBDA_BEGIN:
13141 case PM_TOKEN_KEYWORD_DO:
13142 case PM_TOKEN_KEYWORD_DO_BLOCK:
13143 case PM_TOKEN_KEYWORD_DO_LAMBDA:
13144 case PM_TOKEN_KEYWORD_DO_LOOP:
13145 case PM_TOKEN_KEYWORD_END:
13146 case PM_TOKEN_KEYWORD_ELSE:
13147 case PM_TOKEN_KEYWORD_ELSIF:
13148 case PM_TOKEN_KEYWORD_ENSURE:
13149 case PM_TOKEN_KEYWORD_THEN:
13150 case PM_TOKEN_KEYWORD_RESCUE:
13151 case PM_TOKEN_KEYWORD_WHEN:
13152 case PM_TOKEN_NEWLINE:
13153 case PM_TOKEN_PARENTHESIS_RIGHT:
13154 case PM_TOKEN_SEMICOLON:
13155 // The reason we need this short-circuit is because we're using the
13156 // binding powers table to tell us if the subsequent token could
13157 // potentially be the start of an expression. If there _is_ a binding
13158 // power for one of these tokens, then we should remove it from this list
13159 // and let it be handled by the default case below.
13160 PRISM_ASSERT(pm_binding_powers[type].left == PM_BINDING_POWER_UNSET);
13161 return false;
13162 case PM_TOKEN_UAMPERSAND:
13163 // This is a special case because this unary operator cannot appear
13164 // as a general operator, it only appears in certain circumstances.
13165 return false;
13166 case PM_TOKEN_UCOLON_COLON:
13167 case PM_TOKEN_UMINUS:
13168 case PM_TOKEN_UMINUS_NUM:
13169 case PM_TOKEN_UPLUS:
13170 case PM_TOKEN_BANG:
13171 case PM_TOKEN_TILDE:
13172 case PM_TOKEN_UDOT_DOT:
13173 case PM_TOKEN_UDOT_DOT_DOT:
13174 // These unary tokens actually do have binding power associated with them
13175 // so that we can correctly place them into the precedence order. But we
13176 // want them to be marked as beginning an expression, so we need to
13177 // special case them here.
13178 return true;
13179 default:
13180 return pm_binding_powers[type].left == PM_BINDING_POWER_UNSET;
13181 }
13182}
13183
13199static PRISM_INLINE bool
13200token_begins_pattern_p(pm_token_type_t type) {
13201 return (
13202 token_begins_expression_p(type) ||
13203 type == PM_TOKEN_USTAR ||
13204 type == PM_TOKEN_USTAR_STAR ||
13205 type == PM_TOKEN_CARET
13206 );
13207}
13208
13216static PRISM_INLINE bool
13217token_begins_argument_p(pm_token_type_t type) {
13218 return (
13219 token_begins_expression_p(type) ||
13220 type == PM_TOKEN_UAMPERSAND ||
13221 type == PM_TOKEN_USTAR ||
13222 type == PM_TOKEN_USTAR_STAR
13223 );
13224}
13225
13230static pm_node_t *
13231parse_starred_expression(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
13232 if (accept1(parser, PM_TOKEN_USTAR)) {
13233 pm_token_t operator = parser->previous;
13234 pm_node_t *expression = parse_value_expression(parser, binding_power, (uint8_t) (flags & PM_PARSE_ACCEPTS_DO_BLOCK), PM_ERR_EXPECT_EXPRESSION_AFTER_STAR, (uint16_t) (depth + 1));
13235 return UP(pm_splat_node_create(parser, &operator, expression));
13236 }
13237
13238 return parse_value_expression(parser, binding_power, flags, diag_id, depth);
13239}
13240
13241static pm_visit_result_t
13242pm_node_unreference_each(const pm_node_t *node, void *data) {
13243 switch (PM_NODE_TYPE(node)) {
13244 /* When we are about to destroy a set of nodes that could potentially
13245 * contain block exits for the current scope, we need to check if they
13246 * are contained in the list of block exits and remove them if they are.
13247 */
13248 case PM_BREAK_NODE:
13249 case PM_NEXT_NODE:
13250 case PM_REDO_NODE: {
13251 pm_parser_t *parser = (pm_parser_t *) data;
13252 size_t index = 0;
13253
13254 while (index < parser->current_block_exits->size) {
13255 pm_node_t *block_exit = parser->current_block_exits->nodes[index];
13256
13257 if (block_exit == node) {
13258 if (index + 1 < parser->current_block_exits->size) {
13259 memmove(
13260 &parser->current_block_exits->nodes[index],
13261 &parser->current_block_exits->nodes[index + 1],
13262 (parser->current_block_exits->size - index - 1) * sizeof(pm_node_t *)
13263 );
13264 }
13265 parser->current_block_exits->size--;
13266
13267 /* Note descending here because these nodes could have
13268 * arguments that are themselves block exits. */
13269 return PM_VISIT_DESCEND;
13270 }
13271
13272 index++;
13273 }
13274
13275 return PM_VISIT_DESCEND;
13276 }
13277 /* When an implicit local variable is written to or targeted, it becomes
13278 * a regular, named local variable. This branch removes it from the list
13279 * of implicit parameters when that happens. */
13280 case PM_LOCAL_VARIABLE_READ_NODE:
13281 case PM_IT_LOCAL_VARIABLE_READ_NODE: {
13282 pm_parser_t *parser = (pm_parser_t *) data;
13283 pm_scratch_node_list_t *implicit_parameters = &parser->current_scope->implicit_parameters;
13284
13285 for (size_t index = 0; index < implicit_parameters->size; index++) {
13286 if (implicit_parameters->nodes[index] == node) {
13287 /* If the node is not the last one in the list, we need to
13288 * shift the remaining nodes down to fill the gap. This is
13289 * extremely unlikely to happen. */
13290 if (index != implicit_parameters->size - 1) {
13291 memmove(&implicit_parameters->nodes[index], &implicit_parameters->nodes[index + 1], (implicit_parameters->size - index - 1) * sizeof(pm_node_t *));
13292 }
13293
13294 implicit_parameters->size--;
13295 break;
13296 }
13297 }
13298
13299 return PM_VISIT_SKIP;
13300 }
13301 default:
13302 return PM_VISIT_DESCEND;
13303 }
13304}
13305
13311static void
13312pm_node_unreference(pm_parser_t *parser, const pm_node_t *node) {
13313 pm_visit_node(node, pm_node_unreference_each, parser);
13314}
13315
13320static void
13321parse_write_name(pm_parser_t *parser, pm_constant_id_t *name_field) {
13322 // The method name needs to change. If we previously had
13323 // foo, we now need foo=. In this case we'll allocate a new
13324 // owned string, copy the previous method name in, and
13325 // append an =.
13326 pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, *name_field);
13327 size_t length = constant->length;
13328 uint8_t *name = PM_ARENA_ALLOC_ARRAY(parser->arena, uint8_t, length + 1);
13329
13330 memcpy(name, constant->start, length);
13331 name[length] = '=';
13332
13333 *name_field = pm_constant_pool_insert_owned(&parser->metadata_arena, &parser->constant_pool, name, length + 1);
13334}
13335
13342static pm_node_t *
13343parse_unwriteable_target(pm_parser_t *parser, pm_node_t *target) {
13344 switch (PM_NODE_TYPE(target)) {
13345 case PM_SOURCE_ENCODING_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_ENCODING); break;
13346 case PM_FALSE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_FALSE); break;
13347 case PM_SOURCE_FILE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_FILE); break;
13348 case PM_SOURCE_LINE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_LINE); break;
13349 case PM_NIL_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_NIL); break;
13350 case PM_SELF_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_SELF); break;
13351 case PM_TRUE_NODE: pm_parser_err_node(parser, target, PM_ERR_EXPRESSION_NOT_WRITABLE_TRUE); break;
13352 default: break;
13353 }
13354
13355 pm_constant_id_t name = pm_parser_constant_id_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target));
13356 pm_local_variable_target_node_t *result = pm_local_variable_target_node_create(parser, &target->location, name, 0);
13357
13358 return UP(result);
13359}
13360
13369static pm_node_t *
13370parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_parent) {
13371 switch (PM_NODE_TYPE(target)) {
13372 case PM_ERROR_RECOVERY_NODE:
13373 return target;
13374 case PM_SOURCE_ENCODING_NODE:
13375 case PM_FALSE_NODE:
13376 case PM_SOURCE_FILE_NODE:
13377 case PM_SOURCE_LINE_NODE:
13378 case PM_NIL_NODE:
13379 case PM_SELF_NODE:
13380 case PM_TRUE_NODE: {
13381 // In these special cases, we have specific error messages and we
13382 // will replace them with local variable writes.
13383 return parse_unwriteable_target(parser, target);
13384 }
13385 case PM_CLASS_VARIABLE_READ_NODE:
13386 PRISM_ASSERT(sizeof(pm_class_variable_target_node_t) == sizeof(pm_class_variable_read_node_t));
13387 target->type = PM_CLASS_VARIABLE_TARGET_NODE;
13388 return target;
13389 case PM_CONSTANT_PATH_NODE:
13390 if (context_def_p(parser)) {
13391 pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_IN_METHOD);
13392 }
13393
13394 PRISM_ASSERT(sizeof(pm_constant_path_target_node_t) == sizeof(pm_constant_path_node_t));
13395 target->type = PM_CONSTANT_PATH_TARGET_NODE;
13396
13397 return target;
13398 case PM_CONSTANT_READ_NODE:
13399 if (context_def_p(parser)) {
13400 pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_IN_METHOD);
13401 }
13402
13403 PRISM_ASSERT(sizeof(pm_constant_target_node_t) == sizeof(pm_constant_read_node_t));
13404 target->type = PM_CONSTANT_TARGET_NODE;
13405
13406 return target;
13407 case PM_BACK_REFERENCE_READ_NODE:
13408 case PM_NUMBERED_REFERENCE_READ_NODE:
13409 PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser, target, PM_ERR_WRITE_TARGET_READONLY);
13410 return UP(pm_error_recovery_node_create_unexpected(parser, target));
13411 case PM_GLOBAL_VARIABLE_READ_NODE:
13412 PRISM_ASSERT(sizeof(pm_global_variable_target_node_t) == sizeof(pm_global_variable_read_node_t));
13413 target->type = PM_GLOBAL_VARIABLE_TARGET_NODE;
13414 return target;
13415 case PM_LOCAL_VARIABLE_READ_NODE: {
13416 if (pm_token_is_numbered_parameter(parser, PM_NODE_START(target), PM_NODE_LENGTH(target))) {
13417 PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(target), PM_NODE_LENGTH(target), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(target));
13418 pm_node_unreference(parser, target);
13419 }
13420
13421 const pm_local_variable_read_node_t *cast = (const pm_local_variable_read_node_t *) target;
13422 PRISM_ASSERT(pm_parser_local_depth_constant_id(parser, cast->name) == (int) cast->depth);
13423 pm_locals_unread(parser, cast->name);
13424
13425 PRISM_ASSERT(sizeof(pm_local_variable_target_node_t) == sizeof(pm_local_variable_read_node_t));
13426 target->type = PM_LOCAL_VARIABLE_TARGET_NODE;
13427
13428 return target;
13429 }
13430 case PM_IT_LOCAL_VARIABLE_READ_NODE: {
13431 pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2);
13432 pm_node_t *node = UP(pm_local_variable_target_node_create(parser, &target->location, name, 0));
13433
13434 pm_node_unreference(parser, target);
13435
13436 return node;
13437 }
13438 case PM_INSTANCE_VARIABLE_READ_NODE:
13439 PRISM_ASSERT(sizeof(pm_instance_variable_target_node_t) == sizeof(pm_instance_variable_read_node_t));
13440 target->type = PM_INSTANCE_VARIABLE_TARGET_NODE;
13441 return target;
13442 case PM_MULTI_TARGET_NODE:
13443 if (splat_parent) {
13444 // Multi target is not accepted in all positions. If this is one
13445 // of them, then we need to add an error.
13446 pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_UNEXPECTED);
13447 }
13448
13449 return target;
13450 case PM_SPLAT_NODE: {
13451 pm_splat_node_t *splat = (pm_splat_node_t *) target;
13452
13453 if (splat->expression != NULL) {
13454 splat->expression = parse_target(parser, splat->expression, multiple, true);
13455 }
13456
13457 return UP(splat);
13458 }
13459 case PM_CALL_NODE: {
13460 pm_call_node_t *call = (pm_call_node_t *) target;
13461
13462 // If we have no arguments to the call node and we need this to be a
13463 // target then this is either a method call or a local variable
13464 // write.
13465 if (pm_call_node_writable_p(parser, call)) {
13466 if (call->receiver == NULL) {
13467 // When we get here, we have a local variable write, because it
13468 // was previously marked as a method call but now we have an =.
13469 // This looks like:
13470 //
13471 // foo = 1
13472 //
13473 // When it was parsed in the prefix position, foo was seen as a
13474 // method call with no receiver and no arguments. Now we have an
13475 // =, so we know it's a local variable write.
13476 pm_location_t message_loc = call->message_loc;
13477 pm_constant_id_t name = pm_parser_local_add_location(parser, &message_loc, 0);
13478
13479 return UP(pm_local_variable_target_node_create(parser, &message_loc, name, 0));
13480 }
13481
13482 if (multiple && PM_NODE_FLAG_P(call, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) {
13483 pm_parser_err_node(parser, (const pm_node_t *) call, PM_ERR_UNEXPECTED_SAFE_NAVIGATION);
13484 }
13485
13486 parse_write_name(parser, &call->name);
13487 return UP(pm_call_target_node_create(parser, call));
13488 }
13489
13490 // If there is no call operator and the message is "[]" then this is
13491 // an aref expression, and we can transform it into an aset
13492 // expression.
13493 if (PM_NODE_FLAG_P(call, PM_CALL_NODE_FLAGS_INDEX)) {
13494 return UP(pm_index_target_node_create(parser, call));
13495 }
13496 }
13498 default:
13499 /* Keep an invalid expression out of fields that require a target. */
13500 pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_UNEXPECTED);
13501 return UP(pm_error_recovery_node_create_unexpected(parser, target));
13502 }
13503}
13504
13509static pm_node_t *
13510parse_target_validate(pm_parser_t *parser, pm_node_t *target, bool multiple) {
13511 pm_node_t *result = parse_target(parser, target, multiple, false);
13512
13513 bool parenthesized = context_p(parser, PM_CONTEXT_PARENS);
13514 bool newline = parenthesized && accept1(parser, PM_TOKEN_NEWLINE);
13515
13516 /* Only the closing parenthesis may follow a newline. */
13517 bool assignment = !newline && (match1(parser, PM_TOKEN_EQUAL) || (context_p(parser, PM_CONTEXT_FOR_INDEX) && match1(parser, PM_TOKEN_KEYWORD_IN)));
13518 if (!assignment && !(parenthesized && match1(parser, PM_TOKEN_PARENTHESIS_RIGHT))) {
13519 pm_parser_err_node(parser, result, PM_ERR_WRITE_TARGET_UNEXPECTED);
13520 }
13521
13522 return result;
13523}
13524
13529static pm_node_t *
13530parse_shareable_constant_write(pm_parser_t *parser, pm_node_t *write) {
13531 pm_shareable_constant_value_t shareable_constant = pm_parser_scope_shareable_constant_get(parser);
13532
13533 if (shareable_constant != PM_SCOPE_SHAREABLE_CONSTANT_NONE) {
13534 return UP(pm_shareable_constant_node_create(parser, write, shareable_constant));
13535 }
13536
13537 return write;
13538}
13539
13543static pm_node_t *
13544parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_node_t *value) {
13545 switch (PM_NODE_TYPE(target)) {
13546 case PM_ERROR_RECOVERY_NODE:
13547 return target;
13548 case PM_CLASS_VARIABLE_READ_NODE: {
13549 pm_class_variable_write_node_t *node = pm_class_variable_write_node_create(parser, (pm_class_variable_read_node_t *) target, operator, value);
13550 return UP(node);
13551 }
13552 case PM_CONSTANT_PATH_NODE: {
13553 pm_node_t *node = UP(pm_constant_path_write_node_create(parser, (pm_constant_path_node_t *) target, operator, value));
13554
13555 if (context_def_p(parser)) {
13556 pm_parser_err_node(parser, node, PM_ERR_WRITE_TARGET_IN_METHOD);
13557 }
13558
13559 return parse_shareable_constant_write(parser, node);
13560 }
13561 case PM_CONSTANT_READ_NODE: {
13562 pm_node_t *node = UP(pm_constant_write_node_create(parser, (pm_constant_read_node_t *) target, operator, value));
13563
13564 if (context_def_p(parser)) {
13565 pm_parser_err_node(parser, node, PM_ERR_WRITE_TARGET_IN_METHOD);
13566 }
13567
13568 return parse_shareable_constant_write(parser, node);
13569 }
13570 case PM_BACK_REFERENCE_READ_NODE:
13571 case PM_NUMBERED_REFERENCE_READ_NODE:
13572 PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser, target, PM_ERR_WRITE_TARGET_READONLY);
13574 case PM_GLOBAL_VARIABLE_READ_NODE: {
13575 pm_global_variable_write_node_t *node = pm_global_variable_write_node_create(parser, target, operator, value);
13576 return UP(node);
13577 }
13578 case PM_LOCAL_VARIABLE_READ_NODE: {
13579 pm_local_variable_read_node_t *local_read = (pm_local_variable_read_node_t *) target;
13580
13581 pm_location_t location = target->location;
13582 pm_constant_id_t name = local_read->name;
13583 uint32_t depth = local_read->depth;
13584 pm_scope_t *scope = pm_parser_scope_find(parser, depth);
13585
13586 if (pm_token_is_numbered_parameter(parser, PM_NODE_START(target), PM_NODE_LENGTH(target))) {
13587 pm_diagnostic_id_t diag_id = (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) ? PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED : PM_ERR_PARAMETER_NUMBERED_RESERVED;
13588 PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(target), PM_NODE_LENGTH(target), diag_id, parser->start + PM_NODE_START(target));
13589 pm_node_unreference(parser, target);
13590 }
13591
13592 PRISM_ASSERT(pm_parser_local_depth_constant_id(parser, name) == (int) depth);
13593 pm_locals_unread(parser, name);
13594
13595 return UP(pm_local_variable_write_node_create(parser, name, depth, value, &location, operator));
13596 }
13597 case PM_IT_LOCAL_VARIABLE_READ_NODE: {
13598 pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2);
13599 pm_node_t *node = UP(pm_local_variable_write_node_create(parser, name, 0, value, &target->location, operator));
13600
13601 pm_node_unreference(parser, target);
13602
13603 return node;
13604 }
13605 case PM_INSTANCE_VARIABLE_READ_NODE: {
13606 pm_node_t *write_node = UP(pm_instance_variable_write_node_create(parser, (pm_instance_variable_read_node_t *) target, operator, value));
13607 return write_node;
13608 }
13609 case PM_MULTI_TARGET_NODE:
13610 return UP(pm_multi_write_node_create(parser, (pm_multi_target_node_t *) target, operator, value));
13611 case PM_SPLAT_NODE: {
13612 pm_splat_node_t *splat = (pm_splat_node_t *) target;
13613
13614 if (splat->expression != NULL) {
13615 splat->expression = parse_write(parser, splat->expression, operator, value);
13616 }
13617
13618 pm_multi_targets_t targets = { 0 };
13619 pm_multi_targets_append(parser, &targets, UP(splat));
13620 pm_multi_target_node_t *multi_target = pm_multi_target_node_create(parser, &targets, NULL, NULL);
13621
13622 return UP(pm_multi_write_node_create(parser, multi_target, operator, value));
13623 }
13624 case PM_CALL_NODE: {
13625 pm_call_node_t *call = (pm_call_node_t *) target;
13626
13627 // If we have no arguments to the call node and we need this to be a
13628 // target then this is either a method call or a local variable
13629 // write.
13630 if (pm_call_node_writable_p(parser, call)) {
13631 if (call->receiver == NULL) {
13632 // When we get here, we have a local variable write, because it
13633 // was previously marked as a method call but now we have an =.
13634 // This looks like:
13635 //
13636 // foo = 1
13637 //
13638 // When it was parsed in the prefix position, foo was seen as a
13639 // method call with no receiver and no arguments. Now we have an
13640 // =, so we know it's a local variable write.
13641 pm_location_t message_loc = call->message_loc;
13642
13643 pm_refute_numbered_parameter(parser, message_loc.start, message_loc.length);
13644 pm_parser_local_add_location(parser, &message_loc, 0);
13645
13646 pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, parser->start + PM_LOCATION_START(&message_loc), parser->start + PM_LOCATION_END(&message_loc));
13647 target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message_loc, operator));
13648
13649 return target;
13650 }
13651
13652 /* Turn an attribute read such as `foo.bar` into `foo.bar = value`. */
13653 pm_arguments_node_t *arguments = pm_arguments_node_single_create(parser, value);
13654 call->arguments = arguments;
13655 PM_NODE_LENGTH_SET_NODE(call, arguments);
13656 call->equal_loc = TOK2LOC(parser, operator);
13657
13658 parse_write_name(parser, &call->name);
13659 pm_node_flag_set(UP(call), PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE | pm_implicit_array_write_flags(value, PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY));
13660
13661 return UP(call);
13662 }
13663
13664 // If there is no call operator and the message is "[]" then this is
13665 // an aref expression, and we can transform it into an aset
13666 // expression.
13667 if (PM_NODE_FLAG_P(call, PM_CALL_NODE_FLAGS_INDEX)) {
13668 call->arguments = pm_arguments_node_extend(parser, call->arguments, value);
13669 PM_NODE_LENGTH_SET_NODE(target, value);
13670
13671 // Replace the name with "[]=".
13672 call->name = pm_parser_constant_id_constant(parser, "[]=", 3);
13673 call->equal_loc = TOK2LOC(parser, operator);
13674
13675 // Ensure that the arguments for []= don't contain keywords
13676 pm_index_arguments_check(parser, call->arguments, call->block);
13677 pm_node_flag_set(UP(call), PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE | pm_implicit_array_write_flags(value, PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY));
13678
13679 return target;
13680 }
13681
13682 // If there are arguments on the call node, then it can't be a
13683 // method call ending with = or a local variable write, so it must
13684 // be a syntax error. In this case we'll fall through to our default
13685 // handling. We need to free the value that we parsed because there
13686 // is no way for us to attach it to the tree at this point.
13687 //
13688 // Since it is possible for the value to contain an implicit
13689 // parameter somewhere in its subtree, we need to walk it and remove
13690 // any implicit parameters from the list of implicit parameters for
13691 // the current scope.
13692 pm_node_unreference(parser, value);
13693 }
13695 default:
13696 // In this case we have a node that we don't know how to convert into a
13697 // target. We need to treat it as an error. For now, we'll mark it as an
13698 // error and just skip right past it.
13699 pm_parser_err_token(parser, operator, PM_ERR_WRITE_TARGET_UNEXPECTED);
13700 return target;
13701 }
13702}
13703
13710static pm_node_t *
13711parse_unwriteable_write(pm_parser_t *parser, pm_node_t *target, const pm_token_t *equals, pm_node_t *value) {
13712 switch (PM_NODE_TYPE(target)) {
13713 case PM_SOURCE_ENCODING_NODE: pm_parser_err_token(parser, equals, PM_ERR_EXPRESSION_NOT_WRITABLE_ENCODING); break;
13714 case PM_FALSE_NODE: pm_parser_err_token(parser, equals, PM_ERR_EXPRESSION_NOT_WRITABLE_FALSE); break;
13715 case PM_SOURCE_FILE_NODE: pm_parser_err_token(parser, equals, PM_ERR_EXPRESSION_NOT_WRITABLE_FILE); break;
13716 case PM_SOURCE_LINE_NODE: pm_parser_err_token(parser, equals, PM_ERR_EXPRESSION_NOT_WRITABLE_LINE); break;
13717 case PM_NIL_NODE: pm_parser_err_token(parser, equals, PM_ERR_EXPRESSION_NOT_WRITABLE_NIL); break;
13718 case PM_SELF_NODE: pm_parser_err_token(parser, equals, PM_ERR_EXPRESSION_NOT_WRITABLE_SELF); break;
13719 case PM_TRUE_NODE: pm_parser_err_token(parser, equals, PM_ERR_EXPRESSION_NOT_WRITABLE_TRUE); break;
13720 default: break;
13721 }
13722
13723 pm_constant_id_t name = pm_parser_local_add_location(parser, &target->location, 1);
13724 pm_local_variable_write_node_t *result = pm_local_variable_write_node_create(parser, name, 0, value, &target->location, equals);
13725
13726 return UP(result);
13727}
13728
13739static pm_node_t *
13740parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t binding_power, uint16_t depth) {
13741 bool has_rest = PM_NODE_TYPE_P(first_target, PM_SPLAT_NODE);
13742
13743 pm_multi_targets_t targets = { 0 };
13744 pm_multi_targets_append(parser, &targets, parse_target(parser, first_target, true, false));
13745
13746 while (accept1(parser, PM_TOKEN_COMMA)) {
13747 if (accept1(parser, PM_TOKEN_USTAR)) {
13748 // Here we have a splat operator. It can have a name or be
13749 // anonymous. It can be the final target or be in the middle if
13750 // there haven't been any others yet.
13751 if (has_rest) {
13752 pm_parser_err_previous(parser, PM_ERR_MULTI_ASSIGN_MULTI_SPLATS);
13753 }
13754
13755 pm_token_t star_operator = parser->previous;
13756 pm_node_t *name = NULL;
13757
13758 if (token_begins_expression_p(parser->current.type)) {
13759 name = parse_expression(parser, binding_power, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_STAR, (uint16_t) (depth + 1));
13760 name = parse_target(parser, name, true, true);
13761 }
13762
13763 pm_node_t *splat = UP(pm_splat_node_create(parser, &star_operator, name));
13764 pm_multi_targets_append(parser, &targets, splat);
13765 has_rest = true;
13766 } else if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) {
13767 context_push(parser, PM_CONTEXT_MULTI_TARGET);
13768 pm_node_t *target = parse_expression(parser, binding_power, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA, (uint16_t) (depth + 1));
13769 target = parse_target(parser, target, true, false);
13770
13771 pm_multi_targets_append(parser, &targets, target);
13772 context_pop(parser);
13773 } else if (token_begins_expression_p(parser->current.type)) {
13774 pm_node_t *target = parse_expression(parser, binding_power, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA, (uint16_t) (depth + 1));
13775 target = parse_target(parser, target, true, false);
13776
13777 pm_multi_targets_append(parser, &targets, target);
13778 } else if (!match1(parser, PM_TOKEN_EOF)) {
13779 // If we get here, then we have a trailing , in a multi target node.
13780 // We'll add an implicit rest node to represent this.
13781 pm_node_t *rest = UP(pm_implicit_rest_node_create(parser, &parser->previous));
13782 pm_multi_targets_append(parser, &targets, rest);
13783 break;
13784 }
13785 }
13786
13787 return UP(pm_multi_target_node_create(parser, &targets, NULL, NULL));
13788}
13789
13794static pm_node_t *
13795parse_targets_validate(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t binding_power, uint16_t depth) {
13796 pm_node_t *result = parse_targets(parser, first_target, binding_power, depth);
13797
13798 /* A newline inside a parenthesized target may precede its closing parenthesis. */
13799 bool newline = (context_p(parser, PM_CONTEXT_PARENS) || context_p(parser, PM_CONTEXT_MULTI_TARGET)) && accept1(parser, PM_TOKEN_NEWLINE);
13800 if (!match1(parser, PM_TOKEN_PARENTHESIS_RIGHT) && (newline || !match1(parser, PM_TOKEN_EQUAL))) {
13801 pm_parser_err_node(parser, result, PM_ERR_WRITE_TARGET_UNEXPECTED);
13802 }
13803
13804 return result;
13805}
13806
13810static pm_statements_node_t *
13811parse_statements(pm_parser_t *parser, pm_context_t context, uint16_t depth) {
13812 // First, skip past any optional terminators that might be at the beginning
13813 // of the statements.
13814 while (accept2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE));
13815
13816 // If we have a terminator, then we can just return NULL.
13817 if (context_terminator(context, &parser->current)) return NULL;
13818
13819 pm_statements_t body = { 0 };
13820
13821 // At this point we know we have at least one statement, and that it
13822 // immediately follows the current token.
13823 context_push(parser, context);
13824
13825 while (true) {
13826 pm_node_t *node = parse_expression(parser, PM_BINDING_POWER_STATEMENT, PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_COMMAND_DEF | PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_CANNOT_PARSE_EXPRESSION, (uint16_t) (depth + 1));
13827 pm_statements_body_append(parser, &body, node, true);
13828
13829 // If we're recovering from a syntax error, then we need to stop parsing
13830 // the statements now.
13831 if (parser->recovering) {
13832 // If this is the level of context where the recovery has happened,
13833 // then we can mark the parser as done recovering.
13834 if (context_terminator(context, &parser->current)) parser->recovering = false;
13835 break;
13836 }
13837
13838 // If we have a terminator, then we will parse all consecutive
13839 // terminators and then continue parsing the statements list.
13840 if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) {
13841 // If we have a terminator, then we will continue parsing the
13842 // statements list.
13843 while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON));
13844 if (context_terminator(context, &parser->current)) break;
13845
13846 // Now we can continue parsing the list of statements.
13847 continue;
13848 }
13849
13850 // At this point we have a list of statements that are not terminated by
13851 // a newline or semicolon. At this point we need to check if we're at
13852 // the end of the statements list. If we are, then we should break out
13853 // of the loop.
13854 if (context_terminator(context, &parser->current)) break;
13855
13856 // At this point, we have a syntax error, because the statement was not
13857 // terminated by a newline or semicolon, and we're not at the end of the
13858 // statements list. Ideally we should scan forward to determine if we
13859 // should insert a missing terminator or break out of parsing the
13860 // statements list at this point.
13861 //
13862 // We don't have that yet, so instead we'll do a more naive approach. If
13863 // we were unable to parse an expression, then we will skip past this
13864 // token and continue parsing the statements list. Otherwise we'll add
13865 // an error and continue parsing the statements list.
13866 if (PM_NODE_TYPE_P(node, PM_ERROR_RECOVERY_NODE)) {
13867 parse_discard_token(parser, depth);
13868
13869 // If we are at the end of the file, then we need to stop parsing
13870 // the statements entirely at this point. Mark the parser as
13871 // recovering, as we know that EOF closes the top-level context, and
13872 // then break out of the loop.
13873 if (match1(parser, PM_TOKEN_EOF)) {
13874 parser->recovering = true;
13875 break;
13876 }
13877
13878 while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON));
13879 if (context_terminator(context, &parser->current)) break;
13880 } else if (!accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_EOF)) {
13881 // This is an inlined version of accept1 because the error that we
13882 // want to add has varargs. If this happens again, we should
13883 // probably extract a helper function.
13884 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->current.type));
13885 parser->previous.start = parser->previous.end;
13886 parser->previous.type = 0;
13887 }
13888 }
13889
13890 context_pop(parser);
13891
13892 bool last_value = true;
13893 switch (context) {
13894 case PM_CONTEXT_BEGIN_ENSURE:
13895 case PM_CONTEXT_DEF_ENSURE:
13896 last_value = false;
13897 break;
13898 default:
13899 break;
13900 }
13901
13902 pm_statements_node_t *statements = pm_statements_node_create(parser, &body);
13903 pm_void_statements_check(parser, statements, last_value);
13904
13905 return statements;
13906}
13907
13911static void
13912pm_hash_key_duplicated_warn(pm_parser_t *parser, const pm_node_t *duplicated, const pm_node_t *node) {
13913 pm_buffer_t buffer = { 0 };
13914 pm_static_literal_inspect(&buffer, parser, duplicated);
13915
13916 pm_diagnostic_list_append_format(
13917 &parser->metadata_arena,
13918 &parser->warning_list,
13919 duplicated->location.start,
13920 duplicated->location.length,
13921 PM_WARN_DUPLICATED_HASH_KEY,
13922 (int) pm_buffer_length(&buffer),
13923 pm_buffer_value(&buffer),
13924 pm_line_offset_list_line_column(&parser->line_offsets, PM_NODE_START(node), parser->start_line).line
13925 );
13926
13927 pm_buffer_cleanup(&buffer);
13928}
13929
13934static void
13935pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) {
13936 const pm_node_t *duplicated = pm_static_literals_add(parser, literals, node, true);
13937
13938 if (duplicated != NULL) {
13939 pm_hash_key_duplicated_warn(parser, duplicated, node);
13940 }
13941}
13942
13950static void
13951pm_hash_key_static_literals_merge(pm_parser_t *parser, pm_static_literals_t *literals, const pm_hash_node_t *hash, uint32_t boundary) {
13952 const pm_node_list_t *elements = &hash->elements;
13953
13954 for (size_t index = 0; index < elements->size; index++) {
13955 pm_node_t *element = elements->nodes[index];
13956
13957 switch (PM_NODE_TYPE(element)) {
13958 case PM_ASSOC_NODE: {
13959 pm_node_t *key = ((pm_assoc_node_t *) element)->key;
13960 const pm_node_t *duplicated = pm_static_literals_add(parser, literals, key, true);
13961
13962 if (duplicated != NULL && PM_NODE_START(duplicated) < boundary) {
13963 pm_hash_key_duplicated_warn(parser, duplicated, key);
13964 }
13965
13966 break;
13967 }
13968 case PM_ASSOC_SPLAT_NODE: {
13969 const pm_node_t *value = ((pm_assoc_splat_node_t *) element)->value;
13970
13971 if (value != NULL && PM_NODE_TYPE_P(value, PM_HASH_NODE)) {
13972 pm_hash_key_static_literals_merge(parser, literals, (const pm_hash_node_t *) value, boundary);
13973 }
13974
13975 break;
13976 }
13977 default:
13978 break;
13979 }
13980 }
13981}
13982
13987static void
13988pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) {
13989 pm_node_t *previous;
13990
13991 if ((previous = pm_static_literals_add(parser, literals, node, false)) != NULL) {
13992 pm_diagnostic_list_append_format(
13993 &parser->metadata_arena,
13994 &parser->warning_list,
13995 PM_NODE_START(node),
13996 PM_NODE_LENGTH(node),
13997 PM_WARN_DUPLICATED_WHEN_CLAUSE,
13998 pm_line_offset_list_line_column(&parser->line_offsets, PM_NODE_START(node), parser->start_line).line,
13999 pm_line_offset_list_line_column(&parser->line_offsets, PM_NODE_START(previous), parser->start_line).line
14000 );
14001 }
14002}
14003
14008static bool
14009parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_hash_elements_t *elements, uint16_t depth) {
14010 bool contains_keyword_splat = false;
14011
14012 while (true) {
14013 pm_node_t *element;
14014
14015 switch (parser->current.type) {
14016 case PM_TOKEN_USTAR_STAR: {
14017 parser_lex(parser);
14018 pm_token_t operator = parser->previous;
14019 pm_node_t *value = NULL;
14020
14021 if (token_begins_expression_p(parser->current.type)) {
14022 value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH, (uint16_t) (depth + 1));
14023
14024 /* If the splatted value is itself a hash literal, its keys
14025 * become part of this hash for the duplicate key warning. */
14026 if (value != NULL && PM_NODE_TYPE_P(value, PM_HASH_NODE)) {
14027 pm_hash_key_static_literals_merge(parser, literals, (const pm_hash_node_t *) value, PM_NODE_START(value));
14028 }
14029 } else {
14030 pm_parser_scope_forwarding_keywords_check(parser, &operator);
14031 }
14032
14033 element = UP(pm_assoc_splat_node_create(parser, value, &operator));
14034 contains_keyword_splat = true;
14035 break;
14036 }
14037 case PM_TOKEN_LABEL: {
14038 pm_token_t label = parser->current;
14039 parser_lex(parser);
14040
14041 pm_node_t *key = UP(pm_symbol_node_label_create(parser, &label));
14042 pm_hash_key_static_literals_add(parser, literals, key);
14043
14044 pm_node_t *value = NULL;
14045
14046 if (token_begins_expression_p(parser->current.type)) {
14047 value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_HASH_EXPRESSION_AFTER_LABEL, (uint16_t) (depth + 1));
14048 } else {
14049 pm_token_t identifier = { .type = PM_TOKEN_IDENTIFIER, .start = label.start, .end = label.end - 1 };
14050 bool constant = pm_encoding_isupper_char(parser->encoding, identifier.start, identifier.end - identifier.start, parser->version);
14051 bool invalid = identifier.end[-1] == '!' || identifier.end[-1] == '?';
14052
14053 if (invalid) {
14054 PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &identifier, constant ? PM_ERR_INVALID_CONSTANT_READ : PM_ERR_INVALID_LOCAL_VARIABLE_READ);
14055 }
14056
14057 if (constant) {
14058 identifier.type = PM_TOKEN_CONSTANT;
14059 value = UP(pm_constant_read_node_create(parser, &identifier));
14060 } else {
14061 int depth = invalid ? -1 : pm_parser_local_depth(parser, &identifier);
14062 if (depth == -1) {
14063 value = UP(pm_call_node_variable_call_create(parser, &identifier));
14064 } else {
14065 value = UP(pm_local_variable_read_node_create(parser, &identifier, (uint32_t) depth));
14066 }
14067 }
14068
14069 value->location.length++;
14070 value = UP(pm_implicit_node_create(parser, value));
14071 }
14072
14073 element = UP(pm_assoc_node_create(parser, key, NULL, value));
14074 break;
14075 }
14076 default: {
14077 pm_node_t *key = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, PM_PARSE_ACCEPTS_DO_BLOCK | PM_PARSE_ACCEPTS_LABEL, PM_ERR_HASH_KEY, (uint16_t) (depth + 1));
14078
14079 // Hash keys that are strings are automatically frozen. We will
14080 // mark that here.
14081 if (PM_NODE_TYPE_P(key, PM_STRING_NODE)) {
14082 pm_node_flag_set(key, PM_STRING_FLAGS_FROZEN | PM_NODE_FLAG_STATIC_LITERAL);
14083 }
14084
14085 pm_hash_key_static_literals_add(parser, literals, key);
14086
14087 pm_token_t operator = { 0 };
14088 if (!pm_symbol_node_label_p(parser, key)) {
14089 expect1(parser, PM_TOKEN_EQUAL_GREATER, PM_ERR_HASH_ROCKET);
14090 operator = parser->previous;
14091 }
14092
14093 pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1));
14094 element = UP(pm_assoc_node_create(parser, key, NTOK2PTR(operator), value));
14095 break;
14096 }
14097 }
14098
14099 pm_hash_elements_append(parser, elements, element);
14100
14101 // If there's no comma after the element, then we're done.
14102 if (!accept1(parser, PM_TOKEN_COMMA)) break;
14103
14104 // If the next element starts with a label or a **, then we know we have
14105 // another element in the hash, so we'll continue parsing.
14106 if (match2(parser, PM_TOKEN_USTAR_STAR, PM_TOKEN_LABEL)) continue;
14107
14108 // Otherwise we need to check if the subsequent token begins an expression.
14109 // If it does, then we'll continue parsing.
14110 if (token_begins_expression_p(parser->current.type)) continue;
14111
14112 // Otherwise by default we will exit out of this loop.
14113 break;
14114 }
14115
14116 return contains_keyword_splat;
14117}
14118
14119static PRISM_INLINE bool
14120argument_allowed_for_bare_hash(pm_parser_t *parser, pm_node_t *argument) {
14121 if (pm_symbol_node_label_p(parser, argument)) {
14122 return true;
14123 }
14124
14125 switch (PM_NODE_TYPE(argument)) {
14126 case PM_CALL_NODE: {
14127 pm_call_node_t *cast = (pm_call_node_t *) argument;
14128 if (cast->opening_loc.length == 0 && cast->arguments != NULL) {
14129 if (PM_NODE_FLAG_P(cast->arguments, PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS | PM_ARGUMENTS_NODE_FLAGS_CONTAINS_SPLAT)) {
14130 return false;
14131 }
14132 if (cast->block != NULL) {
14133 return false;
14134 }
14135 }
14136 break;
14137 }
14138 default: break;
14139 }
14140 return accept1(parser, PM_TOKEN_EQUAL_GREATER);
14141}
14142
14146static PRISM_INLINE void
14147parse_arguments_append(pm_parser_t *parser, pm_arguments_t *arguments, pm_node_t *argument) {
14148 if (arguments->list.size == 0) {
14149 arguments->list_loc.start = PM_NODE_START(argument);
14150 }
14151
14152 if (PM_LOCATION_END(&arguments->list_loc) < PM_NODE_END(argument)) {
14153 arguments->list_loc.length = PM_NODE_END(argument) - arguments->list_loc.start;
14154 }
14155
14156 pm_arguments_flags_update(&arguments->list_flags, argument);
14157 pm_node_list_open_append(&parser->node_chunks, &arguments->list, argument);
14158}
14159
14164static PRISM_INLINE bool
14165pm_call_node_command_p(const pm_call_node_t *node) {
14166 return (
14167 (node->opening_loc.length == 0) &&
14168 (node->block == NULL || PM_NODE_TYPE_P(node->block, PM_BLOCK_ARGUMENT_NODE)) &&
14169 (node->arguments != NULL || node->block != NULL)
14170 );
14171}
14172
14182static bool
14183pm_constant_path_command_call_p(const pm_parser_t *parser, const pm_call_node_t *call) {
14184 return (
14185 call->receiver != NULL &&
14186 call->opening_loc.length == 0 &&
14187 call->block != NULL && PM_NODE_TYPE_P(call->block, PM_BLOCK_NODE) &&
14188 call->call_operator_loc.length > 0 &&
14189 parser->start[call->call_operator_loc.start] == ':' &&
14190 call->message_loc.length > 0 &&
14191 pm_encoding_isupper_char(parser->encoding, parser->start + call->message_loc.start, (ptrdiff_t) call->message_loc.length, parser->version)
14192 );
14193}
14194
14200static bool
14201pm_command_call_value_p(const pm_parser_t *parser, const pm_node_t *node) {
14202 switch (PM_NODE_TYPE(node)) {
14203 case PM_CALL_NODE: {
14204 const pm_call_node_t *call = (const pm_call_node_t *) node;
14205
14206 /* Command-style calls (e.g., foo bar, obj.foo bar). Attribute
14207 * writes (e.g., a.b = 1) are not commands. */
14208 if (pm_call_node_command_p(call) && !PM_NODE_FLAG_P(node, PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE) && (call->receiver == NULL || call->call_operator_loc.length > 0)) {
14209 return true;
14210 }
14211
14212 /* A constant-path command with a brace block, e.g. `Foo::Bar { }`. */
14213 if (pm_constant_path_command_call_p(parser, call)) {
14214 return true;
14215 }
14216
14217 /* A `!` or `not` prefix wrapping a command call (e.g., `!foo bar`,
14218 * `not foo bar`) is also a command-call value. */
14219 if (call->receiver != NULL && call->arguments == NULL && call->opening_loc.length == 0 && call->call_operator_loc.length == 0) {
14220 return pm_command_call_value_p(parser, call->receiver);
14221 }
14222
14223 return false;
14224 }
14225 case PM_SUPER_NODE: {
14226 /* A command-style super (no parens). A super carrying a do-block is
14227 * a block call (it permits chaining), so it is excluded here and
14228 * handled by pm_block_call_p instead. */
14229 const pm_super_node_t *cast = (const pm_super_node_t *) node;
14230 return cast->lparen_loc.length == 0 &&
14231 (cast->arguments != NULL || cast->block != NULL) &&
14232 !(cast->block != NULL && PM_NODE_TYPE_P(cast->block, PM_BLOCK_NODE));
14233 }
14234 case PM_YIELD_NODE: {
14235 const pm_yield_node_t *cast = (const pm_yield_node_t *) node;
14236 return cast->lparen_loc.length == 0 && cast->arguments != NULL;
14237 }
14238 case PM_RESCUE_MODIFIER_NODE:
14239 return pm_command_call_value_p(parser, ((const pm_rescue_modifier_node_t *) node)->expression);
14240 case PM_DEF_NODE: {
14241 const pm_def_node_t *cast = (const pm_def_node_t *) node;
14242 if (cast->equal_loc.length > 0 && cast->body != NULL) {
14243 const pm_node_t *body = cast->body;
14244 if (PM_NODE_TYPE_P(body, PM_STATEMENTS_NODE)) {
14245 body = ((const pm_statements_node_t *) body)->body.nodes[((const pm_statements_node_t *) body)->body.size - 1];
14246 }
14247 return pm_command_call_value_p(parser, body);
14248 }
14249 return false;
14250 }
14251 default:
14252 return false;
14253 }
14254}
14255
14265static bool
14266pm_command_before_comma_p(const pm_parser_t *parser, const pm_node_t *node) {
14267 switch (PM_NODE_TYPE(node)) {
14268 case PM_CALL_NODE:
14269 return pm_constant_path_command_call_p(parser, (const pm_call_node_t *) node);
14270 case PM_DEF_NODE:
14271 return pm_command_call_value_p(parser, node);
14272 default:
14273 return false;
14274 }
14275}
14276
14283static bool
14284pm_block_call_p(const pm_node_t *node) {
14285 while (PM_NODE_TYPE_P(node, PM_CALL_NODE)) {
14286 const pm_call_node_t *call = (const pm_call_node_t *) node;
14287
14288 /* Root: a command (no parentheses) carrying command arguments and a
14289 * block (brace or do), e.g. `foo bar do end`, `foo bar { }`. The
14290 * no-parentheses requirement is what distinguishes a command root from
14291 * a method call root like `foo.bar(1) { }`, which is a primary value
14292 * and may be used as an argument.
14293 */
14294 if (call->opening_loc.length == 0 && call->arguments != NULL && call->block != NULL && PM_NODE_TYPE_P(call->block, PM_BLOCK_NODE)) {
14295 return true;
14296 }
14297
14298 /* Walk up the receiver chain of a `.`/`::`/`&.` call (e.g.,
14299 * `foo bar do end.baz(1)`). Parentheses on the chained call are allowed
14300 * here -- in parse.y a `block_call` can be extended by
14301 * `call_op2 operation2 opt_paren_args` and remains a block call.
14302 */
14303 if (call->call_operator_loc.length > 0 && call->receiver != NULL) {
14304 node = call->receiver;
14305 continue;
14306 }
14307
14308 return false;
14309 }
14310
14311 /* A `super` with command arguments and a do-block is also a block-call root
14312 * (parse.y: `command do_block`, where the command is `keyword_super
14313 * command_args`). `super do end` with no arguments is a forwarding super
14314 * (a primary value) and is handled elsewhere.
14315 */
14316 if (PM_NODE_TYPE_P(node, PM_SUPER_NODE)) {
14317 const pm_super_node_t *super = (const pm_super_node_t *) node;
14318 return super->lparen_loc.length == 0 && super->block != NULL && PM_NODE_TYPE_P(super->block, PM_BLOCK_NODE);
14319 }
14320
14321 return false;
14322}
14323
14327static void
14328parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_forwarding, pm_token_type_t terminator, uint8_t flags, uint16_t depth) {
14329 pm_binding_power_t binding_power = pm_binding_powers[parser->current.type].left;
14330
14331 // First we need to check if the next token is one that could be the start
14332 // of an argument. If it's not, then we can just return.
14333 if (
14334 match2(parser, terminator, PM_TOKEN_EOF) ||
14335 (binding_power != PM_BINDING_POWER_UNSET && binding_power < PM_BINDING_POWER_RANGE) ||
14336 context_terminator(parser->current_context->context, &parser->current)
14337 ) {
14338 return;
14339 }
14340
14341 bool parsed_first_argument = false;
14342 bool parsed_bare_hash = false;
14343 bool parsed_block_argument = false;
14344 bool parsed_forwarding_arguments = false;
14345
14346 while (!match1(parser, PM_TOKEN_EOF)) {
14347 if (parsed_forwarding_arguments) {
14348 pm_parser_err_current(parser, PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES);
14349 }
14350
14351 pm_node_t *argument = NULL;
14352
14353 switch (parser->current.type) {
14354 case PM_TOKEN_USTAR_STAR:
14355 case PM_TOKEN_LABEL: {
14356 if (parsed_bare_hash) {
14357 pm_parser_err_current(parser, PM_ERR_ARGUMENT_BARE_HASH);
14358 }
14359
14360 pm_hash_elements_t elements = { 0 };
14361 pm_static_literals_t hash_keys = { 0 };
14362 bool contains_keyword_splat = parse_assocs(parser, &hash_keys, &elements, (uint16_t) (depth + 1));
14363 argument = UP(pm_keyword_hash_node_create(parser, &elements));
14364
14365 parse_arguments_append(parser, arguments, argument);
14366
14367 pm_node_flags_t node_flags = PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS;
14368 if (contains_keyword_splat) node_flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT;
14369 arguments->list_flags |= node_flags;
14370
14371 pm_static_literals_free(&hash_keys);
14372 parsed_bare_hash = true;
14373
14374 break;
14375 }
14376 case PM_TOKEN_UAMPERSAND: {
14377 parser_lex(parser);
14378 pm_token_t operator = parser->previous;
14379 pm_node_t *expression = NULL;
14380
14381 if (token_begins_expression_p(parser->current.type)) {
14382 expression = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_ARGUMENT, (uint16_t) (depth + 1));
14383 } else {
14384 pm_parser_scope_forwarding_block_check(parser, &operator);
14385 }
14386
14387 argument = UP(pm_block_argument_node_create(parser, &operator, expression));
14388 if (parsed_block_argument) {
14389 parse_arguments_append(parser, arguments, argument);
14390 } else {
14391 arguments->block = argument;
14392 }
14393
14394 if (match1(parser, PM_TOKEN_COMMA)) {
14395 pm_parser_err_current(parser, PM_ERR_ARGUMENT_AFTER_BLOCK);
14396 }
14397
14398 parsed_block_argument = true;
14399 break;
14400 }
14401 case PM_TOKEN_USTAR: {
14402 parser_lex(parser);
14403 pm_token_t operator = parser->previous;
14404
14405 if (match4(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_COMMA, PM_TOKEN_SEMICOLON, PM_TOKEN_BRACKET_RIGHT)) {
14406 pm_parser_scope_forwarding_positionals_check(parser, &operator);
14407 argument = UP(pm_splat_node_create(parser, &operator, NULL));
14408 if (parsed_bare_hash) {
14409 pm_parser_err_previous(parser, PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT);
14410 }
14411 } else {
14412 pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT, (uint16_t) (depth + 1));
14413
14414 if (parsed_bare_hash) {
14415 pm_parser_err(parser, PM_TOKEN_START(parser, &operator), PM_NODE_END(expression) - PM_TOKEN_START(parser, &operator), PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT);
14416 }
14417
14418 argument = UP(pm_splat_node_create(parser, &operator, expression));
14419 }
14420
14421 parse_arguments_append(parser, arguments, argument);
14422 break;
14423 }
14424 case PM_TOKEN_UDOT_DOT_DOT: {
14425 if (accepts_forwarding) {
14426 parser_lex(parser);
14427
14428 if (token_begins_expression_p(parser->current.type)) {
14429 // If the token begins an expression then this ... was
14430 // not actually argument forwarding but was instead a
14431 // range.
14432 pm_token_t operator = parser->previous;
14433 pm_node_t *right = parse_expression(parser, PM_BINDING_POWER_RANGE, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
14434
14435 /* A binary or endless range chains with the outer
14436 * operator. A beginless range is a valid operand. */
14437 if (PM_NODE_TYPE_P(right, PM_RANGE_NODE)) {
14438 pm_range_node_t *range = (pm_range_node_t *) right;
14439 if (range->left != NULL) {
14440 pm_parser_err(parser, range->operator_loc.start, range->operator_loc.length, PM_ERR_UNEXPECTED_RANGE_OPERATOR);
14441 }
14442 }
14443
14444 argument = UP(pm_range_node_create(parser, NULL, &operator, right));
14445 } else {
14446 pm_parser_scope_forwarding_all_check(parser, &parser->previous);
14447 if (parsed_first_argument && terminator == PM_TOKEN_EOF) {
14448 pm_parser_err_previous(parser, PM_ERR_ARGUMENT_FORWARDING_UNBOUND);
14449 }
14450
14451 argument = UP(pm_forwarding_arguments_node_create(parser, &parser->previous));
14452 parse_arguments_append(parser, arguments, argument);
14453 arguments->list_flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_FORWARDING;
14454 arguments->has_forwarding = true;
14455 parsed_forwarding_arguments = true;
14456 break;
14457 }
14458 }
14459 }
14461 default: {
14462 if (argument == NULL) {
14463 /* The first argument may be a command call, and since Ruby
14464 * 4.0 also an endless method with a command body
14465 * (`private def f = foo 1`), either of which must then be
14466 * the sole argument. */
14467 uint8_t command_flags = 0;
14468 if (!parsed_first_argument) {
14469 command_flags = PM_PARSE_ACCEPTS_COMMAND_CALL;
14470 if (parser->version >= PM_OPTIONS_VERSION_CRUBY_4_0) command_flags |= PM_PARSE_ACCEPTS_COMMAND_DEF;
14471 }
14472
14473 argument = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | command_flags | PM_PARSE_ACCEPTS_LABEL), PM_ERR_EXPECT_ARGUMENT, (uint16_t) (depth + 1));
14474 }
14475
14476 bool contains_keywords = false;
14477 bool contains_keyword_splat = false;
14478
14479 if (argument_allowed_for_bare_hash(parser, argument)) {
14480 if (parsed_bare_hash) {
14481 pm_parser_err_previous(parser, PM_ERR_ARGUMENT_BARE_HASH);
14482 }
14483
14484 /* A hash key must be an argument (`arg`). A command call or
14485 * block call (e.g. `Foo::Bar { } => v`, `foo bar do end =>
14486 * v`) is not an argument, so reject it as a key. Plain
14487 * command calls never reach here as a key because they
14488 * absorb the `=>` into their own arguments first.
14489 */
14490 if (pm_command_call_value_p(parser, argument) || pm_block_call_p(argument)) {
14491 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->previous.type));
14492 }
14493
14494 pm_token_t operator = { 0 };
14495 if (parser->previous.type == PM_TOKEN_EQUAL_GREATER) {
14496 operator = parser->previous;
14497 }
14498
14499 pm_hash_elements_t elements = { 0 };
14500 contains_keywords = true;
14501
14502 // Create the set of static literals for this hash.
14503 pm_static_literals_t hash_keys = { 0 };
14504 pm_hash_key_static_literals_add(parser, &hash_keys, argument);
14505
14506 // Finish parsing the one we are part way through.
14507 pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1));
14508 argument = UP(pm_assoc_node_create(parser, argument, NTOK2PTR(operator), value));
14509
14510 pm_hash_elements_append(parser, &elements, argument);
14511
14512 // Then parse more if we have a comma
14513 if (accept1(parser, PM_TOKEN_COMMA) && (
14514 token_begins_expression_p(parser->current.type) ||
14515 match2(parser, PM_TOKEN_USTAR_STAR, PM_TOKEN_LABEL)
14516 )) {
14517 contains_keyword_splat = parse_assocs(parser, &hash_keys, &elements, (uint16_t) (depth + 1));
14518 }
14519
14520 argument = UP(pm_keyword_hash_node_create(parser, &elements));
14521
14522 pm_static_literals_free(&hash_keys);
14523 parsed_bare_hash = true;
14524 }
14525
14526 parse_arguments_append(parser, arguments, argument);
14527
14528 pm_node_flags_t node_flags = 0;
14529 if (contains_keywords) node_flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS;
14530 if (contains_keyword_splat) node_flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT;
14531 arguments->list_flags |= node_flags;
14532
14533 break;
14534 }
14535 }
14536
14537 parsed_first_argument = true;
14538
14539 // If parsing the argument failed, we need to stop parsing arguments.
14540 if (PM_NODE_TYPE_P(argument, PM_ERROR_RECOVERY_NODE) || parser->recovering) break;
14541
14542 // If the terminator of these arguments is not EOF, then we have a
14543 // specific token we're looking for. In that case we can accept a
14544 // newline here because it is not functioning as a statement terminator.
14545 bool accepted_newline = false;
14546 if (terminator != PM_TOKEN_EOF) {
14547 accepted_newline = accept1(parser, PM_TOKEN_NEWLINE);
14548 }
14549
14550 if (parser->previous.type == PM_TOKEN_COMMA && parsed_bare_hash) {
14551 // If we previously were on a comma and we just parsed a bare hash,
14552 // then we want to continue parsing arguments. This is because the
14553 // comma was grabbed up by the hash parser.
14554 } else if (accept1(parser, PM_TOKEN_COMMA)) {
14555 // If there was a comma, then we need to check if we also accepted a
14556 // newline. If we did, then this is a syntax error.
14557 if (accepted_newline) {
14558 pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
14559 }
14560
14561 // If this is a command call and an argument takes a block,
14562 // there can be no further arguments. For example,
14563 // `foo(bar 1 do end, 2)` should be rejected.
14564 if (PM_NODE_TYPE_P(argument, PM_CALL_NODE)) {
14565 pm_call_node_t *call = (pm_call_node_t *) argument;
14566 if (call->opening_loc.length == 0 && call->arguments != NULL && call->block != NULL) {
14567 pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
14568 break;
14569 }
14570 }
14571
14572 /* A command can only be the sole argument, as in
14573 * `foo Foo::Bar { }, y` or `foo def f = bar 1 rescue z, y`. */
14574 if (pm_command_before_comma_p(parser, argument)) {
14575 pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
14576 }
14577 } else {
14578 // If there is no comma at the end of the argument list then we're
14579 // done parsing arguments and can break out of this loop.
14580 break;
14581 }
14582
14583 // If we hit the terminator, then that means we have a trailing comma so
14584 // we can accept that output as well.
14585 if (match1(parser, terminator)) {
14586 // A forwarding `...` argument must be the last argument and cannot
14587 // be followed by a trailing comma, e.g. `foo(...,)`. A comma
14588 // followed by another argument is already rejected at the top of
14589 // this loop, so the only case left to reject here is the trailing
14590 // one.
14591 if (parsed_forwarding_arguments) {
14592 pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
14593 }
14594
14595 break;
14596 }
14597 }
14598}
14599
14610static pm_node_t *
14611parse_required_destructured_parameter(pm_parser_t *parser, uint16_t depth) {
14612 if (PRISM_UNLIKELY(depth >= PRISM_DEPTH_MAXIMUM || pm_parser_stack_exhausted(parser))) {
14613 return parse_resource_recovery(parser, depth);
14614 }
14615
14616 expect1(parser, PM_TOKEN_PARENTHESIS_LEFT_GROUPING, PM_ERR_EXPECT_LPAREN_REQ_PARAMETER);
14617
14618 pm_token_t lparen = parser->previous;
14619 pm_multi_targets_t targets = { 0 };
14620
14621 do {
14622 pm_node_t *param;
14623
14624 // If we get here then we have a trailing comma, which isn't allowed in
14625 // the grammar. In other places, multi targets _do_ allow trailing
14626 // commas, so here we'll assume this is a mistake of the user not
14627 // knowing it's not allowed here.
14628 if ((targets.rest == NULL ? targets.nodes.size : targets.rest_index) > 0 && match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
14629 param = UP(pm_implicit_rest_node_create(parser, &parser->previous));
14630 pm_multi_targets_append(parser, &targets, param);
14631 pm_parser_err_current(parser, PM_ERR_PARAMETER_WILD_LOOSE_COMMA);
14632 break;
14633 }
14634
14635 if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) {
14636 param = parse_required_destructured_parameter(parser, (uint16_t) (depth + 1));
14637 } else if (accept1(parser, PM_TOKEN_USTAR)) {
14638 pm_token_t star = parser->previous;
14639 pm_node_t *value = NULL;
14640
14641 if (accept1(parser, PM_TOKEN_IDENTIFIER)) {
14642 pm_token_t name = parser->previous;
14643 value = UP(pm_required_parameter_node_create(parser, &name));
14644 if (pm_parser_parameter_name_check(parser, &name)) {
14645 pm_node_flag_set_repeated_parameter(value);
14646 }
14647 pm_parser_local_add_token(parser, &name, 1);
14648 }
14649
14650 param = UP(pm_splat_node_create(parser, &star, value));
14651 } else {
14652 expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_EXPECT_IDENT_REQ_PARAMETER);
14653 pm_token_t name = parser->previous;
14654
14655 param = UP(pm_required_parameter_node_create(parser, &name));
14656 if (pm_parser_parameter_name_check(parser, &name)) {
14657 pm_node_flag_set_repeated_parameter(param);
14658 }
14659 pm_parser_local_add_token(parser, &name, 1);
14660 }
14661
14662 pm_multi_targets_append(parser, &targets, param);
14663 } while (accept1(parser, PM_TOKEN_COMMA));
14664
14665 accept1(parser, PM_TOKEN_NEWLINE);
14666 expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN_REQ_PARAMETER);
14667
14668 return UP(pm_multi_target_node_create(parser, &targets, &lparen, &parser->previous));
14669}
14670
14675typedef enum {
14676 PM_PARAMETERS_NO_CHANGE = 0, // Extra state for tokens that should not change the state
14677 PM_PARAMETERS_ORDER_NOTHING_AFTER = 1,
14678 PM_PARAMETERS_ORDER_KEYWORDS_REST,
14679 PM_PARAMETERS_ORDER_KEYWORDS,
14680 PM_PARAMETERS_ORDER_REST,
14681 PM_PARAMETERS_ORDER_AFTER_OPTIONAL,
14682 PM_PARAMETERS_ORDER_OPTIONAL,
14683 PM_PARAMETERS_ORDER_NAMED,
14684 PM_PARAMETERS_ORDER_NONE,
14685} pm_parameters_order_t;
14686
14690static pm_parameters_order_t parameters_ordering[PM_TOKEN_MAXIMUM] = {
14691 [0] = PM_PARAMETERS_NO_CHANGE,
14692 [PM_TOKEN_UAMPERSAND] = PM_PARAMETERS_ORDER_NOTHING_AFTER,
14693 [PM_TOKEN_AMPERSAND] = PM_PARAMETERS_ORDER_NOTHING_AFTER,
14694 [PM_TOKEN_UDOT_DOT_DOT] = PM_PARAMETERS_ORDER_NOTHING_AFTER,
14695 [PM_TOKEN_IDENTIFIER] = PM_PARAMETERS_ORDER_NAMED,
14696 [PM_TOKEN_PARENTHESIS_LEFT_GROUPING] = PM_PARAMETERS_ORDER_NAMED,
14697 [PM_TOKEN_EQUAL] = PM_PARAMETERS_ORDER_OPTIONAL,
14698 [PM_TOKEN_LABEL] = PM_PARAMETERS_ORDER_KEYWORDS,
14699 [PM_TOKEN_USTAR] = PM_PARAMETERS_ORDER_AFTER_OPTIONAL,
14700 [PM_TOKEN_STAR] = PM_PARAMETERS_ORDER_AFTER_OPTIONAL,
14701 [PM_TOKEN_USTAR_STAR] = PM_PARAMETERS_ORDER_KEYWORDS_REST,
14702 [PM_TOKEN_STAR_STAR] = PM_PARAMETERS_ORDER_KEYWORDS_REST
14703};
14704
14712static bool
14713update_parameter_state(pm_parser_t *parser, pm_token_t *token, pm_parameters_order_t *current) {
14714 pm_parameters_order_t state = parameters_ordering[token->type];
14715 if (state == PM_PARAMETERS_NO_CHANGE) return true;
14716
14717 // If we see another ordered argument after a optional argument
14718 // we only continue parsing ordered arguments until we stop seeing ordered arguments.
14719 if (*current == PM_PARAMETERS_ORDER_OPTIONAL && state == PM_PARAMETERS_ORDER_NAMED) {
14720 *current = PM_PARAMETERS_ORDER_AFTER_OPTIONAL;
14721 return true;
14722 } else if (*current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL && state == PM_PARAMETERS_ORDER_NAMED) {
14723 return true;
14724 }
14725
14726 if (token->type == PM_TOKEN_USTAR && *current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL) {
14727 pm_parser_err_token(parser, token, PM_ERR_PARAMETER_STAR);
14728 return false;
14729 } else if (token->type == PM_TOKEN_UDOT_DOT_DOT && (*current >= PM_PARAMETERS_ORDER_KEYWORDS_REST && *current <= PM_PARAMETERS_ORDER_AFTER_OPTIONAL)) {
14730 pm_parser_err_token(parser, token, *current == PM_PARAMETERS_ORDER_AFTER_OPTIONAL ? PM_ERR_PARAMETER_FORWARDING_AFTER_REST : PM_ERR_PARAMETER_ORDER);
14731 return false;
14732 } else if (*current == PM_PARAMETERS_ORDER_NOTHING_AFTER || state > *current) {
14733 // We know what transition we failed on, so we can provide a better error here.
14734 pm_parser_err_token(parser, token, PM_ERR_PARAMETER_ORDER);
14735 return false;
14736 }
14737
14738 if (state < *current) *current = state;
14739 return true;
14740}
14741
14742static PRISM_INLINE void
14743parse_parameters_handle_trailing_comma(
14744 pm_parser_t *parser,
14745 pm_parameters_t *params,
14746 pm_parameters_order_t order,
14747 bool in_block,
14748 bool allows_trailing_comma
14749) {
14750 if (!allows_trailing_comma) {
14751 pm_parser_err_previous(parser, PM_ERR_PARAMETER_WILD_LOOSE_COMMA);
14752 return;
14753 }
14754
14755 if (in_block) {
14756 if (order >= PM_PARAMETERS_ORDER_NAMED) {
14757 // foo do |bar,|; end
14758 pm_node_t *param = UP(pm_implicit_rest_node_create(parser, &parser->previous));
14759
14760 if (params->rest == NULL) {
14761 pm_parameters_rest_set(params, param);
14762 } else {
14763 pm_parser_err_node(parser, UP(param), PM_ERR_PARAMETER_SPLAT_MULTI);
14764 pm_parameters_posts_append(parser, params, UP(param));
14765 }
14766 } else {
14767 // foo do |*bar,|; end
14768 pm_parser_err_previous(parser, PM_ERR_PARAMETER_WILD_LOOSE_COMMA);
14769 }
14770 } else {
14771 // https://bugs.ruby-lang.org/issues/19107
14772 // Allow `def foo(bar,); end`, `def foo(*bar,); end`, etc. but not `def foo(...,); end`
14773 if (parser->version < PM_OPTIONS_VERSION_CRUBY_4_1 || order == PM_PARAMETERS_ORDER_NOTHING_AFTER) {
14774 pm_parser_err_previous(parser, PM_ERR_PARAMETER_WILD_LOOSE_COMMA);
14775 }
14776 }
14777}
14778
14782static pm_parameters_node_t *
14783parse_parameters(
14784 pm_parser_t *parser,
14785 pm_binding_power_t binding_power,
14786 bool uses_parentheses,
14787 bool allows_trailing_comma,
14788 bool allows_forwarding_parameters,
14789 bool accepts_blocks_in_defaults,
14790 bool in_block,
14791 pm_diagnostic_id_t diag_id_forwarding,
14792 uint16_t depth
14793) {
14794 pm_do_loop_stack_push(parser, false);
14795
14796 pm_parameters_t params = { 0 };
14797 pm_parameters_order_t order = PM_PARAMETERS_ORDER_NONE;
14798
14799 while (true) {
14800 bool parsing = true;
14801
14802 switch (parser->current.type) {
14803 case PM_TOKEN_PARENTHESIS_LEFT_GROUPING: {
14804 update_parameter_state(parser, &parser->current, &order);
14805 pm_node_t *param = parse_required_destructured_parameter(parser, (uint16_t) (depth + 1));
14806
14807 if (order > PM_PARAMETERS_ORDER_AFTER_OPTIONAL) {
14808 pm_parameters_requireds_append(parser, &params, param);
14809 } else {
14810 pm_parameters_posts_append(parser, &params, param);
14811 }
14812 break;
14813 }
14814 case PM_TOKEN_UAMPERSAND:
14815 case PM_TOKEN_AMPERSAND: {
14816 update_parameter_state(parser, &parser->current, &order);
14817 parser_lex(parser);
14818
14819 pm_token_t operator = parser->previous;
14820 pm_node_t *param;
14821
14822 if (parser->version >= PM_OPTIONS_VERSION_CRUBY_4_1 && accept1(parser, PM_TOKEN_KEYWORD_NIL)) {
14823 param = (pm_node_t *) pm_no_block_parameter_node_create(parser, &operator, &parser->previous);
14824 } else {
14825 pm_token_t name = {0};
14826
14827 bool repeated = false;
14828 if (accept1(parser, PM_TOKEN_IDENTIFIER)) {
14829 name = parser->previous;
14830 repeated = pm_parser_parameter_name_check(parser, &name);
14831 pm_parser_local_add_token(parser, &name, 1);
14832 } else {
14833 parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_BLOCK;
14834 }
14835
14836 param = (pm_node_t *) pm_block_parameter_node_create(parser, NTOK2PTR(name), &operator);
14837 if (repeated) {
14838 pm_node_flag_set_repeated_parameter(param);
14839 }
14840 }
14841
14842 if (params.block == NULL) {
14843 pm_parameters_block_set(&params, param);
14844 } else {
14845 pm_parser_err_node(parser, param, PM_ERR_PARAMETER_BLOCK_MULTI);
14846 pm_parameters_posts_append(parser, &params, UP(pm_error_recovery_node_create_unexpected(parser, param)));
14847 }
14848
14849 break;
14850 }
14851 case PM_TOKEN_UDOT_DOT_DOT: {
14852 if (!allows_forwarding_parameters) {
14853 pm_parser_err_current(parser, diag_id_forwarding);
14854 }
14855
14856 bool succeeded = update_parameter_state(parser, &parser->current, &order);
14857 parser_lex(parser);
14858
14859 parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_ALL;
14860 pm_forwarding_parameter_node_t *param = pm_forwarding_parameter_node_create(parser, &parser->previous);
14861
14862 if (params.keyword_rest != NULL) {
14863 // If we already have a keyword rest parameter, then we replace it with the
14864 // forwarding parameter and move the keyword rest parameter to the posts list.
14865 pm_node_t *keyword_rest = params.keyword_rest;
14866 pm_parameters_posts_append(parser, &params, UP(pm_error_recovery_node_create_unexpected(parser, keyword_rest)));
14867 if (succeeded) pm_parser_err_previous(parser, PM_ERR_PARAMETER_UNEXPECTED_FWD);
14868 params.keyword_rest = NULL;
14869 }
14870
14871 pm_parameters_keyword_rest_set(&params, UP(param));
14872 break;
14873 }
14874 case PM_TOKEN_CLASS_VARIABLE:
14875 case PM_TOKEN_IDENTIFIER:
14876 case PM_TOKEN_CONSTANT:
14877 case PM_TOKEN_INSTANCE_VARIABLE:
14878 case PM_TOKEN_GLOBAL_VARIABLE:
14879 case PM_TOKEN_METHOD_NAME: {
14880 parser_lex(parser);
14881 switch (parser->previous.type) {
14882 case PM_TOKEN_CONSTANT:
14883 pm_parser_err_previous(parser, PM_ERR_ARGUMENT_FORMAL_CONSTANT);
14884 break;
14885 case PM_TOKEN_INSTANCE_VARIABLE:
14886 pm_parser_err_previous(parser, PM_ERR_ARGUMENT_FORMAL_IVAR);
14887 break;
14888 case PM_TOKEN_GLOBAL_VARIABLE:
14889 pm_parser_err_previous(parser, PM_ERR_ARGUMENT_FORMAL_GLOBAL);
14890 break;
14891 case PM_TOKEN_CLASS_VARIABLE:
14892 pm_parser_err_previous(parser, PM_ERR_ARGUMENT_FORMAL_CLASS);
14893 break;
14894 case PM_TOKEN_METHOD_NAME:
14895 pm_parser_err_previous(parser, PM_ERR_PARAMETER_METHOD_NAME);
14896 break;
14897 default: break;
14898 }
14899
14900 if (parser->current.type == PM_TOKEN_EQUAL) {
14901 update_parameter_state(parser, &parser->current, &order);
14902 } else {
14903 update_parameter_state(parser, &parser->previous, &order);
14904 }
14905
14906 pm_token_t name = parser->previous;
14907 bool repeated = pm_parser_parameter_name_check(parser, &name);
14908 pm_parser_local_add_token(parser, &name, 1);
14909
14910 if (match1(parser, PM_TOKEN_EQUAL)) {
14911 pm_token_t operator = parser->current;
14912 context_push(parser, PM_CONTEXT_DEFAULT_PARAMS);
14913 parser_lex(parser);
14914
14915 pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &name);
14916 uint32_t reads = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? pm_locals_reads(parser, name_id) : 0;
14917
14918 if (accepts_blocks_in_defaults) pm_accepts_block_stack_push(parser, true);
14919 pm_node_t *value = parse_value_expression(parser, binding_power, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_PARAMETER_NO_DEFAULT, (uint16_t) (depth + 1));
14920 if (accepts_blocks_in_defaults) pm_accepts_block_stack_pop(parser);
14921
14922 pm_optional_parameter_node_t *param = pm_optional_parameter_node_create(parser, &name, &operator, value);
14923
14924 if (repeated) {
14925 pm_node_flag_set_repeated_parameter(UP(param));
14926 }
14927 pm_parameters_optionals_append(parser, &params, param);
14928
14929 // If the value of the parameter increased the number of
14930 // reads of that parameter, then we need to warn that we
14931 // have a circular definition.
14932 if ((parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3) && (pm_locals_reads(parser, name_id) != reads)) {
14933 PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &name, PM_ERR_PARAMETER_CIRCULAR);
14934 }
14935
14936 context_pop(parser);
14937
14938 // If parsing the value of the parameter resulted in error recovery,
14939 // then we can put a missing node in its place and stop parsing the
14940 // parameters entirely now.
14941 if (parser->recovering) {
14942 parsing = false;
14943 break;
14944 }
14945 } else if (order > PM_PARAMETERS_ORDER_AFTER_OPTIONAL) {
14946 pm_required_parameter_node_t *param = pm_required_parameter_node_create(parser, &name);
14947 if (repeated) {
14948 pm_node_flag_set_repeated_parameter(UP(param));
14949 }
14950 pm_parameters_requireds_append(parser, &params, UP(param));
14951 } else {
14952 pm_required_parameter_node_t *param = pm_required_parameter_node_create(parser, &name);
14953 if (repeated) {
14954 pm_node_flag_set_repeated_parameter(UP(param));
14955 }
14956 pm_parameters_posts_append(parser, &params, UP(param));
14957 }
14958
14959 break;
14960 }
14961 case PM_TOKEN_LABEL: {
14962 if (!uses_parentheses && !in_block) parser->in_keyword_arg = true;
14963 update_parameter_state(parser, &parser->current, &order);
14964
14965 context_push(parser, PM_CONTEXT_DEFAULT_PARAMS);
14966
14967 pm_token_t name = parser->current;
14968 pm_token_t local = name;
14969 local.end -= 1;
14970
14971 if (parser->encoding_changed ? pm_encoding_isupper_char(parser->encoding, local.start, local.end - local.start, parser->version) : pm_encoding_utf_8_isupper_char(local.start, local.end - local.start, parser->version)) {
14972 pm_parser_err(parser, PM_TOKEN_START(parser, &local), PM_TOKEN_LENGTH(&local), PM_ERR_ARGUMENT_FORMAL_CONSTANT);
14973 } else if (local.end[-1] == '!' || local.end[-1] == '?') {
14974 PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE);
14975 }
14976
14977 bool repeated = pm_parser_parameter_name_check(parser, &local);
14978 /* Make the parameter visible before lexing its default. */
14979 pm_parser_local_add_token(parser, &local, 1);
14980 parser_lex(parser);
14981
14982 switch (parser->current.type) {
14983 case PM_TOKEN_COMMA:
14984 case PM_TOKEN_PARENTHESIS_RIGHT:
14985 case PM_TOKEN_PIPE: {
14986 context_pop(parser);
14987
14988 pm_node_t *param = UP(pm_required_keyword_parameter_node_create(parser, &name));
14989 if (repeated) {
14990 pm_node_flag_set_repeated_parameter(param);
14991 }
14992
14993 pm_parameters_keywords_append(parser, &params, param);
14994 break;
14995 }
14996 case PM_TOKEN_SEMICOLON:
14997 case PM_TOKEN_NEWLINE: {
14998 context_pop(parser);
14999
15000 if (uses_parentheses) {
15001 parsing = false;
15002 break;
15003 }
15004
15005 pm_node_t *param = UP(pm_required_keyword_parameter_node_create(parser, &name));
15006 if (repeated) {
15007 pm_node_flag_set_repeated_parameter(param);
15008 }
15009
15010 pm_parameters_keywords_append(parser, &params, param);
15011 break;
15012 }
15013 default: {
15014 pm_node_t *param;
15015
15016 if (token_begins_expression_p(parser->current.type)) {
15017 pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &local);
15018 uint32_t reads = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? pm_locals_reads(parser, name_id) : 0;
15019
15020 if (accepts_blocks_in_defaults) pm_accepts_block_stack_push(parser, true);
15021 pm_node_t *value = parse_value_expression(parser, binding_power, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_PARAMETER_NO_DEFAULT_KW, (uint16_t) (depth + 1));
15022 if (accepts_blocks_in_defaults) pm_accepts_block_stack_pop(parser);
15023
15024 if (parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 && (pm_locals_reads(parser, name_id) != reads)) {
15025 PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &local, PM_ERR_PARAMETER_CIRCULAR);
15026 }
15027
15028 param = UP(pm_optional_keyword_parameter_node_create(parser, &name, value));
15029 }
15030 else {
15031 param = UP(pm_required_keyword_parameter_node_create(parser, &name));
15032 }
15033
15034 if (repeated) {
15035 pm_node_flag_set_repeated_parameter(param);
15036 }
15037
15038 context_pop(parser);
15039 pm_parameters_keywords_append(parser, &params, param);
15040
15041 // If parsing the value of the parameter resulted in error recovery,
15042 // then we can put a missing node in its place and stop parsing the
15043 // parameters entirely now.
15044 if (parser->recovering) {
15045 parsing = false;
15046 break;
15047 }
15048 }
15049 }
15050
15051 parser->in_keyword_arg = false;
15052 break;
15053 }
15054 case PM_TOKEN_USTAR:
15055 case PM_TOKEN_STAR: {
15056 update_parameter_state(parser, &parser->current, &order);
15057 parser_lex(parser);
15058
15059 pm_token_t operator = parser->previous;
15060 pm_token_t name = { 0 };
15061 bool repeated = false;
15062
15063 if (accept1(parser, PM_TOKEN_IDENTIFIER)) {
15064 name = parser->previous;
15065 repeated = pm_parser_parameter_name_check(parser, &name);
15066 pm_parser_local_add_token(parser, &name, 1);
15067 } else {
15068 parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS;
15069 }
15070
15071 pm_node_t *param = UP(pm_rest_parameter_node_create(parser, &operator, NTOK2PTR(name)));
15072 if (repeated) {
15073 pm_node_flag_set_repeated_parameter(param);
15074 }
15075
15076 if (params.rest == NULL) {
15077 pm_parameters_rest_set(&params, param);
15078 } else {
15079 pm_parser_err_node(parser, param, PM_ERR_PARAMETER_SPLAT_MULTI);
15080 pm_parameters_posts_append(parser, &params, param);
15081 }
15082
15083 break;
15084 }
15085 case PM_TOKEN_STAR_STAR:
15086 case PM_TOKEN_USTAR_STAR: {
15087 pm_parameters_order_t previous_order = order;
15088 update_parameter_state(parser, &parser->current, &order);
15089 parser_lex(parser);
15090
15091 pm_token_t operator = parser->previous;
15092 pm_node_t *param;
15093
15094 if (accept1(parser, PM_TOKEN_KEYWORD_NIL)) {
15095 if (previous_order <= PM_PARAMETERS_ORDER_KEYWORDS) {
15096 pm_parser_err_previous(parser, PM_ERR_PARAMETER_UNEXPECTED_NO_KW);
15097 }
15098
15099 param = UP(pm_no_keywords_parameter_node_create(parser, &operator, &parser->previous));
15100 } else {
15101 pm_token_t name = { 0 };
15102
15103 bool repeated = false;
15104 if (accept1(parser, PM_TOKEN_IDENTIFIER)) {
15105 name = parser->previous;
15106 repeated = pm_parser_parameter_name_check(parser, &name);
15107 pm_parser_local_add_token(parser, &name, 1);
15108 } else {
15109 parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS;
15110 }
15111
15112 param = UP(pm_keyword_rest_parameter_node_create(parser, &operator, NTOK2PTR(name)));
15113 if (repeated) {
15114 pm_node_flag_set_repeated_parameter(param);
15115 }
15116 }
15117
15118 if (params.keyword_rest == NULL) {
15119 pm_parameters_keyword_rest_set(&params, param);
15120 } else {
15121 pm_parser_err_node(parser, param, PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI);
15122 pm_parameters_posts_append(parser, &params, UP(pm_error_recovery_node_create_unexpected(parser, param)));
15123 }
15124
15125 break;
15126 }
15127 default:
15128 if (parser->previous.type == PM_TOKEN_COMMA) {
15129 parse_parameters_handle_trailing_comma(parser, &params, order, in_block, allows_trailing_comma);
15130 }
15131
15132 parsing = false;
15133 break;
15134 }
15135
15136 // If we hit some kind of issue while parsing the parameter, this would
15137 // have been set to false. In that case, we need to break out of the
15138 // loop.
15139 if (!parsing) break;
15140
15141 bool accepted_newline = false;
15142 if (uses_parentheses) {
15143 accepted_newline = accept1(parser, PM_TOKEN_NEWLINE);
15144 }
15145
15146 if (accept1(parser, PM_TOKEN_COMMA)) {
15147 // If there was a comma, but we also accepted a newline, then this
15148 // is a syntax error.
15149 if (accepted_newline) {
15150 pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
15151 }
15152 } else {
15153 // If there was no comma, then we're done parsing parameters.
15154 break;
15155 }
15156 }
15157
15158 pm_do_loop_stack_pop(parser);
15159
15160 // If we don't have any parameters, return `NULL` instead of an empty `ParametersNode`.
15161 if (params.location.length == 0) {
15162 return NULL;
15163 }
15164
15165 return pm_parameters_node_create(parser, &params);
15166}
15167
15172static size_t
15173token_newline_index(const pm_parser_t *parser) {
15174 if (parser->heredoc_end == NULL) {
15175 // This is the common case. In this case we can look at the previously
15176 // recorded newline in the newline list and subtract from the current
15177 // offset.
15178 return parser->line_offsets.size - 1;
15179 } else {
15180 // This is unlikely. This is the case that we have already parsed the
15181 // start of a heredoc, so we cannot rely on looking at the previous
15182 // offset of the newline list, and instead must go through the whole
15183 // process of a binary search for the line number.
15184 return (size_t) pm_line_offset_list_line(&parser->line_offsets, PM_TOKEN_START(parser, &parser->current), 0);
15185 }
15186}
15187
15192static int64_t
15193token_column(const pm_parser_t *parser, size_t newline_index, const pm_token_t *token, bool break_on_non_space) {
15194 const uint8_t *cursor = parser->start + parser->line_offsets.offsets[newline_index];
15195 const uint8_t *end = token->start;
15196
15197 // Skip over the BOM if it is present.
15198 if (
15199 newline_index == 0 &&
15200 parser->start[0] == 0xef &&
15201 parser->start[1] == 0xbb &&
15202 parser->start[2] == 0xbf
15203 ) cursor += 3;
15204
15205 int64_t column = 0;
15206 for (; cursor < end; cursor++) {
15207 switch (*cursor) {
15208 case '\t':
15209 column = ((column / PM_TAB_WHITESPACE_SIZE) + 1) * PM_TAB_WHITESPACE_SIZE;
15210 break;
15211 case ' ':
15212 column++;
15213 break;
15214 default:
15215 column++;
15216 if (break_on_non_space) return -1;
15217 break;
15218 }
15219 }
15220
15221 return column;
15222}
15223
15228static void
15229parser_warn_indentation_mismatch(pm_parser_t *parser, size_t opening_newline_index, const pm_token_t *opening_token, bool if_after_else, bool allow_indent) {
15230 // If these warnings are disabled (unlikely), then we can just return.
15231 if (!parser->warn_mismatched_indentation) return;
15232
15233 // If the tokens are on the same line, we do not warn.
15234 size_t closing_newline_index = token_newline_index(parser);
15235 if (opening_newline_index == closing_newline_index) return;
15236
15237 // If the opening token has anything other than spaces or tabs before it,
15238 // then we do not warn. This is unless we are matching up an `if`/`end` pair
15239 // and the `if` immediately follows an `else` keyword.
15240 int64_t opening_column = token_column(parser, opening_newline_index, opening_token, !if_after_else);
15241 if (!if_after_else && (opening_column == -1)) return;
15242
15243 // Get a reference to the closing token off the current parser. This assumes
15244 // that the caller has placed this in the correct position.
15245 pm_token_t *closing_token = &parser->current;
15246
15247 // If the tokens are at the same indentation, we do not warn.
15248 int64_t closing_column = token_column(parser, closing_newline_index, closing_token, true);
15249 if ((closing_column == -1) || (opening_column == closing_column)) return;
15250
15251 // If the closing column is greater than the opening column and we are
15252 // allowing indentation, then we do not warn.
15253 if (allow_indent && (closing_column > opening_column)) return;
15254
15255 // Otherwise, add a warning.
15256 PM_PARSER_WARN_FORMAT(
15257 parser,
15258 PM_TOKEN_START(parser, closing_token),
15259 PM_TOKEN_LENGTH(closing_token),
15260 PM_WARN_INDENTATION_MISMATCH,
15261 (int) (closing_token->end - closing_token->start),
15262 (const char *) closing_token->start,
15263 (int) (opening_token->end - opening_token->start),
15264 (const char *) opening_token->start,
15265 pm_line_offset_list_line_number(opening_newline_index, parser->start_line)
15266 );
15267}
15268
15269typedef enum {
15270 PM_RESCUES_BEGIN = 1,
15271 PM_RESCUES_BLOCK,
15272 PM_RESCUES_CLASS,
15273 PM_RESCUES_DEF,
15274 PM_RESCUES_LAMBDA,
15275 PM_RESCUES_MODULE,
15276 PM_RESCUES_SCLASS
15277} pm_rescues_type_t;
15278
15283static PRISM_INLINE void
15284parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_t *opening, pm_begin_node_t *parent_node, pm_rescues_type_t type, uint16_t depth) {
15285 pm_rescue_node_t *current = NULL;
15286
15287 while (match1(parser, PM_TOKEN_KEYWORD_RESCUE)) {
15288 if (opening != NULL) parser_warn_indentation_mismatch(parser, opening_newline_index, opening, false, false);
15289 parser_lex(parser);
15290
15291 pm_token_t rescue_keyword = parser->previous;
15292 pm_node_list_t exceptions = { 0 };
15293 pm_token_t operator = { 0 };
15294 pm_node_t *reference = NULL;
15295 pm_token_t then_keyword = { 0 };
15296 pm_statements_node_t *statements = NULL;
15297
15298 switch (parser->current.type) {
15299 case PM_TOKEN_EQUAL_GREATER: {
15300 // Here we have an immediate => after the rescue keyword, in which case
15301 // we're going to have an empty list of exceptions to rescue (which
15302 // implies StandardError).
15303 parser_lex(parser);
15304 operator = parser->previous;
15305
15306 reference = parse_expression(parser, PM_BINDING_POWER_INDEX, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_RESCUE_VARIABLE, (uint16_t) (depth + 1));
15307 reference = parse_target(parser, reference, false, false);
15308 break;
15309 }
15310 case PM_TOKEN_NEWLINE:
15311 case PM_TOKEN_SEMICOLON:
15312 case PM_TOKEN_KEYWORD_THEN:
15313 // Here we have a terminator for the rescue keyword, in which
15314 // case we're going to just continue on.
15315 break;
15316 default: {
15317 if (token_begins_expression_p(parser->current.type) || match1(parser, PM_TOKEN_USTAR)) {
15318 // Here we have something that could be an exception expression, so
15319 // we'll attempt to parse it here and any others delimited by commas.
15320
15321 do {
15322 pm_node_t *expression = parse_starred_expression(parser, PM_BINDING_POWER_DEFINED, false, PM_ERR_RESCUE_EXPRESSION, (uint16_t) (depth + 1));
15323 pm_node_list_open_append(&parser->node_chunks, &exceptions, expression);
15324
15325 // If we hit a newline, then this is the end of the rescue expression. We
15326 // can continue on to parse the statements.
15327 if (match3(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_TOKEN_KEYWORD_THEN)) break;
15328
15329 // If we hit a `=>` then we're going to parse the exception variable. Once
15330 // we've done that, we'll break out of the loop and parse the statements.
15331 if (accept1(parser, PM_TOKEN_EQUAL_GREATER)) {
15332 operator = parser->previous;
15333
15334 reference = parse_expression(parser, PM_BINDING_POWER_INDEX, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_RESCUE_VARIABLE, (uint16_t) (depth + 1));
15335 reference = parse_target(parser, reference, false, false);
15336 break;
15337 }
15338 } while (accept1(parser, PM_TOKEN_COMMA));
15339 }
15340 }
15341 }
15342
15343 if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) {
15344 if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) {
15345 then_keyword = parser->previous;
15346 }
15347 } else {
15348 expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_RESCUE_TERM);
15349 then_keyword = parser->previous;
15350 }
15351
15352 if (!match3(parser, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_END)) {
15353 pm_accepts_block_stack_push(parser, true);
15354 pm_context_t context;
15355
15356 switch (type) {
15357 case PM_RESCUES_BEGIN: context = PM_CONTEXT_BEGIN_RESCUE; break;
15358 case PM_RESCUES_BLOCK: context = PM_CONTEXT_BLOCK_RESCUE; break;
15359 case PM_RESCUES_CLASS: context = PM_CONTEXT_CLASS_RESCUE; break;
15360 case PM_RESCUES_DEF: context = PM_CONTEXT_DEF_RESCUE; break;
15361 case PM_RESCUES_LAMBDA: context = PM_CONTEXT_LAMBDA_RESCUE; break;
15362 case PM_RESCUES_MODULE: context = PM_CONTEXT_MODULE_RESCUE; break;
15363 case PM_RESCUES_SCLASS: context = PM_CONTEXT_SCLASS_RESCUE; break;
15364 default: PRISM_ASSERT(false && "unreachable"); context = PM_CONTEXT_BEGIN_RESCUE; break;
15365 }
15366
15367 statements = parse_statements(parser, context, (uint16_t) (depth + 1));
15368
15369 pm_accepts_block_stack_pop(parser);
15370 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
15371 }
15372
15373 pm_rescue_node_t *rescue = pm_rescue_node_create(parser, &rescue_keyword, &exceptions, NTOK2PTR(operator), reference, NTOK2PTR(then_keyword), statements);
15374
15375 if (current == NULL) {
15376 pm_begin_node_rescue_clause_set(parent_node, rescue);
15377 } else {
15378 pm_rescue_node_subsequent_set(current, rescue);
15379 }
15380
15381 current = rescue;
15382 }
15383
15384 // The end node locations on rescue nodes will not be set correctly
15385 // since we won't know the end until we've found all subsequent
15386 // clauses. This sets the end location on all rescues once we know it.
15387 if (current != NULL) {
15388 pm_rescue_node_t *clause = parent_node->rescue_clause;
15389
15390 while (clause != NULL) {
15391 PM_NODE_LENGTH_SET_NODE(clause, current);
15392 clause = clause->subsequent;
15393 }
15394 }
15395
15396 pm_token_t else_keyword;
15397 if (match1(parser, PM_TOKEN_KEYWORD_ELSE)) {
15398 if (opening != NULL) parser_warn_indentation_mismatch(parser, opening_newline_index, opening, false, false);
15399 opening_newline_index = token_newline_index(parser);
15400
15401 else_keyword = parser->current;
15402 opening = &else_keyword;
15403
15404 parser_lex(parser);
15405 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
15406
15407 pm_statements_node_t *else_statements = NULL;
15408 if (!match2(parser, PM_TOKEN_KEYWORD_END, PM_TOKEN_KEYWORD_ENSURE)) {
15409 pm_accepts_block_stack_push(parser, true);
15410 pm_context_t context;
15411
15412 switch (type) {
15413 case PM_RESCUES_BEGIN: context = PM_CONTEXT_BEGIN_ELSE; break;
15414 case PM_RESCUES_BLOCK: context = PM_CONTEXT_BLOCK_ELSE; break;
15415 case PM_RESCUES_CLASS: context = PM_CONTEXT_CLASS_ELSE; break;
15416 case PM_RESCUES_DEF: context = PM_CONTEXT_DEF_ELSE; break;
15417 case PM_RESCUES_LAMBDA: context = PM_CONTEXT_LAMBDA_ELSE; break;
15418 case PM_RESCUES_MODULE: context = PM_CONTEXT_MODULE_ELSE; break;
15419 case PM_RESCUES_SCLASS: context = PM_CONTEXT_SCLASS_ELSE; break;
15420 default: PRISM_ASSERT(false && "unreachable"); context = PM_CONTEXT_BEGIN_ELSE; break;
15421 }
15422
15423 else_statements = parse_statements(parser, context, (uint16_t) (depth + 1));
15424 pm_accepts_block_stack_pop(parser);
15425
15426 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
15427 }
15428
15429 pm_else_node_t *else_clause = pm_else_node_create(parser, &else_keyword, else_statements, &parser->current);
15430 pm_begin_node_else_clause_set(parent_node, else_clause);
15431
15432 // If we don't have a `current` rescue node, then this is a dangling
15433 // else, and it's an error.
15434 if (current == NULL) pm_parser_err_node(parser, UP(else_clause), PM_ERR_BEGIN_LONELY_ELSE);
15435 }
15436
15437 if (match1(parser, PM_TOKEN_KEYWORD_ENSURE)) {
15438 if (opening != NULL) parser_warn_indentation_mismatch(parser, opening_newline_index, opening, false, false);
15439 pm_token_t ensure_keyword = parser->current;
15440
15441 parser_lex(parser);
15442 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
15443
15444 pm_statements_node_t *ensure_statements = NULL;
15445 if (!match1(parser, PM_TOKEN_KEYWORD_END)) {
15446 pm_accepts_block_stack_push(parser, true);
15447 pm_context_t context;
15448
15449 switch (type) {
15450 case PM_RESCUES_BEGIN: context = PM_CONTEXT_BEGIN_ENSURE; break;
15451 case PM_RESCUES_BLOCK: context = PM_CONTEXT_BLOCK_ENSURE; break;
15452 case PM_RESCUES_CLASS: context = PM_CONTEXT_CLASS_ENSURE; break;
15453 case PM_RESCUES_DEF: context = PM_CONTEXT_DEF_ENSURE; break;
15454 case PM_RESCUES_LAMBDA: context = PM_CONTEXT_LAMBDA_ENSURE; break;
15455 case PM_RESCUES_MODULE: context = PM_CONTEXT_MODULE_ENSURE; break;
15456 case PM_RESCUES_SCLASS: context = PM_CONTEXT_SCLASS_ENSURE; break;
15457 default: PRISM_ASSERT(false && "unreachable"); context = PM_CONTEXT_BEGIN_RESCUE; break;
15458 }
15459
15460 ensure_statements = parse_statements(parser, context, (uint16_t) (depth + 1));
15461 pm_accepts_block_stack_pop(parser);
15462
15463 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
15464 }
15465
15466 pm_ensure_node_t *ensure_clause = pm_ensure_node_create(parser, &ensure_keyword, ensure_statements, &parser->current);
15467 pm_begin_node_ensure_clause_set(parent_node, ensure_clause);
15468 }
15469
15470 if (match1(parser, PM_TOKEN_KEYWORD_END)) {
15471 if (opening != NULL) parser_warn_indentation_mismatch(parser, opening_newline_index, opening, false, false);
15472 pm_begin_node_end_keyword_set(parser, parent_node, &parser->current);
15473 } else {
15474 pm_token_t end_keyword = (pm_token_t) { .type = PM_TOKEN_KEYWORD_END, .start = parser->previous.end, .end = parser->previous.end };
15475 pm_begin_node_end_keyword_set(parser, parent_node, &end_keyword);
15476 }
15477}
15478
15483static pm_begin_node_t *
15484parse_rescues_implicit_begin(pm_parser_t *parser, size_t opening_newline_index, const pm_token_t *opening, const uint8_t *start, pm_statements_node_t *statements, pm_rescues_type_t type, uint16_t depth) {
15485 pm_begin_node_t *node = pm_begin_node_create(parser, NULL, statements);
15486 parse_rescues(parser, opening_newline_index, opening, node, type, (uint16_t) (depth + 1));
15487
15488 node->base.location.start = U32(start - parser->start);
15489 PM_NODE_LENGTH_SET_TOKEN(parser, node, &parser->current);
15490
15491 return node;
15492}
15493
15497static pm_block_parameters_node_t *
15498parse_block_parameters(
15499 pm_parser_t *parser,
15500 bool allows_trailing_comma,
15501 const pm_token_t *opening,
15502 bool is_lambda_literal,
15503 bool accepts_blocks_in_defaults,
15504 uint16_t depth
15505) {
15506 pm_parameters_node_t *parameters = NULL;
15507 if (!match1(parser, PM_TOKEN_SEMICOLON)) {
15508 if (!is_lambda_literal) {
15509 context_push(parser, PM_CONTEXT_BLOCK_PARAMETERS);
15510 }
15511 parameters = parse_parameters(
15512 parser,
15513 is_lambda_literal ? PM_BINDING_POWER_DEFINED : PM_BINDING_POWER_INDEX,
15514 false,
15515 allows_trailing_comma,
15516 false,
15517 accepts_blocks_in_defaults,
15518 true,
15519 is_lambda_literal ? PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_LAMBDA : PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_BLOCK,
15520 (uint16_t) (depth + 1)
15521 );
15522 if (!is_lambda_literal) {
15523 context_pop(parser);
15524 }
15525 }
15526
15527 pm_node_list_t locals = { 0 };
15528 if (opening != NULL) {
15529 accept1(parser, PM_TOKEN_NEWLINE);
15530
15531 if (accept1(parser, PM_TOKEN_SEMICOLON)) {
15532 do {
15533 switch (parser->current.type) {
15534 case PM_TOKEN_CONSTANT:
15535 pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_CONSTANT);
15536 parser_lex(parser);
15537 break;
15538 case PM_TOKEN_INSTANCE_VARIABLE:
15539 pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_IVAR);
15540 parser_lex(parser);
15541 break;
15542 case PM_TOKEN_GLOBAL_VARIABLE:
15543 pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_GLOBAL);
15544 parser_lex(parser);
15545 break;
15546 case PM_TOKEN_CLASS_VARIABLE:
15547 pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_CLASS);
15548 parser_lex(parser);
15549 break;
15550 default:
15551 expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE);
15552 break;
15553 }
15554
15555 bool repeated = pm_parser_parameter_name_check(parser, &parser->previous);
15556 pm_parser_local_add_token(parser, &parser->previous, 1);
15557
15558 pm_block_local_variable_node_t *local = pm_block_local_variable_node_create(parser, &parser->previous);
15559 if (repeated) pm_node_flag_set_repeated_parameter(UP(local));
15560
15561 pm_node_list_open_append(&parser->node_chunks, &locals, UP(local));
15562 } while (accept1(parser, PM_TOKEN_COMMA));
15563 }
15564 }
15565
15566 return pm_block_parameters_node_create(parser, parameters, opening, &locals);
15567}
15568
15573static bool
15574outer_scope_using_numbered_parameters_p(pm_parser_t *parser) {
15575 for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) {
15576 if (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) return true;
15577 }
15578
15579 return false;
15580}
15581
15587static const char * const pm_numbered_parameter_names[] = {
15588 "_1", "_2", "_3", "_4", "_5", "_6", "_7", "_8", "_9"
15589};
15590
15596static pm_node_t *
15597parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_token_t *opening, const pm_token_t *closing) {
15598 pm_scratch_node_list_t *implicit_parameters = &parser->current_scope->implicit_parameters;
15599
15600 // If we have ordinary parameters, then we will return them as the set of
15601 // parameters.
15602 if (parameters != NULL) {
15603 // If we also have implicit parameters, then this is an error.
15604 if (implicit_parameters->size > 0) {
15605 pm_node_t *node = implicit_parameters->nodes[0];
15606
15607 if (PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)) {
15608 pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_ORDINARY);
15609 } else if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) {
15610 pm_parser_err_node(parser, node, PM_ERR_IT_NOT_ALLOWED_ORDINARY);
15611 } else {
15612 PRISM_ASSERT(false && "unreachable");
15613 }
15614 }
15615
15616 return parameters;
15617 }
15618
15619 // If we don't have any implicit parameters, then the set of parameters is
15620 // NULL.
15621 if (implicit_parameters->size == 0) {
15622 return NULL;
15623 }
15624
15625 // If we don't have ordinary parameters, then we now must validate our set
15626 // of implicit parameters. We can only have numbered parameters or it, but
15627 // they cannot be mixed.
15628 uint8_t numbered_parameter = 0;
15629 bool it_parameter = false;
15630
15631 for (size_t index = 0; index < implicit_parameters->size; index++) {
15632 pm_node_t *node = implicit_parameters->nodes[index];
15633
15634 if (PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)) {
15635 if (it_parameter) {
15636 pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_IT);
15637 } else if (outer_scope_using_numbered_parameters_p(parser)) {
15638 pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK);
15639 } else if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_INNER) {
15640 pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK);
15641 } else if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) {
15642 numbered_parameter = MAX(numbered_parameter, (uint8_t) (parser->start[node->location.start + 1] - '0'));
15643 } else {
15644 PRISM_ASSERT(false && "unreachable");
15645 }
15646 } else if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) {
15647 if (numbered_parameter > 0) {
15648 pm_parser_err_node(parser, node, PM_ERR_IT_NOT_ALLOWED_NUMBERED);
15649 } else {
15650 it_parameter = true;
15651 }
15652 }
15653 }
15654
15655 if (numbered_parameter > 0) {
15656 // Go through the parent scopes and mark them as being disallowed from
15657 // using numbered parameters because this inner scope is using them.
15658 for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) {
15659 scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_INNER;
15660 }
15661 return UP(pm_numbered_parameters_node_create(parser, opening, closing, numbered_parameter));
15662 }
15663
15664 if (it_parameter) {
15665 return UP(pm_it_parameters_node_create(parser, opening, closing));
15666 }
15667
15668 return NULL;
15669}
15670
15674static pm_block_node_t *
15675parse_block(pm_parser_t *parser, uint16_t depth) {
15676 pm_token_t opening = parser->previous;
15677 accept1(parser, PM_TOKEN_NEWLINE);
15678
15679 /* A brace block is delimited by `{`/`}`, whose block-accepting frame is
15680 * managed by the lexer. A `do`/`end` block is delimited by keywords, so we
15681 * push the frame here (covering the block parameters and body) and pop it
15682 * before consuming `end`, mirroring parse.y's `do_body` rule. */
15683 bool do_block = opening.type != PM_TOKEN_BRACE_LEFT && opening.type != PM_TOKEN_BRACE_LEFT_ARGUMENT;
15684 if (do_block) pm_accepts_block_stack_push(parser, true);
15685 pm_parser_scope_push(parser, false);
15686
15687 pm_block_parameters_node_t *block_parameters = NULL;
15688
15689 if (accept1(parser, PM_TOKEN_PIPE)) {
15690 pm_token_t block_parameters_opening = parser->previous;
15691 if (match1(parser, PM_TOKEN_PIPE)) {
15692 block_parameters = pm_block_parameters_node_create(parser, NULL, &block_parameters_opening, &((pm_node_list_t) { 0 }));
15693 parser->command_start = true;
15694 parser_lex(parser);
15695 } else {
15696 block_parameters = parse_block_parameters(parser, true, &block_parameters_opening, false, true, (uint16_t) (depth + 1));
15697 accept1(parser, PM_TOKEN_NEWLINE);
15698 parser->command_start = true;
15699 expect1(parser, PM_TOKEN_PIPE, PM_ERR_BLOCK_PARAM_PIPE_TERM);
15700 }
15701
15702 pm_block_parameters_node_closing_set(parser, block_parameters, &parser->previous);
15703 }
15704
15705 accept1(parser, PM_TOKEN_NEWLINE);
15706 pm_node_t *statements = NULL;
15707
15708 if (!do_block) {
15709 if (!match1(parser, PM_TOKEN_BRACE_RIGHT)) {
15710 statements = UP(parse_statements(parser, PM_CONTEXT_BLOCK_BRACES, (uint16_t) (depth + 1)));
15711 }
15712
15713 expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_BLOCK_TERM_BRACE, &opening);
15714 } else {
15715 if (!match1(parser, PM_TOKEN_KEYWORD_END)) {
15716 if (!match3(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_ENSURE)) {
15717 statements = UP(parse_statements(parser, PM_CONTEXT_BLOCK_KEYWORDS, (uint16_t) (depth + 1)));
15718 }
15719
15720 if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
15721 PRISM_ASSERT(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
15722 statements = UP(parse_rescues_implicit_begin(parser, 0, NULL, opening.start, (pm_statements_node_t *) statements, PM_RESCUES_BLOCK, (uint16_t) (depth + 1)));
15723 }
15724 }
15725
15726 /* Pop the `do`/`end` frame before consuming `end` so the token
15727 * following the block is lexed in the enclosing context. */
15728 pm_accepts_block_stack_pop(parser);
15729 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_BLOCK_TERM_END, &opening);
15730 }
15731
15732 pm_constant_id_list_t locals;
15733 pm_locals_order(parser, &locals, pm_parser_scope_toplevel_p(parser));
15734 pm_node_t *parameters = parse_blocklike_parameters(parser, UP(block_parameters), &opening, &parser->previous);
15735
15736 pm_parser_scope_pop(parser);
15737 return pm_block_node_create(parser, &locals, &opening, parameters, statements, &parser->previous);
15738}
15739
15750static uint8_t
15751parse_delimited_arguments_flags(const pm_parser_t *parser, uint8_t flags, bool index) {
15752 if (parser->version >= PM_OPTIONS_VERSION_CRUBY_3_4 && (!index || parser->version <= PM_OPTIONS_VERSION_CRUBY_4_0)) {
15753 return (uint8_t) (flags | PM_PARSE_ACCEPTS_DO_BLOCK);
15754 }
15755
15756 return (uint8_t) (flags & ~PM_PARSE_ACCEPTS_DO_BLOCK);
15757}
15758
15770static bool
15771parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_arguments, uint8_t flags, uint16_t depth) {
15772 /* Fast path: if the current token can't begin an expression and isn't
15773 * a parenthesis, block opener, or splat/block-pass operator, there are
15774 * no arguments to parse. */
15775 if (
15776 !token_begins_expression_p(parser->current.type) &&
15777 !match6(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_KEYWORD_DO, PM_TOKEN_KEYWORD_DO_BLOCK, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR, PM_TOKEN_UAMPERSAND)
15778 ) {
15779 return false;
15780 }
15781
15782 bool found = false;
15783 bool parsed_command_args = false;
15784
15785 if (accept1(parser, PM_TOKEN_PARENTHESIS_LEFT)) {
15786 found |= true;
15787 arguments->opening_loc = TOK2LOC(parser, &parser->previous);
15788
15789 if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
15790 arguments->closing_loc = TOK2LOC(parser, &parser->previous);
15791 } else {
15792 parse_arguments(parser, arguments, full_arguments, PM_TOKEN_PARENTHESIS_RIGHT, parse_delimited_arguments_flags(parser, flags, false), (uint16_t) (depth + 1));
15793
15794 // `yield` parses its arguments through the restricted `call_args`
15795 // grammar, which (unlike the `opt_call_args` that method calls and
15796 // `super` use) permits neither a block argument nor a trailing
15797 // comma. `full_arguments` is false only for `yield`, so we use it
15798 // to reject the trailing comma in `yield(a,)` that the arguments
15799 // parser otherwise accepts before the closing parenthesis.
15800 if (!full_arguments && parser->previous.type == PM_TOKEN_COMMA) {
15801 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_str(parser->current.type));
15802 }
15803
15804 if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
15805 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_str(parser->current.type));
15806 parser->previous.start = parser->previous.end;
15807 parser->previous.type = 0;
15808 }
15809
15810 arguments->closing_loc = TOK2LOC(parser, &parser->previous);
15811 }
15812 } else if ((flags & PM_PARSE_ACCEPTS_COMMAND_CALL) && token_begins_argument_p(parser->current.type) && !match1(parser, PM_TOKEN_BRACE_LEFT)) {
15813 found |= true;
15814 parsed_command_args = true;
15815
15816 /* The command-args frame does not accept blocks, so that a trailing
15817 * `do` binds to this command rather than to an argument. Mirroring
15818 * parse.y's `command_args` rule: when the first argument begins with an
15819 * opening delimiter, the lexer has already pushed that delimiter's
15820 * (block-accepting) frame. We must push the command-args frame beneath
15821 * it, so pop the delimiter frame, push the command-args frame, and then
15822 * restore the delimiter frame on top (the delimiter's closing token
15823 * will pop it back off during argument parsing). */
15824 bool lookahead_delimiter = match5(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_GROUPING, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES, PM_TOKEN_BRACKET_LEFT, PM_TOKEN_BRACKET_LEFT_ARRAY);
15825 if (lookahead_delimiter) pm_accepts_block_stack_pop(parser);
15826 pm_accepts_block_stack_push(parser, false);
15827 if (lookahead_delimiter) pm_accepts_block_stack_push(parser, true);
15828
15829 // If we get here, then the subsequent token cannot be used as an infix
15830 // operator. In this case we assume the subsequent token is part of an
15831 // argument to this method call.
15832 parse_arguments(parser, arguments, full_arguments, PM_TOKEN_EOF, flags, (uint16_t) (depth + 1));
15833
15834 // If we have done with the arguments and still not consumed the comma,
15835 // then we have a trailing comma where we need to check whether it is
15836 // allowed or not.
15837 if (parser->previous.type == PM_TOKEN_COMMA && !match1(parser, PM_TOKEN_SEMICOLON)) {
15838 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_str(parser->current.type));
15839 }
15840
15841 /* Symmetrically, if the command arguments are followed by a brace block
15842 * (`m args { }`), the lexer has already pushed that block's frame. Pop
15843 * it, pop the command-args frame beneath it, and restore the block
15844 * frame so the block's `}` still pops it. This mirrors the `tLBRACE_ARG`
15845 * lookahead handling in parse.y's `command_args` rule. */
15846 bool lookahead_brace = match2(parser, PM_TOKEN_BRACE_LEFT, PM_TOKEN_BRACE_LEFT_ARGUMENT);
15847 if (lookahead_brace) pm_accepts_block_stack_pop(parser);
15848 pm_accepts_block_stack_pop(parser);
15849 if (lookahead_brace) pm_accepts_block_stack_push(parser, true);
15850 }
15851
15852 // If we're at the end of the arguments, we can now check if there is a block
15853 // node that starts with a {. If there is, then we can parse it and add it to
15854 // the arguments.
15855 if (full_arguments) {
15856 pm_block_node_t *block = NULL;
15857
15858 if (accept2(parser, PM_TOKEN_BRACE_LEFT, PM_TOKEN_BRACE_LEFT_ARGUMENT)) {
15859 found |= true;
15860 block = parse_block(parser, (uint16_t) (depth + 1));
15861 pm_arguments_validate_block(parser, arguments, block);
15862 } else if (pm_accepts_block_stack_p(parser) && accept1(parser, PM_TOKEN_KEYWORD_DO)) {
15863 found |= true;
15864 block = parse_block(parser, (uint16_t) (depth + 1));
15865 } else if (parsed_command_args && pm_accepts_block_stack_p(parser) && (flags & PM_PARSE_ACCEPTS_DO_BLOCK) && accept1(parser, PM_TOKEN_KEYWORD_DO_BLOCK)) {
15866 found |= true;
15867 block = parse_block(parser, (uint16_t) (depth + 1));
15868 }
15869
15870 if (block != NULL) {
15871 if (arguments->block == NULL && !arguments->has_forwarding) {
15872 arguments->block = UP(block);
15873 } else {
15874 pm_parser_err_node(parser, UP(block), PM_ERR_ARGUMENT_BLOCK_MULTI);
15875
15876 if (arguments->block != NULL) {
15877 parse_arguments_append(parser, arguments, arguments->block);
15878 }
15879 arguments->block = UP(block);
15880 }
15881 }
15882 }
15883
15884 return found;
15885}
15886
15891static void
15892ensure_valid_return(pm_parser_t *parser, pm_node_t *node) {
15893 bool in_sclass = false;
15894 for (pm_context_node_t *context_node = parser->current_context; context_node != NULL; context_node = context_node->prev) {
15895 switch (context_node->context) {
15896 case PM_CONTEXT_BEGIN_ELSE:
15897 case PM_CONTEXT_BEGIN_ENSURE:
15898 case PM_CONTEXT_BEGIN_RESCUE:
15899 case PM_CONTEXT_BEGIN:
15900 case PM_CONTEXT_CASE_IN:
15901 case PM_CONTEXT_CASE_WHEN:
15902 case PM_CONTEXT_DEFAULT_PARAMS:
15903 case PM_CONTEXT_DEFINED:
15904 case PM_CONTEXT_ELSE:
15905 case PM_CONTEXT_ELSIF:
15906 case PM_CONTEXT_EMBEXPR:
15907 case PM_CONTEXT_FOR_INDEX:
15908 case PM_CONTEXT_FOR:
15909 case PM_CONTEXT_IF:
15910 case PM_CONTEXT_LOOP_PREDICATE:
15911 case PM_CONTEXT_MAIN:
15912 case PM_CONTEXT_MULTI_TARGET:
15913 case PM_CONTEXT_PARENS:
15914 case PM_CONTEXT_POSTEXE:
15915 case PM_CONTEXT_PREDICATE:
15916 case PM_CONTEXT_PREEXE:
15917 case PM_CONTEXT_RESCUE_MODIFIER:
15918 case PM_CONTEXT_TERNARY:
15919 case PM_CONTEXT_UNLESS:
15920 case PM_CONTEXT_UNTIL:
15921 case PM_CONTEXT_WHILE:
15922 // Keep iterating up the lists of contexts, because returns can
15923 // see through these.
15924 continue;
15925 case PM_CONTEXT_SCLASS_ELSE:
15926 case PM_CONTEXT_SCLASS_ENSURE:
15927 case PM_CONTEXT_SCLASS_RESCUE:
15928 case PM_CONTEXT_SCLASS:
15929 in_sclass = true;
15930 continue;
15931 case PM_CONTEXT_CLASS_ELSE:
15932 case PM_CONTEXT_CLASS_ENSURE:
15933 case PM_CONTEXT_CLASS_RESCUE:
15934 case PM_CONTEXT_CLASS:
15935 case PM_CONTEXT_MODULE_ELSE:
15936 case PM_CONTEXT_MODULE_ENSURE:
15937 case PM_CONTEXT_MODULE_RESCUE:
15938 case PM_CONTEXT_MODULE:
15939 // These contexts are invalid for a return.
15940 pm_parser_err_node(parser, node, PM_ERR_RETURN_INVALID);
15941 return;
15942 case PM_CONTEXT_BLOCK_BRACES:
15943 case PM_CONTEXT_BLOCK_ELSE:
15944 case PM_CONTEXT_BLOCK_ENSURE:
15945 case PM_CONTEXT_BLOCK_KEYWORDS:
15946 case PM_CONTEXT_BLOCK_RESCUE:
15947 case PM_CONTEXT_BLOCK_PARAMETERS:
15948 case PM_CONTEXT_DEF_ELSE:
15949 case PM_CONTEXT_DEF_ENSURE:
15950 case PM_CONTEXT_DEF_PARAMS:
15951 case PM_CONTEXT_DEF_RESCUE:
15952 case PM_CONTEXT_DEF:
15953 case PM_CONTEXT_LAMBDA_BRACES:
15954 case PM_CONTEXT_LAMBDA_DO_END:
15955 case PM_CONTEXT_LAMBDA_ELSE:
15956 case PM_CONTEXT_LAMBDA_ENSURE:
15957 case PM_CONTEXT_LAMBDA_RESCUE:
15958 // These contexts are valid for a return, and we should not
15959 // continue to loop.
15960 return;
15961 case PM_CONTEXT_NONE:
15962 case PM_CONTEXT_MAXIMUM:
15963 // This case should never happen.
15964 PRISM_ASSERT(false && "unreachable");
15965 break;
15966 }
15967 }
15968 if (in_sclass && parser->version >= PM_OPTIONS_VERSION_CRUBY_3_4) {
15969 pm_parser_err_node(parser, node, PM_ERR_RETURN_INVALID);
15970 }
15971}
15972
15977static void
15978ensure_valid_jump(pm_parser_t *parser, pm_node_t *node) {
15979 for (pm_context_node_t *context_node = parser->current_context; context_node != NULL; context_node = context_node->prev) {
15980 switch (context_node->context) {
15981 case PM_CONTEXT_BLOCK_BRACES:
15982 case PM_CONTEXT_BLOCK_KEYWORDS:
15983 case PM_CONTEXT_BLOCK_ELSE:
15984 case PM_CONTEXT_BLOCK_ENSURE:
15985 case PM_CONTEXT_BLOCK_PARAMETERS:
15986 case PM_CONTEXT_BLOCK_RESCUE:
15987 case PM_CONTEXT_DEFINED:
15988 case PM_CONTEXT_FOR:
15989 case PM_CONTEXT_LAMBDA_BRACES:
15990 case PM_CONTEXT_LAMBDA_DO_END:
15991 case PM_CONTEXT_LAMBDA_ELSE:
15992 case PM_CONTEXT_LAMBDA_ENSURE:
15993 case PM_CONTEXT_LAMBDA_RESCUE:
15994 case PM_CONTEXT_LOOP_PREDICATE:
15995 case PM_CONTEXT_UNTIL:
15996 case PM_CONTEXT_WHILE:
15997 // These are the good cases. We're allowed to have a block exit
15998 // in these contexts.
15999 return;
16000 case PM_CONTEXT_POSTEXE:
16001 // https://bugs.ruby-lang.org/issues/20409
16002 if (context_node->context == PM_CONTEXT_POSTEXE) {
16003 if (parser->version < PM_OPTIONS_VERSION_CRUBY_4_1) {
16004 return;
16005 }
16006 }
16008 case PM_CONTEXT_DEF:
16009 case PM_CONTEXT_DEF_PARAMS:
16010 case PM_CONTEXT_DEF_ELSE:
16011 case PM_CONTEXT_DEF_ENSURE:
16012 case PM_CONTEXT_DEF_RESCUE:
16013 case PM_CONTEXT_MAIN:
16014 case PM_CONTEXT_PREEXE:
16015 case PM_CONTEXT_SCLASS:
16016 case PM_CONTEXT_SCLASS_ELSE:
16017 case PM_CONTEXT_SCLASS_ENSURE:
16018 case PM_CONTEXT_SCLASS_RESCUE:
16019 // These are the bad cases. We're not allowed to have a block
16020 // exit in these contexts.
16021 //
16022 // If we get here, then we're about to mark this block exit
16023 // as invalid. However, it could later _become_ valid if we
16024 // find a trailing while/until on the expression. In this
16025 // case instead of adding the error here, we'll add the
16026 // block exit to the list of exits for the expression, and
16027 // the node parsing will handle validating it instead.
16028 PRISM_ASSERT(parser->current_block_exits != NULL);
16029 pm_scratch_node_list_append(&parser->scratch_arena, parser->current_block_exits, node);
16030 return;
16031 case PM_CONTEXT_BEGIN_ELSE:
16032 case PM_CONTEXT_BEGIN_ENSURE:
16033 case PM_CONTEXT_BEGIN_RESCUE:
16034 case PM_CONTEXT_BEGIN:
16035 case PM_CONTEXT_CASE_IN:
16036 case PM_CONTEXT_CASE_WHEN:
16037 case PM_CONTEXT_CLASS_ELSE:
16038 case PM_CONTEXT_CLASS_ENSURE:
16039 case PM_CONTEXT_CLASS_RESCUE:
16040 case PM_CONTEXT_CLASS:
16041 case PM_CONTEXT_DEFAULT_PARAMS:
16042 case PM_CONTEXT_ELSE:
16043 case PM_CONTEXT_ELSIF:
16044 case PM_CONTEXT_EMBEXPR:
16045 case PM_CONTEXT_FOR_INDEX:
16046 case PM_CONTEXT_IF:
16047 case PM_CONTEXT_MODULE_ELSE:
16048 case PM_CONTEXT_MODULE_ENSURE:
16049 case PM_CONTEXT_MODULE_RESCUE:
16050 case PM_CONTEXT_MODULE:
16051 case PM_CONTEXT_MULTI_TARGET:
16052 case PM_CONTEXT_PARENS:
16053 case PM_CONTEXT_PREDICATE:
16054 case PM_CONTEXT_RESCUE_MODIFIER:
16055 case PM_CONTEXT_TERNARY:
16056 case PM_CONTEXT_UNLESS:
16057 // In these contexts we should continue walking up the list of
16058 // contexts.
16059 break;
16060 case PM_CONTEXT_NONE:
16061 case PM_CONTEXT_MAXIMUM:
16062 // This case should never happen.
16063 PRISM_ASSERT(false && "unreachable");
16064 break;
16065 }
16066 }
16067}
16068
16074push_block_exits(pm_parser_t *parser, pm_scratch_node_list_t *current_block_exits) {
16075 pm_scratch_node_list_t *previous_block_exits = parser->current_block_exits;
16076 parser->current_block_exits = current_block_exits;
16077 return previous_block_exits;
16078}
16079
16085static void
16086flush_block_exits(pm_parser_t *parser, pm_scratch_node_list_t *previous_block_exits) {
16087 pm_node_t *block_exit;
16088 PM_NODE_LIST_FOREACH(parser->current_block_exits, index, block_exit) {
16089 const char *type;
16090
16091 switch (PM_NODE_TYPE(block_exit)) {
16092 case PM_BREAK_NODE: type = "break"; break;
16093 case PM_NEXT_NODE: type = "next"; break;
16094 case PM_REDO_NODE: type = "redo"; break;
16095 default: PRISM_ASSERT(false && "unreachable"); type = ""; break;
16096 }
16097
16098 PM_PARSER_ERR_NODE_FORMAT(parser, block_exit, PM_ERR_INVALID_BLOCK_EXIT, type);
16099 }
16100
16101 parser->current_block_exits = previous_block_exits;
16102}
16103
16108static void
16109pop_block_exits(pm_parser_t *parser, pm_scratch_node_list_t *previous_block_exits) {
16110 if (match2(parser, PM_TOKEN_KEYWORD_WHILE_MODIFIER, PM_TOKEN_KEYWORD_UNTIL_MODIFIER)) {
16111 // If we matched a trailing while/until, then all of the block exits in
16112 // the contained list are valid. In this case we do not need to do
16113 // anything.
16114 parser->current_block_exits = previous_block_exits;
16115 } else if (previous_block_exits != NULL) {
16116 // If we did not matching a trailing while/until, then all of the block
16117 // exits contained in the list are invalid for this specific context.
16118 // However, they could still become valid in a higher level context if
16119 // there is another list above this one. In this case we'll push all of
16120 // the block exits up to the previous list.
16121 pm_scratch_node_list_concat(&parser->scratch_arena, previous_block_exits, parser->current_block_exits);
16122 parser->current_block_exits = previous_block_exits;
16123 } else {
16124 // If we did not match a trailing while/until and this was the last
16125 // chance to do so, then all of the block exits in the list are invalid
16126 // and we need to add an error for each of them.
16127 flush_block_exits(parser, previous_block_exits);
16128 }
16129}
16130
16131static PRISM_INLINE pm_node_t *
16132parse_predicate(pm_parser_t *parser, pm_binding_power_t binding_power, pm_context_t context, pm_token_t *then_keyword, uint16_t depth) {
16133 context_push(parser, PM_CONTEXT_PREDICATE);
16134 pm_diagnostic_id_t error_id = context == PM_CONTEXT_IF ? PM_ERR_CONDITIONAL_IF_PREDICATE : PM_ERR_CONDITIONAL_UNLESS_PREDICATE;
16135 pm_node_t *predicate = parse_value_expression(parser, binding_power, PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_DO_BLOCK, error_id, (uint16_t) (depth + 1));
16136
16137 // Predicates are closed by a term, a "then", or a term and then a "then".
16138 bool predicate_closed = accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
16139
16140 if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) {
16141 predicate_closed = true;
16142 *then_keyword = parser->previous;
16143 }
16144
16145 if (!predicate_closed) {
16146 pm_parser_err_current(parser, PM_ERR_CONDITIONAL_PREDICATE_TERM);
16147 }
16148
16149 context_pop(parser);
16150 return predicate;
16151}
16152
16153static PRISM_INLINE pm_node_t *
16154parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newline_index, bool if_after_else, uint16_t depth) {
16155 pm_scratch_node_list_t current_block_exits = { 0 };
16156 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
16157
16158 pm_token_t keyword = parser->previous;
16159 pm_token_t then_keyword = { 0 };
16160
16161 pm_node_t *predicate = parse_predicate(parser, PM_BINDING_POWER_COMPOSITION, context, &then_keyword, (uint16_t) (depth + 1));
16162 pm_statements_node_t *statements = NULL;
16163
16164 if (!match3(parser, PM_TOKEN_KEYWORD_ELSIF, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
16165 pm_accepts_block_stack_push(parser, true);
16166 statements = parse_statements(parser, context, (uint16_t) (depth + 1));
16167 pm_accepts_block_stack_pop(parser);
16168 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
16169 }
16170
16171 pm_node_t *parent = NULL;
16172
16173 switch (context) {
16174 case PM_CONTEXT_IF:
16175 parent = UP(pm_if_node_create(parser, &keyword, predicate, NTOK2PTR(then_keyword), statements, NULL, NULL));
16176 break;
16177 case PM_CONTEXT_UNLESS:
16178 parent = UP(pm_unless_node_create(parser, &keyword, predicate, NTOK2PTR(then_keyword), statements));
16179 break;
16180 default:
16181 PRISM_ASSERT(false && "unreachable");
16182 break;
16183 }
16184
16185 pm_node_t *current = parent;
16186
16187 // Parse any number of elsif clauses. This will form a linked list of if
16188 // nodes pointing to each other from the top.
16189 if (context == PM_CONTEXT_IF) {
16190 while (match1(parser, PM_TOKEN_KEYWORD_ELSIF)) {
16191 if (parser_end_of_line_p(parser)) {
16192 PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_KEYWORD_EOL);
16193 }
16194
16195 parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false);
16196 pm_token_t elsif_keyword = parser->current;
16197 parser_lex(parser);
16198
16199 pm_node_t *predicate = parse_predicate(parser, PM_BINDING_POWER_COMPOSITION, PM_CONTEXT_ELSIF, &then_keyword, (uint16_t) (depth + 1));
16200 pm_accepts_block_stack_push(parser, true);
16201
16202 pm_statements_node_t *statements = parse_statements(parser, PM_CONTEXT_ELSIF, (uint16_t) (depth + 1));
16203 pm_accepts_block_stack_pop(parser);
16204 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
16205
16206 pm_node_t *elsif = UP(pm_if_node_create(parser, &elsif_keyword, predicate, NTOK2PTR(then_keyword), statements, NULL, NULL));
16207 ((pm_if_node_t *) current)->subsequent = elsif;
16208 current = elsif;
16209 }
16210 }
16211
16212 if (match1(parser, PM_TOKEN_KEYWORD_ELSE)) {
16213 parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false);
16214 opening_newline_index = token_newline_index(parser);
16215
16216 parser_lex(parser);
16217 pm_token_t else_keyword = parser->previous;
16218
16219 pm_accepts_block_stack_push(parser, true);
16220 pm_statements_node_t *else_statements = parse_statements(parser, PM_CONTEXT_ELSE, (uint16_t) (depth + 1));
16221 pm_accepts_block_stack_pop(parser);
16222
16223 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
16224 parser_warn_indentation_mismatch(parser, opening_newline_index, &else_keyword, false, false);
16225 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CONDITIONAL_TERM_ELSE, &keyword);
16226
16227 pm_else_node_t *else_node = pm_else_node_create(parser, &else_keyword, else_statements, &parser->previous);
16228
16229 switch (context) {
16230 case PM_CONTEXT_IF:
16231 ((pm_if_node_t *) current)->subsequent = UP(else_node);
16232 break;
16233 case PM_CONTEXT_UNLESS:
16234 ((pm_unless_node_t *) parent)->else_clause = else_node;
16235 break;
16236 default:
16237 PRISM_ASSERT(false && "unreachable");
16238 break;
16239 }
16240 } else {
16241 parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, if_after_else, false);
16242 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CONDITIONAL_TERM, &keyword);
16243 }
16244
16245 // Set the appropriate end location for all of the nodes in the subtree.
16246 switch (context) {
16247 case PM_CONTEXT_IF: {
16248 pm_node_t *current = parent;
16249 bool recursing = true;
16250
16251 while (recursing) {
16252 switch (PM_NODE_TYPE(current)) {
16253 case PM_IF_NODE:
16254 pm_if_node_end_keyword_loc_set(parser, (pm_if_node_t *) current, &parser->previous);
16255 current = ((pm_if_node_t *) current)->subsequent;
16256 recursing = current != NULL;
16257 break;
16258 case PM_ELSE_NODE:
16259 pm_else_node_end_keyword_loc_set(parser, (pm_else_node_t *) current, &parser->previous);
16260 recursing = false;
16261 break;
16262 default: {
16263 recursing = false;
16264 break;
16265 }
16266 }
16267 }
16268 break;
16269 }
16270 case PM_CONTEXT_UNLESS:
16271 pm_unless_node_end_keyword_loc_set(parser, (pm_unless_node_t *) parent, &parser->previous);
16272 break;
16273 default:
16274 PRISM_ASSERT(false && "unreachable");
16275 break;
16276 }
16277
16278 pop_block_exits(parser, previous_block_exits);
16279 return parent;
16280}
16281
16286#define PM_CASE_KEYWORD PM_TOKEN_KEYWORD___ENCODING__: case PM_TOKEN_KEYWORD___FILE__: case PM_TOKEN_KEYWORD___LINE__: \
16287 case PM_TOKEN_KEYWORD_ALIAS: case PM_TOKEN_KEYWORD_AND: case PM_TOKEN_KEYWORD_BEGIN: case PM_TOKEN_KEYWORD_BEGIN_UPCASE: \
16288 case PM_TOKEN_KEYWORD_BREAK: case PM_TOKEN_KEYWORD_CASE: case PM_TOKEN_KEYWORD_CLASS: case PM_TOKEN_KEYWORD_DEF: \
16289 case PM_TOKEN_KEYWORD_DEFINED: case PM_TOKEN_KEYWORD_DO: case PM_TOKEN_KEYWORD_DO_BLOCK: case PM_TOKEN_KEYWORD_DO_LAMBDA: case PM_TOKEN_KEYWORD_DO_LOOP: case PM_TOKEN_KEYWORD_ELSE: \
16290 case PM_TOKEN_KEYWORD_ELSIF: case PM_TOKEN_KEYWORD_END: case PM_TOKEN_KEYWORD_END_UPCASE: case PM_TOKEN_KEYWORD_ENSURE: \
16291 case PM_TOKEN_KEYWORD_FALSE: case PM_TOKEN_KEYWORD_FOR: case PM_TOKEN_KEYWORD_IF: case PM_TOKEN_KEYWORD_IN: \
16292 case PM_TOKEN_KEYWORD_MODULE: case PM_TOKEN_KEYWORD_NEXT: case PM_TOKEN_KEYWORD_NIL: case PM_TOKEN_KEYWORD_NOT: \
16293 case PM_TOKEN_KEYWORD_OR: case PM_TOKEN_KEYWORD_REDO: case PM_TOKEN_KEYWORD_RESCUE: case PM_TOKEN_KEYWORD_RETRY: \
16294 case PM_TOKEN_KEYWORD_RETURN: case PM_TOKEN_KEYWORD_SELF: case PM_TOKEN_KEYWORD_SUPER: case PM_TOKEN_KEYWORD_THEN: \
16295 case PM_TOKEN_KEYWORD_TRUE: case PM_TOKEN_KEYWORD_UNDEF: case PM_TOKEN_KEYWORD_UNLESS: case PM_TOKEN_KEYWORD_UNTIL: \
16296 case PM_TOKEN_KEYWORD_WHEN: case PM_TOKEN_KEYWORD_WHILE: case PM_TOKEN_KEYWORD_YIELD
16297
16302#define PM_CASE_OPERATOR PM_TOKEN_AMPERSAND: case PM_TOKEN_BACKTICK: case PM_TOKEN_BANG_EQUAL: \
16303 case PM_TOKEN_BANG_TILDE: case PM_TOKEN_BANG: case PM_TOKEN_BRACKET_LEFT_RIGHT_EQUAL: \
16304 case PM_TOKEN_BRACKET_LEFT_RIGHT: case PM_TOKEN_CARET: case PM_TOKEN_EQUAL_EQUAL_EQUAL: case PM_TOKEN_EQUAL_EQUAL: \
16305 case PM_TOKEN_EQUAL_TILDE: case PM_TOKEN_GREATER_EQUAL: case PM_TOKEN_GREATER_GREATER: case PM_TOKEN_GREATER: \
16306 case PM_TOKEN_LESS_EQUAL_GREATER: case PM_TOKEN_LESS_EQUAL: case PM_TOKEN_LESS_LESS: case PM_TOKEN_LESS: \
16307 case PM_TOKEN_MINUS: case PM_TOKEN_PERCENT: case PM_TOKEN_PIPE: case PM_TOKEN_PLUS: case PM_TOKEN_SLASH: \
16308 case PM_TOKEN_STAR_STAR: case PM_TOKEN_STAR: case PM_TOKEN_TILDE: case PM_TOKEN_UAMPERSAND: case PM_TOKEN_UMINUS: \
16309 case PM_TOKEN_UMINUS_NUM: case PM_TOKEN_UPLUS: case PM_TOKEN_USTAR: case PM_TOKEN_USTAR_STAR
16310
16316#define PM_CASE_PRIMITIVE PM_TOKEN_INTEGER: case PM_TOKEN_INTEGER_IMAGINARY: case PM_TOKEN_INTEGER_RATIONAL: \
16317 case PM_TOKEN_INTEGER_RATIONAL_IMAGINARY: case PM_TOKEN_FLOAT: case PM_TOKEN_FLOAT_IMAGINARY: \
16318 case PM_TOKEN_FLOAT_RATIONAL: case PM_TOKEN_FLOAT_RATIONAL_IMAGINARY: case PM_TOKEN_SYMBOL_BEGIN: \
16319 case PM_TOKEN_REGEXP_BEGIN: case PM_TOKEN_XSTRING_BEGIN: case PM_TOKEN_PERCENT_LOWER_X: case PM_TOKEN_PERCENT_LOWER_I: \
16320 case PM_TOKEN_PERCENT_LOWER_W: case PM_TOKEN_PERCENT_UPPER_I: case PM_TOKEN_PERCENT_UPPER_W: \
16321 case PM_TOKEN_STRING_BEGIN: case PM_TOKEN_KEYWORD_NIL: case PM_TOKEN_KEYWORD_SELF: case PM_TOKEN_KEYWORD_TRUE: \
16322 case PM_TOKEN_KEYWORD_FALSE: case PM_TOKEN_KEYWORD___FILE__: case PM_TOKEN_KEYWORD___LINE__: \
16323 case PM_TOKEN_KEYWORD___ENCODING__: case PM_TOKEN_MINUS_GREATER: case PM_TOKEN_HEREDOC_START: \
16324 case PM_TOKEN_UMINUS_NUM: case PM_TOKEN_CHARACTER_LITERAL
16325
16330#define PM_CASE_PARAMETER PM_TOKEN_UAMPERSAND: case PM_TOKEN_AMPERSAND: case PM_TOKEN_UDOT_DOT_DOT: \
16331 case PM_TOKEN_IDENTIFIER: case PM_TOKEN_LABEL: case PM_TOKEN_USTAR: case PM_TOKEN_STAR: case PM_TOKEN_STAR_STAR: \
16332 case PM_TOKEN_USTAR_STAR: case PM_TOKEN_CONSTANT: case PM_TOKEN_INSTANCE_VARIABLE: case PM_TOKEN_GLOBAL_VARIABLE: \
16333 case PM_TOKEN_CLASS_VARIABLE
16334
16339#define PM_CASE_WRITABLE PM_CLASS_VARIABLE_READ_NODE: case PM_CONSTANT_PATH_NODE: \
16340 case PM_CONSTANT_READ_NODE: case PM_GLOBAL_VARIABLE_READ_NODE: case PM_LOCAL_VARIABLE_READ_NODE: \
16341 case PM_INSTANCE_VARIABLE_READ_NODE: case PM_MULTI_TARGET_NODE: case PM_BACK_REFERENCE_READ_NODE: \
16342 case PM_NUMBERED_REFERENCE_READ_NODE: case PM_IT_LOCAL_VARIABLE_READ_NODE
16343
16344// Assert here that the flags are the same so that we can safely switch the type
16345// of the node without having to move the flags.
16346PM_STATIC_ASSERT(__LINE__, ((int) PM_STRING_FLAGS_FORCED_UTF8_ENCODING) == ((int) PM_ENCODING_FLAGS_FORCED_UTF8_ENCODING), "Expected the flags to match.");
16347
16352static PRISM_INLINE pm_node_flags_t
16353parse_unescaped_encoding(const pm_parser_t *parser, const pm_encoding_t *explicit_encoding) {
16354 if (explicit_encoding != NULL) {
16355 if (explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
16356 // If the there's an explicit encoding and it's using a UTF-8 escape
16357 // sequence, then mark the string as UTF-8.
16358 return PM_STRING_FLAGS_FORCED_UTF8_ENCODING;
16359 } else if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
16360 // If there's a non-UTF-8 escape sequence being used, then the
16361 // string uses the source encoding, unless the source is marked as
16362 // US-ASCII. In that case the string is forced as ASCII-8BIT in
16363 // order to keep the string valid.
16364 return PM_STRING_FLAGS_FORCED_BINARY_ENCODING;
16365 }
16366 }
16367 return 0;
16368}
16369
16374static pm_node_t *
16375parse_string_part(pm_parser_t *parser, uint16_t depth) {
16376 switch (parser->current.type) {
16377 // Here the lexer has returned to us plain string content. In this case
16378 // we'll create a string node that has no opening or closing and return that
16379 // as the part. These kinds of parts look like:
16380 //
16381 // "aaa #{bbb} #@ccc ddd"
16382 // ^^^^ ^ ^^^^
16383 case PM_TOKEN_STRING_CONTENT: {
16384 pm_node_t *node = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
16385 pm_node_flag_set(node, parse_unescaped_encoding(parser, parser->explicit_encoding));
16386
16387 parser_lex(parser);
16388 return node;
16389 }
16390 // Here the lexer has returned the beginning of an embedded expression. In
16391 // that case we'll parse the inner statements and return that as the part.
16392 // These kinds of parts look like:
16393 //
16394 // "aaa #{bbb} #@ccc ddd"
16395 // ^^^^^^
16396 case PM_TOKEN_EMBEXPR_BEGIN: {
16397 // Ruby disallows seeing encoding around interpolation in strings,
16398 // even though it is known at parse time.
16399 parser->explicit_encoding = NULL;
16400
16401 pm_lex_state_t state = parser->lex_state;
16402 int brace_nesting = parser->brace_nesting;
16403
16404 parser->brace_nesting = 0;
16405 lex_state_set(parser, PM_LEX_STATE_BEG);
16406 parser_lex(parser);
16407
16408 pm_token_t opening = parser->previous;
16409 pm_statements_node_t *statements = NULL;
16410
16411 if (!match3(parser, PM_TOKEN_EMBEXPR_END, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) {
16412 statements = parse_statements(parser, PM_CONTEXT_EMBEXPR, (uint16_t) (depth + 1));
16413 }
16414
16415 parser->brace_nesting = brace_nesting;
16416 lex_state_set(parser, state);
16417 expect1(parser, PM_TOKEN_EMBEXPR_END, PM_ERR_EMBEXPR_END);
16418
16419 // If this set of embedded statements only contains a single
16420 // statement, then Ruby does not consider it as a possible statement
16421 // that could emit a line event.
16422 if (statements != NULL && statements->body.size == 1) {
16423 pm_node_flag_unset(statements->body.nodes[0], PM_NODE_FLAG_NEWLINE);
16424 }
16425
16426 return UP(pm_embedded_statements_node_create(parser, &opening, statements, &parser->previous));
16427 }
16428
16429 // Here the lexer has returned the beginning of an embedded variable.
16430 // In that case we'll parse the variable and create an appropriate node
16431 // for it and then return that node. These kinds of parts look like:
16432 //
16433 // "aaa #{bbb} #@ccc ddd"
16434 // ^^^^^
16435 case PM_TOKEN_EMBVAR: {
16436 // Ruby disallows seeing encoding around interpolation in strings,
16437 // even though it is known at parse time.
16438 parser->explicit_encoding = NULL;
16439
16440 lex_state_set(parser, PM_LEX_STATE_BEG);
16441 parser_lex(parser);
16442
16443 pm_token_t operator = parser->previous;
16444 pm_node_t *variable;
16445
16446 switch (parser->current.type) {
16447 // In this case a back reference is being interpolated. We'll
16448 // create a global variable read node.
16449 case PM_TOKEN_BACK_REFERENCE:
16450 parser_lex(parser);
16451 variable = UP(pm_back_reference_read_node_create(parser, &parser->previous));
16452 break;
16453 // In this case an nth reference is being interpolated. We'll
16454 // create a global variable read node.
16455 case PM_TOKEN_NUMBERED_REFERENCE:
16456 parser_lex(parser);
16457 variable = UP(pm_numbered_reference_read_node_create(parser, &parser->previous));
16458 break;
16459 // In this case a global variable is being interpolated. We'll
16460 // create a global variable read node.
16461 case PM_TOKEN_GLOBAL_VARIABLE:
16462 parser_lex(parser);
16463 variable = UP(pm_global_variable_read_node_create(parser, &parser->previous));
16464 break;
16465 // In this case an instance variable is being interpolated.
16466 // We'll create an instance variable read node.
16467 case PM_TOKEN_INSTANCE_VARIABLE:
16468 parser_lex(parser);
16469 variable = UP(pm_instance_variable_read_node_create(parser, &parser->previous));
16470 break;
16471 // In this case a class variable is being interpolated. We'll
16472 // create a class variable read node.
16473 case PM_TOKEN_CLASS_VARIABLE:
16474 parser_lex(parser);
16475 variable = UP(pm_class_variable_read_node_create(parser, &parser->previous));
16476 break;
16477 // We can hit here if we got an invalid token. In that case
16478 // we'll not attempt to lex this token and instead just return a
16479 // missing node.
16480 default:
16481 expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_EMBVAR_INVALID);
16482 variable = UP(pm_error_recovery_node_create(parser, PM_TOKEN_END(parser, &parser->previous), 0));
16483 break;
16484 }
16485
16486 return UP(pm_embedded_variable_node_create(parser, &operator, variable));
16487 }
16488 default:
16489 parse_discard_token(parser, depth);
16490 pm_parser_err_previous(parser, PM_ERR_CANNOT_PARSE_STRING_PART);
16491 return NULL;
16492 }
16493}
16494
16500static const uint8_t *
16501parse_operator_symbol_name(const pm_token_t *name) {
16502 switch (name->type) {
16503 case PM_TOKEN_TILDE:
16504 case PM_TOKEN_BANG:
16505 if (name->end[-1] == '@') return name->end - 1;
16507 default:
16508 return name->end;
16509 }
16510}
16511
16512static pm_node_t *
16513parse_operator_symbol(pm_parser_t *parser, const pm_token_t *opening, pm_lex_state_t next_state) {
16514 pm_symbol_node_t *symbol = pm_symbol_node_create(parser, opening, &parser->current, NULL);
16515 const uint8_t *end = parse_operator_symbol_name(&parser->current);
16516
16517 if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state);
16518 parser_lex(parser);
16519
16520 pm_string_shared_init(&symbol->unescaped, parser->previous.start, end);
16521 pm_node_flag_set(UP(symbol), PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING);
16522
16523 return UP(symbol);
16524}
16525
16531static pm_node_t *
16532parse_symbol(pm_parser_t *parser, pm_lex_state_t next_state, uint16_t depth) {
16533 pm_lex_mode_t lex_mode = *parser->lex_modes.current;
16534 parser_lex(parser);
16535
16536 const pm_token_t opening = parser->previous;
16537
16538 if (lex_mode.mode != PM_LEX_STRING) {
16539 if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state);
16540
16541 switch (parser->current.type) {
16542 case PM_CASE_OPERATOR:
16543 return parse_operator_symbol(parser, &opening, next_state == PM_LEX_STATE_NONE ? PM_LEX_STATE_ENDFN : next_state);
16544 case PM_TOKEN_IDENTIFIER:
16545 case PM_TOKEN_CONSTANT:
16546 case PM_TOKEN_INSTANCE_VARIABLE:
16547 case PM_TOKEN_METHOD_NAME:
16548 case PM_TOKEN_CLASS_VARIABLE:
16549 case PM_TOKEN_GLOBAL_VARIABLE:
16550 case PM_TOKEN_NUMBERED_REFERENCE:
16551 case PM_TOKEN_BACK_REFERENCE:
16552 case PM_CASE_KEYWORD:
16553 parser_lex(parser);
16554 break;
16555 default:
16556 expect2(parser, PM_TOKEN_IDENTIFIER, PM_TOKEN_METHOD_NAME, PM_ERR_SYMBOL_INVALID);
16557 break;
16558 }
16559
16560 pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, NULL);
16561 pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end);
16562 pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, parser->explicit_encoding, &parser->previous, &symbol->unescaped, false));
16563
16564 return UP(symbol);
16565 }
16566
16567 if (lex_mode.as.string.interpolation) {
16568 // If we have the end of the symbol, then we can return an empty symbol.
16569 if (match1(parser, PM_TOKEN_STRING_END)) {
16570 if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state);
16571 parser_lex(parser);
16572 pm_token_t content = {
16573 .type = PM_TOKEN_STRING_CONTENT,
16574 .start = parser->previous.start,
16575 .end = parser->previous.start
16576 };
16577
16578 return UP(pm_symbol_node_create(parser, &opening, &content, &parser->previous));
16579 }
16580
16581 // Now we can parse the first part of the symbol.
16582 pm_node_t *part = parse_string_part(parser, (uint16_t) (depth + 1));
16583
16584 // If we got a string part, then it's possible that we could transform
16585 // what looks like an interpolated symbol into a regular symbol.
16586 if (part && PM_NODE_TYPE_P(part, PM_STRING_NODE) && match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
16587 if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state);
16588 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_INTERPOLATED);
16589
16590 return UP(pm_string_node_to_symbol_node(parser, (pm_string_node_t *) part, &opening, &parser->previous));
16591 }
16592
16593 pm_node_list_t parts = { 0 };
16594 if (part) pm_node_list_open_append(&parser->node_chunks, &parts, part);
16595
16596 while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
16597 if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) != NULL) {
16598 pm_node_list_open_append(&parser->node_chunks, &parts, part);
16599 }
16600 }
16601
16602 if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state);
16603 if (match1(parser, PM_TOKEN_EOF)) {
16604 pm_parser_err_token(parser, &opening, PM_ERR_SYMBOL_TERM_INTERPOLATED);
16605 return UP(pm_interpolated_symbol_node_create(parser, &opening, &parts, NULL));
16606 }
16607
16608 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_INTERPOLATED);
16609 return UP(pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous));
16610 }
16611
16612 pm_token_t content;
16613 pm_string_t unescaped;
16614
16615 if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
16616 content = parser->current;
16617 unescaped = parser->current_string;
16618 parser_lex(parser);
16619
16620 // If we have two string contents in a row, then the content of this
16621 // symbol is split because of heredoc contents. This looks like:
16622 //
16623 // <<A; :'a
16624 // A
16625 // b'
16626 //
16627 // In this case, the best way we have to represent this is as an
16628 // interpolated string node, so that's what we'll do here.
16629 if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
16630 pm_node_list_t parts = { 0 };
16631 pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &unescaped));
16632 pm_node_list_open_append(&parser->node_chunks, &parts, part);
16633
16634 part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->current, NULL, &parser->current_string));
16635 pm_node_list_open_append(&parser->node_chunks, &parts, part);
16636
16637 if (next_state != PM_LEX_STATE_NONE) {
16638 lex_state_set(parser, next_state);
16639 }
16640
16641 parser_lex(parser);
16642 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_DYNAMIC);
16643
16644 return UP(pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous));
16645 }
16646 } else {
16647 content = (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = parser->previous.end, .end = parser->previous.end };
16648 pm_string_shared_init(&unescaped, content.start, content.end);
16649 }
16650
16651 if (next_state != PM_LEX_STATE_NONE) {
16652 lex_state_set(parser, next_state);
16653 }
16654
16655 const pm_token_t *closing = NULL;
16656 if (match1(parser, PM_TOKEN_EOF)) {
16657 pm_parser_err_token(parser, &opening, PM_ERR_SYMBOL_TERM_DYNAMIC);
16658 } else {
16659 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_DYNAMIC);
16660 closing = &parser->previous;
16661 }
16662
16663 return UP(pm_symbol_node_create_unescaped(parser, &opening, &content, closing, &unescaped, parse_symbol_encoding(parser, parser->explicit_encoding, &content, &unescaped, false)));
16664}
16665
16670static PRISM_INLINE pm_node_t *
16671parse_undef_argument(pm_parser_t *parser, uint16_t depth) {
16672 switch (parser->current.type) {
16673 case PM_CASE_OPERATOR:
16674 return parse_operator_symbol(parser, NULL, PM_LEX_STATE_NONE);
16675 case PM_CASE_KEYWORD:
16676 case PM_TOKEN_CONSTANT:
16677 case PM_TOKEN_IDENTIFIER:
16678 case PM_TOKEN_METHOD_NAME: {
16679 parser_lex(parser);
16680
16681 pm_symbol_node_t *symbol = pm_symbol_node_create(parser, NULL, &parser->previous, NULL);
16682 pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end);
16683 pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, parser->explicit_encoding, &parser->previous, &symbol->unescaped, false));
16684
16685 return UP(symbol);
16686 }
16687 case PM_TOKEN_SYMBOL_BEGIN:
16688 return parse_symbol(parser, PM_LEX_STATE_NONE, (uint16_t) (depth + 1));
16689 default:
16690 pm_parser_err_current(parser, PM_ERR_UNDEF_ARGUMENT);
16691 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_END(parser, &parser->previous), 0));
16692 }
16693}
16694
16701static PRISM_INLINE pm_node_t *
16702parse_alias_argument(pm_parser_t *parser, bool first, uint16_t depth) {
16703 switch (parser->current.type) {
16704 case PM_CASE_OPERATOR:
16705 return parse_operator_symbol(parser, NULL, first ? PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM : PM_LEX_STATE_NONE);
16706 case PM_CASE_KEYWORD:
16707 case PM_TOKEN_CONSTANT:
16708 case PM_TOKEN_IDENTIFIER:
16709 case PM_TOKEN_METHOD_NAME: {
16710 if (first) lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM);
16711 parser_lex(parser);
16712
16713 pm_symbol_node_t *symbol = pm_symbol_node_create(parser, NULL, &parser->previous, NULL);
16714 pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end);
16715 pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, parser->explicit_encoding, &parser->previous, &symbol->unescaped, false));
16716
16717 return UP(symbol);
16718 }
16719 case PM_TOKEN_SYMBOL_BEGIN:
16720 return parse_symbol(parser, first ? PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM : PM_LEX_STATE_NONE, (uint16_t) (depth + 1));
16721 case PM_TOKEN_BACK_REFERENCE:
16722 parser_lex(parser);
16723 return UP(pm_back_reference_read_node_create(parser, &parser->previous));
16724 case PM_TOKEN_NUMBERED_REFERENCE:
16725 parser_lex(parser);
16726 return UP(pm_numbered_reference_read_node_create(parser, &parser->previous));
16727 case PM_TOKEN_GLOBAL_VARIABLE:
16728 parser_lex(parser);
16729 return UP(pm_global_variable_read_node_create(parser, &parser->previous));
16730 default:
16731 pm_parser_err_current(parser, PM_ERR_ALIAS_ARGUMENT);
16732 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_END(parser, &parser->previous), 0));
16733 }
16734}
16735
16740static pm_node_t *
16741parse_variable(pm_parser_t *parser) {
16742 pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &parser->previous);
16743 int depth;
16744 bool is_numbered_param = pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous));
16745
16746 if (!is_numbered_param && ((depth = pm_parser_local_depth_constant_id(parser, name_id)) != -1)) {
16747 return UP(pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, (uint32_t) depth, false));
16748 }
16749
16750 pm_scope_t *current_scope = parser->current_scope;
16751 if (!current_scope->closed && !(current_scope->parameters & PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED)) {
16752 if (is_numbered_param) {
16753 // When you use a numbered parameter, it implies the existence of
16754 // all of the locals that exist before it. For example, referencing
16755 // _2 means that _1 must exist. Therefore here we loop through all
16756 // of the possibilities and add them into the constant pool.
16757 uint8_t maximum = (uint8_t) (parser->previous.start[1] - '0');
16758 for (uint8_t number = 1; number <= maximum; number++) {
16759 pm_parser_local_add_constant(parser, pm_numbered_parameter_names[number - 1], 2);
16760 }
16761
16762 if (!match1(parser, PM_TOKEN_EQUAL)) {
16763 parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_FOUND;
16764 }
16765
16766 pm_node_t *node = UP(pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, 0, false));
16767 pm_scratch_node_list_append(&parser->scratch_arena, &current_scope->implicit_parameters, node);
16768
16769 return node;
16770 } else if ((parser->version >= PM_OPTIONS_VERSION_CRUBY_3_4) && pm_token_is_it(parser->previous.start, parser->previous.end)) {
16771 pm_node_t *node = UP(pm_it_local_variable_read_node_create(parser, &parser->previous));
16772 pm_scratch_node_list_append(&parser->scratch_arena, &current_scope->implicit_parameters, node);
16773
16774 return node;
16775 }
16776 }
16777
16778 return NULL;
16779}
16780
16784static pm_node_t *
16785parse_variable_call(pm_parser_t *parser) {
16786 pm_node_flags_t flags = 0;
16787
16788 if (!match1(parser, PM_TOKEN_PARENTHESIS_LEFT) && (parser->previous.end[-1] != '!') && (parser->previous.end[-1] != '?')) {
16789 pm_node_t *node = parse_variable(parser);
16790 if (node != NULL) return node;
16791 flags |= PM_CALL_NODE_FLAGS_VARIABLE_CALL;
16792 }
16793
16794 pm_call_node_t *node = pm_call_node_variable_call_create(parser, &parser->previous);
16795 pm_node_flag_set(UP(node), flags);
16796
16797 return UP(node);
16798}
16799
16806parse_method_definition_name(pm_parser_t *parser) {
16807 switch (parser->current.type) {
16808 case PM_CASE_KEYWORD:
16809 case PM_TOKEN_CONSTANT:
16810 case PM_TOKEN_METHOD_NAME:
16811 parser_lex(parser);
16812 return parser->previous;
16813 case PM_TOKEN_IDENTIFIER:
16814 pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current));
16815 parser_lex(parser);
16816 return parser->previous;
16817 case PM_CASE_OPERATOR:
16818 lex_state_set(parser, PM_LEX_STATE_ENDFN);
16819 parser_lex(parser);
16820 return parser->previous;
16821 default:
16822 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_DEF_NAME, pm_token_str(parser->current.type));
16823 return (pm_token_t) { .type = 0, .start = parser->current.start, .end = parser->current.end };
16824 }
16825}
16826
16827static void
16828parse_heredoc_dedent_string(pm_arena_t *arena, pm_string_t *string, size_t common_whitespace) {
16829 // Make a writable copy of the string in the arena, since strings only view
16830 // their bytes. We keep a mutable pointer to the arena memory so we can
16831 // memmove into it below without casting away const from the string's
16832 // source field.
16833 size_t length = pm_string_length(string);
16834 uint8_t *writable = (uint8_t *) pm_arena_memdup(arena, pm_string_source(string), length, PRISM_ALIGNOF(uint8_t));
16835 pm_string_init(string, writable, length);
16836
16837 // Now get the bounds of the existing string. We'll use this as a
16838 // destination to move bytes into. We'll also use it for bounds checking
16839 // since we don't require that these strings be null terminated.
16840 size_t dest_length = pm_string_length(string);
16841 const uint8_t *source_cursor = writable;
16842 const uint8_t *source_end = source_cursor + dest_length;
16843
16844 // We're going to move bytes backward in the string when we get leading
16845 // whitespace, so we'll maintain a pointer to the current position in the
16846 // string that we're writing to.
16847 size_t trimmed_whitespace = 0;
16848
16849 // While we haven't reached the amount of common whitespace that we need to
16850 // trim and we haven't reached the end of the string, we'll keep trimming
16851 // whitespace. Trimming in this context means skipping over these bytes such
16852 // that they aren't copied into the new string.
16853 while ((source_cursor < source_end) && pm_char_is_inline_whitespace(*source_cursor) && trimmed_whitespace < common_whitespace) {
16854 if (*source_cursor == '\t') {
16855 trimmed_whitespace = (trimmed_whitespace / PM_TAB_WHITESPACE_SIZE + 1) * PM_TAB_WHITESPACE_SIZE;
16856 if (trimmed_whitespace > common_whitespace) break;
16857 } else {
16858 trimmed_whitespace++;
16859 }
16860
16861 source_cursor++;
16862 dest_length--;
16863 }
16864
16865 memmove(writable, source_cursor, (size_t) (source_end - source_cursor));
16866 string->length = dest_length;
16867}
16868
16873static PRISM_INLINE bool
16874heredoc_dedent_discard_string_node(pm_parser_t *parser, pm_string_node_t *string_node) {
16875 if (string_node->unescaped.length == 0) {
16876 const uint8_t *cursor = parser->start + PM_LOCATION_START(&string_node->content_loc);
16877 return pm_memchr(cursor, '\\', string_node->content_loc.length, parser->encoding_changed, parser->encoding) == NULL;
16878 }
16879 return false;
16880}
16881
16885static void
16886parse_heredoc_dedent(pm_parser_t *parser, pm_node_list_t *nodes, size_t common_whitespace) {
16887 // The next node should be dedented if it's the first node in the list or if
16888 // it follows a string node.
16889 bool dedent_next = true;
16890
16891 // Iterate over all nodes, and trim whitespace accordingly. We're going to
16892 // keep around two indices: a read and a write.
16893 uint32_t write_index = 0;
16894
16895 pm_node_t *node;
16896 PM_NODE_LIST_FOREACH(nodes, read_index, node) {
16897 // We're not manipulating child nodes that aren't strings. In this case
16898 // we'll skip past it and indicate that the subsequent node should not
16899 // be dedented.
16900 if (!PM_NODE_TYPE_P(node, PM_STRING_NODE)) {
16901 nodes->nodes[write_index++] = node;
16902 dedent_next = false;
16903 continue;
16904 }
16905
16906 pm_string_node_t *string_node = ((pm_string_node_t *) node);
16907 if (dedent_next) {
16908 parse_heredoc_dedent_string(parser->arena, &string_node->unescaped, common_whitespace);
16909 }
16910
16911 if (heredoc_dedent_discard_string_node(parser, string_node)) {
16912 } else {
16913 nodes->nodes[write_index++] = node;
16914 }
16915
16916 // We always dedent the next node if it follows a string node.
16917 dedent_next = true;
16918 }
16919
16920 nodes->size = write_index;
16921}
16922
16926static pm_token_t
16927parse_strings_empty_content(const uint8_t *location) {
16928 return (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = location, .end = location };
16929}
16930
16934static PRISM_INLINE pm_node_t *
16935parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint16_t depth) {
16936 PRISM_ASSERT(parser->current.type == PM_TOKEN_STRING_BEGIN);
16937 bool concating = false;
16938 pm_node_list_t container = { 0 };
16939
16940 while (match1(parser, PM_TOKEN_STRING_BEGIN)) {
16941 pm_node_t *node = NULL;
16942
16943 // Here we have found a string literal. We'll parse it and add it to
16944 // the list of strings.
16945 const pm_lex_mode_t *lex_mode = parser->lex_modes.current;
16946 PRISM_ASSERT(lex_mode->mode == PM_LEX_STRING);
16947 bool lex_interpolation = lex_mode->as.string.interpolation;
16948 bool label_allowed = lex_mode->as.string.label_allowed && accepts_label;
16949
16950 pm_token_t opening = parser->current;
16951 parser_lex(parser);
16952
16953 if (match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
16954 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF);
16955 // If we get here, then we have an end immediately after a
16956 // start. In that case we'll create an empty content token and
16957 // return an uninterpolated string.
16958 pm_token_t content = parse_strings_empty_content(parser->previous.start);
16959 pm_string_node_t *string = pm_string_node_create(parser, &opening, &content, &parser->previous);
16960
16961 pm_string_shared_init(&string->unescaped, content.start, content.end);
16962 node = UP(string);
16963 } else if (accept1(parser, PM_TOKEN_LABEL_END)) {
16964 // If we get here, then we have an end of a label immediately
16965 // after a start. In that case we'll create an empty symbol
16966 // node.
16967 pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, NULL, &parser->previous);
16968 pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.start);
16969 node = UP(symbol);
16970
16971 if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL);
16972 } else if (!lex_interpolation) {
16973 // If we don't accept interpolation then we expect the string to
16974 // start with a single string content node.
16975 pm_string_t unescaped;
16976 pm_token_t content;
16977
16978 if (match1(parser, PM_TOKEN_EOF)) {
16979 unescaped = PM_STRING_EMPTY;
16980 content = (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = parser->start, .end = parser->start };
16981 } else {
16982 unescaped = parser->current_string;
16983 expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_EXPECT_STRING_CONTENT);
16984 content = parser->previous;
16985 }
16986
16987 // It is unfortunately possible to have multiple string content
16988 // nodes in a row in the case that there's heredoc content in
16989 // the middle of the string, like this cursed example:
16990 //
16991 // <<-END+'b
16992 // a
16993 // END
16994 // c'+'d'
16995 //
16996 // In that case we need to switch to an interpolated string to
16997 // be able to contain all of the parts.
16998 if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
16999 pm_node_list_t parts = { 0 };
17000 pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &unescaped));
17001 pm_node_list_open_append(&parser->node_chunks, &parts, part);
17002
17003 do {
17004 part = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
17005 pm_node_list_open_append(&parser->node_chunks, &parts, part);
17006 parser_lex(parser);
17007 } while (match1(parser, PM_TOKEN_STRING_CONTENT));
17008
17009 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF);
17010 node = UP(pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous));
17011 } else if (accept1(parser, PM_TOKEN_LABEL_END)) {
17012 node = UP(pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, parser->explicit_encoding, &content, &unescaped, true)));
17013 if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL);
17014 } else if (match1(parser, PM_TOKEN_EOF)) {
17015 pm_parser_err_token(parser, &opening, PM_ERR_STRING_LITERAL_EOF);
17016 node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped));
17017 } else if (accept1(parser, PM_TOKEN_STRING_END)) {
17018 node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped));
17019 } else {
17020 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_str(parser->previous.type));
17021 parser->previous.start = parser->previous.end;
17022 parser->previous.type = 0;
17023 node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped));
17024 }
17025 } else if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
17026 // In this case we've hit string content so we know the string
17027 // at least has something in it. We'll need to check if the
17028 // following token is the end (in which case we can return a
17029 // plain string) or if it's not then it has interpolation.
17030 pm_token_t content = parser->current;
17031 pm_string_t unescaped = parser->current_string;
17032 const pm_encoding_t *explicit_encoding = parser->explicit_encoding;
17033 bool joined_at_eof = lex_mode->as.string.joined_at_eof;
17034 parser_lex(parser);
17035
17036 if (match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
17037 node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped));
17038 pm_node_flag_set(node, parse_unescaped_encoding(parser, explicit_encoding));
17039
17040 if (!accept1(parser, PM_TOKEN_STRING_END)) {
17041 if (joined_at_eof) {
17042 /* Keep the diagnostic used when the lexer returned
17043 * this content in multiple parts. */
17044 pm_parser_err_token(parser, &opening, PM_ERR_STRING_INTERPOLATED_TERM);
17045 } else {
17046 /* Point at the last line of content when the string
17047 * ends in a newline. */
17048 const uint8_t *location = parser->previous.end;
17049 if (location > parser->start && location[-1] == '\n') location--;
17050 pm_parser_err(parser, U32(location - parser->start), 0, PM_ERR_STRING_LITERAL_EOF);
17051 }
17052
17053 parser->previous.start = parser->previous.end;
17054 parser->previous.type = 0;
17055 }
17056 } else if (accept1(parser, PM_TOKEN_LABEL_END)) {
17057 node = UP(pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, explicit_encoding, &content, &unescaped, true)));
17058 if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL);
17059 } else {
17060 // If we get here, then we have interpolation so we'll need
17061 // to create a string or symbol node with interpolation.
17062 pm_node_list_t parts = { 0 };
17063 pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped));
17064 pm_node_flag_set(part, parse_unescaped_encoding(parser, explicit_encoding));
17065 pm_node_list_open_append(&parser->node_chunks, &parts, part);
17066
17067 while (!match3(parser, PM_TOKEN_STRING_END, PM_TOKEN_LABEL_END, PM_TOKEN_EOF)) {
17068 if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) != NULL) {
17069 pm_node_list_open_append(&parser->node_chunks, &parts, part);
17070 }
17071 }
17072
17073 if (accept1(parser, PM_TOKEN_LABEL_END)) {
17074 node = UP(pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous));
17075 if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL);
17076 } else if (match1(parser, PM_TOKEN_EOF)) {
17077 pm_parser_err_token(parser, &opening, PM_ERR_STRING_INTERPOLATED_TERM);
17078 node = UP(pm_interpolated_string_node_create(parser, &opening, &parts, &parser->current));
17079 } else {
17080 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM);
17081 node = UP(pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous));
17082 }
17083 }
17084 } else {
17085 // If we get here, then the first part of the string is not plain
17086 // string content, in which case we need to parse the string as an
17087 // interpolated string.
17088 pm_node_list_t parts = { 0 };
17089 pm_node_t *part;
17090
17091 while (!match3(parser, PM_TOKEN_STRING_END, PM_TOKEN_LABEL_END, PM_TOKEN_EOF)) {
17092 if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) != NULL) {
17093 pm_node_list_open_append(&parser->node_chunks, &parts, part);
17094 }
17095 }
17096
17097 if (accept1(parser, PM_TOKEN_LABEL_END)) {
17098 node = UP(pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous));
17099 if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL);
17100 } else if (match1(parser, PM_TOKEN_EOF)) {
17101 pm_parser_err_token(parser, &opening, PM_ERR_STRING_INTERPOLATED_TERM);
17102 node = UP(pm_interpolated_string_node_create(parser, &opening, &parts, &parser->current));
17103 } else {
17104 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM);
17105 node = UP(pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous));
17106 }
17107 }
17108
17109 if (current == NULL) {
17110 // If the node we just parsed is a symbol node, then we can't
17111 // concatenate it with anything else, so we can now return that
17112 // node.
17113 if (PM_NODE_TYPE_P(node, PM_SYMBOL_NODE) || PM_NODE_TYPE_P(node, PM_INTERPOLATED_SYMBOL_NODE)) {
17114 return node;
17115 }
17116
17117 // If we don't already have a node, then it's fine and we can just
17118 // set the result to be the node we just parsed.
17119 current = node;
17120 } else {
17121 // Otherwise we need to check the type of the node we just parsed.
17122 // If it cannot be concatenated with the previous node, then we'll
17123 // need to add a syntax error.
17124 if (!PM_NODE_TYPE_P(node, PM_STRING_NODE) && !PM_NODE_TYPE_P(node, PM_INTERPOLATED_STRING_NODE)) {
17125 pm_parser_err_node(parser, node, PM_ERR_STRING_CONCATENATION);
17126 }
17127
17128 // If we haven't already started the parts of our container for
17129 // concatenation, we'll do that now.
17130 if (!concating) {
17131 if (!PM_NODE_TYPE_P(current, PM_STRING_NODE) && !PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) {
17132 pm_parser_err_node(parser, current, PM_ERR_STRING_CONCATENATION);
17133 }
17134
17135 concating = true;
17136 pm_node_list_open_append(&parser->node_chunks, &container, current);
17137 }
17138
17139 pm_node_list_open_append(&parser->node_chunks, &container, node);
17140 }
17141 }
17142
17143 if (concating) {
17144 return UP(pm_interpolated_string_node_create(parser, NULL, &container, NULL));
17145 }
17146
17147 return current;
17148}
17149
17150#define PM_PARSE_PATTERN_SINGLE 0
17151#define PM_PARSE_PATTERN_TOP 1
17152#define PM_PARSE_PATTERN_MULTI 2
17153
17154static pm_node_t *
17155parse_pattern(pm_parser_t *parser, pm_constant_id_set_t *captures, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth);
17156
17162static void
17163parse_pattern_capture(pm_parser_t *parser, pm_constant_id_set_t *captures, pm_constant_id_t capture, const pm_location_t *location) {
17164 // Skip this capture if it starts with an underscore.
17165 if (peek_at(parser, parser->start + location->start) == '_') return;
17166
17167 if (!pm_constant_id_set_insert(&parser->scratch_arena, captures, capture)) {
17168 pm_parser_err(parser, location->start, location->length, PM_ERR_PATTERN_CAPTURE_DUPLICATE);
17169 }
17170}
17171
17175static pm_node_t *
17176parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_set_t *captures, pm_node_t *node, uint16_t depth) {
17177 // Now, if there are any :: operators that follow, parse them as constant
17178 // path nodes.
17179 while (accept1(parser, PM_TOKEN_COLON_COLON)) {
17180 pm_token_t delimiter = parser->previous;
17181 expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT);
17182 node = UP(pm_constant_path_node_create(parser, node, &delimiter, &parser->previous));
17183 }
17184
17185 // If there is a [ or ( that follows, then this is part of a larger pattern
17186 // expression. We'll parse the inner pattern here, then modify the returned
17187 // inner pattern with our constant path attached.
17188 if (!match2(parser, PM_TOKEN_BRACKET_LEFT, PM_TOKEN_PARENTHESIS_LEFT)) {
17189 return node;
17190 }
17191
17192 pm_token_t opening;
17193 pm_token_t closing;
17194 pm_node_t *inner = NULL;
17195
17196 if (accept1(parser, PM_TOKEN_BRACKET_LEFT)) {
17197 opening = parser->previous;
17198 accept1(parser, PM_TOKEN_NEWLINE);
17199
17200 if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) {
17201 inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET, (uint16_t) (depth + 1));
17202 accept1(parser, PM_TOKEN_NEWLINE);
17203 expect1_opening(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET, &opening);
17204 }
17205
17206 closing = parser->previous;
17207 } else {
17208 parser_lex(parser);
17209 opening = parser->previous;
17210 accept1(parser, PM_TOKEN_NEWLINE);
17211
17212 if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
17213 inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN, (uint16_t) (depth + 1));
17214 accept1(parser, PM_TOKEN_NEWLINE);
17215 expect1_opening(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN, &opening);
17216 }
17217
17218 closing = parser->previous;
17219 }
17220
17221 if (!inner) {
17222 // If there was no inner pattern, then we have something like Foo() or
17223 // Foo[]. In that case we'll create an array pattern with no requireds.
17224 return UP(pm_array_pattern_node_constant_create(parser, node, &opening, NULL, &closing));
17225 }
17226
17227 // Now that we have the inner pattern, check to see if it's an array, find,
17228 // or hash pattern. If it is, then we'll attach our constant path to it if
17229 // it doesn't already have a constant. If it's not one of those node types
17230 // or it does have a constant, then we'll create an array pattern.
17231 switch (PM_NODE_TYPE(inner)) {
17232 case PM_ARRAY_PATTERN_NODE: {
17233 pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner;
17234
17235 if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) {
17236 PM_NODE_START_SET_NODE(pattern_node, node);
17237 PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing);
17238
17239 pattern_node->constant = node;
17240 pattern_node->opening_loc = TOK2LOC(parser, &opening);
17241 pattern_node->closing_loc = TOK2LOC(parser, &closing);
17242
17243 return UP(pattern_node);
17244 }
17245
17246 break;
17247 }
17248 case PM_FIND_PATTERN_NODE: {
17249 pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner;
17250
17251 if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) {
17252 PM_NODE_START_SET_NODE(pattern_node, node);
17253 PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing);
17254
17255 pattern_node->constant = node;
17256 pattern_node->opening_loc = TOK2LOC(parser, &opening);
17257 pattern_node->closing_loc = TOK2LOC(parser, &closing);
17258
17259 return UP(pattern_node);
17260 }
17261
17262 break;
17263 }
17264 case PM_HASH_PATTERN_NODE: {
17265 pm_hash_pattern_node_t *pattern_node = (pm_hash_pattern_node_t *) inner;
17266
17267 if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) {
17268 PM_NODE_START_SET_NODE(pattern_node, node);
17269 PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing);
17270
17271 pattern_node->constant = node;
17272 pattern_node->opening_loc = TOK2LOC(parser, &opening);
17273 pattern_node->closing_loc = TOK2LOC(parser, &closing);
17274
17275 return UP(pattern_node);
17276 }
17277
17278 break;
17279 }
17280 default:
17281 break;
17282 }
17283
17284 // If we got here, then we didn't return one of the inner patterns by
17285 // attaching its constant. In this case we'll create an array pattern and
17286 // attach our constant to it.
17287 return UP(pm_array_pattern_node_constant_create(parser, node, &opening, inner, &closing));
17288}
17289
17293static pm_splat_node_t *
17294parse_pattern_rest(pm_parser_t *parser, pm_constant_id_set_t *captures) {
17295 PRISM_ASSERT(parser->previous.type == PM_TOKEN_USTAR);
17296 pm_token_t operator = parser->previous;
17297 pm_node_t *name = NULL;
17298
17299 // Rest patterns don't necessarily have a name associated with them. So we
17300 // will check for that here. If they do, then we'll add it to the local
17301 // table since this pattern will cause it to become a local variable.
17302 if (accept1(parser, PM_TOKEN_IDENTIFIER)) {
17303 pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous);
17304
17305 int depth;
17306 if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) {
17307 pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0);
17308 }
17309
17310 pm_location_t previous_loc = TOK2LOC(parser, &parser->previous);
17311 parse_pattern_capture(parser, captures, constant_id, &previous_loc);
17312 name = UP(pm_local_variable_target_node_create(
17313 parser,
17314 &previous_loc,
17315 constant_id,
17316 (uint32_t) (depth == -1 ? 0 : depth)
17317 ));
17318 }
17319
17320 // Finally we can return the created node.
17321 return pm_splat_node_create(parser, &operator, name);
17322}
17323
17327static pm_node_t *
17328parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_set_t *captures) {
17329 PRISM_ASSERT(parser->current.type == PM_TOKEN_USTAR_STAR);
17330 parser_lex(parser);
17331
17332 pm_token_t operator = parser->previous;
17333 pm_node_t *value = NULL;
17334
17335 if (accept1(parser, PM_TOKEN_KEYWORD_NIL)) {
17336 return UP(pm_no_keywords_parameter_node_create(parser, &operator, &parser->previous));
17337 }
17338
17339 if (accept1(parser, PM_TOKEN_IDENTIFIER)) {
17340 pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous);
17341
17342 int depth;
17343 if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) {
17344 pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0);
17345 }
17346
17347 pm_location_t previous_loc = TOK2LOC(parser, &parser->previous);
17348 parse_pattern_capture(parser, captures, constant_id, &previous_loc);
17349 value = UP(pm_local_variable_target_node_create(
17350 parser,
17351 &previous_loc,
17352 constant_id,
17353 (uint32_t) (depth == -1 ? 0 : depth)
17354 ));
17355 }
17356
17357 return UP(pm_assoc_splat_node_create(parser, value, &operator));
17358}
17359
17364static bool
17365pm_slice_is_valid_local(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
17366 ptrdiff_t length = end - start;
17367 if (length == 0) return false;
17368
17369 // First ensure that it starts with a valid identifier starting character.
17370 size_t width = char_is_identifier_start(parser, start, end - start);
17371 if (width == 0) return false;
17372
17373 // Next, ensure that it's not an uppercase character.
17374 if (parser->encoding_changed) {
17375 if (pm_encoding_isupper_char(parser->encoding, start, length, parser->version)) return false;
17376 } else {
17377 if (pm_encoding_utf_8_isupper_char(start, length, parser->version)) return false;
17378 }
17379
17380 // Next, iterate through all of the bytes of the string to ensure that they
17381 // are all valid identifier characters.
17382 const uint8_t *cursor = start + width;
17383 while ((width = char_is_identifier(parser, cursor, end - cursor))) cursor += width;
17384 return cursor == end;
17385}
17386
17391static pm_node_t *
17392parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_set_t *captures, pm_symbol_node_t *key) {
17393 const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->content_loc;
17394 const uint8_t *start = parser->start + PM_LOCATION_START(value_loc);
17395 const uint8_t *end = parser->start + PM_LOCATION_END(value_loc);
17396
17397 pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, start, end);
17398 int depth = -1;
17399
17400 if (pm_slice_is_valid_local(parser, start, end)) {
17401 depth = pm_parser_local_depth_constant_id(parser, constant_id);
17402 } else {
17403 pm_parser_err(parser, PM_NODE_START(key), PM_NODE_LENGTH(key), PM_ERR_PATTERN_HASH_KEY_LOCALS);
17404
17405 if ((end > start) && ((end[-1] == '!') || (end[-1] == '?'))) {
17406 PM_PARSER_ERR_FORMAT(parser, value_loc->start, value_loc->length, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (end - start), (const char *) start);
17407 }
17408 }
17409
17410 if (depth == -1) {
17411 pm_parser_local_add(parser, constant_id, start, end, 0);
17412 }
17413
17414 parse_pattern_capture(parser, captures, constant_id, value_loc);
17415 pm_local_variable_target_node_t *target = pm_local_variable_target_node_create(
17416 parser,
17417 value_loc,
17418 constant_id,
17419 (uint32_t) (depth == -1 ? 0 : depth)
17420 );
17421
17422 return UP(pm_implicit_node_create(parser, UP(target)));
17423}
17424
17429static void
17430parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) {
17431 if (pm_static_literals_add(parser, keys, node, true) != NULL) {
17432 pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_KEY_DUPLICATE);
17433 }
17434}
17435
17439static pm_hash_pattern_node_t *
17440parse_pattern_hash(pm_parser_t *parser, pm_constant_id_set_t *captures, pm_node_t *first_node, uint16_t depth) {
17441 pm_node_list_t assocs = { 0 };
17442 pm_static_literals_t keys = { 0 };
17443 pm_node_t *rest = NULL;
17444
17445 switch (PM_NODE_TYPE(first_node)) {
17446 case PM_ASSOC_SPLAT_NODE:
17447 case PM_NO_KEYWORDS_PARAMETER_NODE:
17448 rest = first_node;
17449 break;
17450 case PM_INTERPOLATED_SYMBOL_NODE:
17451 case PM_SYMBOL_NODE: {
17452 if (pm_symbol_node_label_p(parser, first_node)) {
17453 if (PM_NODE_TYPE_P(first_node, PM_INTERPOLATED_SYMBOL_NODE)) {
17454 pm_parser_err_node(parser, first_node, PM_ERR_PATTERN_HASH_KEY_INTERPOLATED);
17455 } else {
17456 parse_pattern_hash_key(parser, &keys, first_node);
17457 }
17458
17459 pm_node_t *value;
17460
17461 /*
17462 * The label has an implicit value when the next token cannot
17463 * begin a pattern, mirroring the grammar's `p_kw: p_kw_label`
17464 * reduction.
17465 */
17466 if (!token_begins_pattern_p(parser->current.type)) {
17467 if (PM_NODE_TYPE_P(first_node, PM_SYMBOL_NODE)) {
17468 value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) first_node);
17469 } else {
17470 value = UP(pm_error_recovery_node_create(parser, PM_NODE_END(first_node), 0));
17471 }
17472 } else {
17473 // Here we have a value for the first assoc in the list, so
17474 // we will parse it now.
17475 value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY, (uint16_t) (depth + 1));
17476 }
17477
17478 pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, NULL, value));
17479 pm_node_list_open_append(&parser->node_chunks, &assocs, assoc);
17480 break;
17481 }
17482 }
17484 default: {
17485 // If we get anything else, then this is an error. For this we'll
17486 // create a missing node for the value and create an assoc node for
17487 // the first node in the list.
17488 pm_diagnostic_id_t diag_id = PM_NODE_TYPE_P(first_node, PM_INTERPOLATED_SYMBOL_NODE) ? PM_ERR_PATTERN_HASH_KEY_INTERPOLATED : PM_ERR_PATTERN_HASH_KEY_LABEL;
17489 pm_parser_err_node(parser, first_node, diag_id);
17490
17491 pm_node_t *value = UP(pm_error_recovery_node_create(parser, PM_NODE_START(first_node), PM_NODE_LENGTH(first_node)));
17492 pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, NULL, value));
17493
17494 pm_node_list_open_append(&parser->node_chunks, &assocs, assoc);
17495 break;
17496 }
17497 }
17498
17499 // If there are any other assocs, then we'll parse them now.
17500 while (accept1(parser, PM_TOKEN_COMMA)) {
17501 /*
17502 * A trailing comma ends the pattern when the next token cannot begin
17503 * another element, mirroring the grammar's `p_kwargs: p_kwarg ','`
17504 * reduction.
17505 */
17506 if (!token_begins_pattern_p(parser->current.type)) {
17507 // Trailing commas are not allowed to follow a rest pattern.
17508 if (rest != NULL) {
17509 pm_parser_err_token(parser, &parser->current, PM_ERR_PATTERN_EXPRESSION_AFTER_REST);
17510 }
17511
17512 break;
17513 }
17514
17515 if (match1(parser, PM_TOKEN_USTAR_STAR)) {
17516 pm_node_t *assoc = parse_pattern_keyword_rest(parser, captures);
17517
17518 if (rest == NULL) {
17519 rest = assoc;
17520 } else {
17521 pm_parser_err_node(parser, assoc, PM_ERR_PATTERN_EXPRESSION_AFTER_REST);
17522 pm_node_list_open_append(&parser->node_chunks, &assocs, assoc);
17523 }
17524 } else {
17525 pm_node_t *key;
17526
17527 if (match1(parser, PM_TOKEN_STRING_BEGIN)) {
17528 key = parse_strings(parser, NULL, true, (uint16_t) (depth + 1));
17529
17530 if (PM_NODE_TYPE_P(key, PM_INTERPOLATED_SYMBOL_NODE)) {
17531 pm_parser_err_node(parser, key, PM_ERR_PATTERN_HASH_KEY_INTERPOLATED);
17532 } else if (!pm_symbol_node_label_p(parser, key)) {
17533 pm_parser_err_node(parser, key, PM_ERR_PATTERN_LABEL_AFTER_COMMA);
17534 }
17535 } else if (accept1(parser, PM_TOKEN_LABEL)) {
17536 key = UP(pm_symbol_node_label_create(parser, &parser->previous));
17537 } else {
17538 expect1(parser, PM_TOKEN_LABEL, PM_ERR_PATTERN_LABEL_AFTER_COMMA);
17539
17540 pm_token_t label = { .type = PM_TOKEN_LABEL, .start = parser->previous.end, .end = parser->previous.end };
17541 key = UP(pm_symbol_node_create(parser, NULL, &label, NULL));
17542 }
17543
17544 parse_pattern_hash_key(parser, &keys, key);
17545 pm_node_t *value = NULL;
17546
17547 /*
17548 * The label has an implicit value when the next token cannot
17549 * begin a pattern, mirroring the grammar's `p_kw: p_kw_label`
17550 * reduction.
17551 */
17552 if (!token_begins_pattern_p(parser->current.type)) {
17553 if (PM_NODE_TYPE_P(key, PM_SYMBOL_NODE)) {
17554 value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) key);
17555 } else {
17556 value = UP(pm_error_recovery_node_create(parser, PM_NODE_END(key), 0));
17557 }
17558 } else {
17559 value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY, (uint16_t) (depth + 1));
17560 }
17561
17562 pm_node_t *assoc = UP(pm_assoc_node_create(parser, key, NULL, value));
17563
17564 if (rest != NULL) {
17565 pm_parser_err_node(parser, assoc, PM_ERR_PATTERN_EXPRESSION_AFTER_REST);
17566 }
17567
17568 pm_node_list_open_append(&parser->node_chunks, &assocs, assoc);
17569 }
17570 }
17571
17572 pm_hash_pattern_node_t *node = pm_hash_pattern_node_node_list_create(parser, &assocs, rest);
17573
17574 pm_static_literals_free(&keys);
17575 return node;
17576}
17577
17581static pm_node_t *
17582parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_set_t *captures, pm_diagnostic_id_t diag_id, uint16_t depth) {
17583 switch (parser->current.type) {
17584 case PM_TOKEN_IDENTIFIER:
17585 case PM_TOKEN_METHOD_NAME: {
17586 parser_lex(parser);
17587 pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous);
17588
17589 int depth;
17590 if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) {
17591 pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0);
17592 }
17593
17594 pm_location_t previous_loc = TOK2LOC(parser, &parser->previous);
17595 parse_pattern_capture(parser, captures, constant_id, &previous_loc);
17596 return UP(pm_local_variable_target_node_create(
17597 parser,
17598 &previous_loc,
17599 constant_id,
17600 (uint32_t) (depth == -1 ? 0 : depth)
17601 ));
17602 }
17603 case PM_TOKEN_BRACKET_LEFT_ARRAY: {
17604 pm_token_t opening = parser->current;
17605 parser_lex(parser);
17606
17607 if (accept1(parser, PM_TOKEN_BRACKET_RIGHT)) {
17608 // If we have an empty array pattern, then we'll just return a new
17609 // array pattern node.
17610 return UP(pm_array_pattern_node_bracket_create(parser, &opening, NULL, &parser->previous));
17611 }
17612
17613 // Otherwise, we'll parse the inner pattern, then deal with it depending
17614 // on the type it returns.
17615 pm_node_t *inner = parse_pattern(parser, captures, PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET, (uint16_t) (depth + 1));
17616
17617 accept1(parser, PM_TOKEN_NEWLINE);
17618 expect1_opening(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET, &opening);
17619 pm_token_t closing = parser->previous;
17620
17621 switch (PM_NODE_TYPE(inner)) {
17622 case PM_ARRAY_PATTERN_NODE: {
17623 pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner;
17624 if (pattern_node->opening_loc.length == 0) {
17625 PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening);
17626 PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing);
17627
17628 pattern_node->opening_loc = TOK2LOC(parser, &opening);
17629 pattern_node->closing_loc = TOK2LOC(parser, &closing);
17630
17631 return UP(pattern_node);
17632 }
17633
17634 break;
17635 }
17636 case PM_FIND_PATTERN_NODE: {
17637 pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner;
17638 if (pattern_node->opening_loc.length == 0) {
17639 PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening);
17640 PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing);
17641
17642 pattern_node->opening_loc = TOK2LOC(parser, &opening);
17643 pattern_node->closing_loc = TOK2LOC(parser, &closing);
17644
17645 return UP(pattern_node);
17646 }
17647
17648 break;
17649 }
17650 default:
17651 break;
17652 }
17653
17654 return UP(pm_array_pattern_node_bracket_create(parser, &opening, inner, &closing));
17655 }
17656 case PM_TOKEN_BRACE_LEFT_HASH: {
17657 bool previous_pattern_matching_newlines = parser->pattern_matching_newlines;
17658 parser->pattern_matching_newlines = false;
17659
17660 pm_hash_pattern_node_t *node;
17661 pm_token_t opening = parser->current;
17662 parser_lex(parser);
17663
17664 if (accept1(parser, PM_TOKEN_BRACE_RIGHT)) {
17665 // If we have an empty hash pattern, then we'll just return a new hash
17666 // pattern node.
17667 node = pm_hash_pattern_node_empty_create(parser, &opening, &parser->previous);
17668 } else {
17669 pm_node_t *first_node;
17670
17671 switch (parser->current.type) {
17672 case PM_TOKEN_LABEL:
17673 parser_lex(parser);
17674 first_node = UP(pm_symbol_node_label_create(parser, &parser->previous));
17675 break;
17676 case PM_TOKEN_USTAR_STAR:
17677 first_node = parse_pattern_keyword_rest(parser, captures);
17678 break;
17679 case PM_TOKEN_STRING_BEGIN:
17680 first_node = parse_expression(parser, PM_BINDING_POWER_MAX, PM_PARSE_ACCEPTS_DO_BLOCK | PM_PARSE_ACCEPTS_LABEL, PM_ERR_PATTERN_HASH_KEY_LABEL, (uint16_t) (depth + 1));
17681 break;
17682 default: {
17683 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_PATTERN_HASH_KEY, pm_token_str(parser->current.type));
17684 parse_discard_token(parser, depth);
17685
17686 first_node = UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)));
17687 break;
17688 }
17689 }
17690
17691 node = parse_pattern_hash(parser, captures, first_node, (uint16_t) (depth + 1));
17692
17693 accept1(parser, PM_TOKEN_NEWLINE);
17694 expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_PATTERN_TERM_BRACE, &opening);
17695 pm_token_t closing = parser->previous;
17696
17697 PM_NODE_START_SET_TOKEN(parser, node, &opening);
17698 PM_NODE_LENGTH_SET_TOKEN(parser, node, &closing);
17699
17700 node->opening_loc = TOK2LOC(parser, &opening);
17701 node->closing_loc = TOK2LOC(parser, &closing);
17702 }
17703
17704 parser->pattern_matching_newlines = previous_pattern_matching_newlines;
17705 return UP(node);
17706 }
17707 case PM_TOKEN_UDOT_DOT:
17708 case PM_TOKEN_UDOT_DOT_DOT: {
17709 pm_token_t operator = parser->current;
17710 parser_lex(parser);
17711
17712 // Since we have a unary range operator, we need to parse the subsequent
17713 // expression as the right side of the range.
17714 switch (parser->current.type) {
17715 case PM_CASE_PRIMITIVE: {
17716 pm_node_t *right = parse_expression(parser, PM_BINDING_POWER_MAX, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE, (uint16_t) (depth + 1));
17717 return UP(pm_range_node_create(parser, NULL, &operator, right));
17718 }
17719 default: {
17720 pm_parser_err_token(parser, &operator, PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE);
17721 pm_node_t *right = UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &operator), PM_TOKEN_LENGTH(&operator)));
17722 return UP(pm_range_node_create(parser, NULL, &operator, right));
17723 }
17724 }
17725 }
17726 case PM_CASE_PRIMITIVE: {
17727 pm_node_t *node = parse_expression(parser, PM_BINDING_POWER_MAX, PM_PARSE_ACCEPTS_LABEL | PM_PARSE_ACCEPTS_DO_BLOCK, diag_id, (uint16_t) (depth + 1));
17728
17729 // If we found a label, we need to immediately return to the caller.
17730 if (pm_symbol_node_label_p(parser, node)) return node;
17731
17732 // Call nodes (arithmetic operations) are not allowed in patterns
17733 if (PM_NODE_TYPE(node) == PM_CALL_NODE) {
17734 pm_parser_err_node(parser, node, diag_id);
17735 return UP(pm_error_recovery_node_create_unexpected(parser, node));
17736 }
17737
17738 // Now that we have a primitive, we need to check if it's part of a range.
17739 if (accept2(parser, PM_TOKEN_DOT_DOT, PM_TOKEN_DOT_DOT_DOT)) {
17740 pm_token_t operator = parser->previous;
17741
17742 // Now that we have the operator, we need to check if this is followed
17743 // by another expression. If it is, then we will create a full range
17744 // node. Otherwise, we'll create an endless range.
17745 switch (parser->current.type) {
17746 case PM_CASE_PRIMITIVE: {
17747 pm_node_t *right = parse_expression(parser, PM_BINDING_POWER_MAX, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE, (uint16_t) (depth + 1));
17748 return UP(pm_range_node_create(parser, node, &operator, right));
17749 }
17750 default:
17751 return UP(pm_range_node_create(parser, node, &operator, NULL));
17752 }
17753 }
17754
17755 return node;
17756 }
17757 case PM_TOKEN_CARET: {
17758 parser_lex(parser);
17759 pm_token_t operator = parser->previous;
17760
17761 // At this point we have a pin operator. We need to check the subsequent
17762 // expression to determine if it's a variable or an expression.
17763 switch (parser->current.type) {
17764 case PM_TOKEN_IDENTIFIER: {
17765 parser_lex(parser);
17766 pm_node_t *variable = UP(parse_variable(parser));
17767
17768 if (variable == NULL) {
17769 PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->previous, PM_ERR_NO_LOCAL_VARIABLE);
17770 variable = UP(pm_local_variable_read_node_missing_create(parser, &parser->previous, 0));
17771 }
17772
17773 return UP(pm_pinned_variable_node_create(parser, &operator, variable));
17774 }
17775 case PM_TOKEN_INSTANCE_VARIABLE: {
17776 parser_lex(parser);
17777 pm_node_t *variable = UP(pm_instance_variable_read_node_create(parser, &parser->previous));
17778
17779 return UP(pm_pinned_variable_node_create(parser, &operator, variable));
17780 }
17781 case PM_TOKEN_CLASS_VARIABLE: {
17782 parser_lex(parser);
17783 pm_node_t *variable = UP(pm_class_variable_read_node_create(parser, &parser->previous));
17784
17785 return UP(pm_pinned_variable_node_create(parser, &operator, variable));
17786 }
17787 case PM_TOKEN_GLOBAL_VARIABLE: {
17788 parser_lex(parser);
17789 pm_node_t *variable = UP(pm_global_variable_read_node_create(parser, &parser->previous));
17790
17791 return UP(pm_pinned_variable_node_create(parser, &operator, variable));
17792 }
17793 case PM_TOKEN_PARENTHESIS_LEFT_GROUPING: {
17794 bool previous_pattern_matching_newlines = parser->pattern_matching_newlines;
17795 parser->pattern_matching_newlines = false;
17796
17797 pm_token_t lparen = parser->current;
17798 parser_lex(parser);
17799
17800 pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, PM_PARSE_ACCEPTS_DO_BLOCK | PM_PARSE_ACCEPTS_COMMAND_CALL, PM_ERR_PATTERN_EXPRESSION_AFTER_PIN, (uint16_t) (depth + 1));
17801 parser->pattern_matching_newlines = previous_pattern_matching_newlines;
17802
17803 accept1(parser, PM_TOKEN_NEWLINE);
17804 expect1_opening(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN, &lparen);
17805 return UP(pm_pinned_expression_node_create(parser, expression, &operator, &lparen, &parser->previous));
17806 }
17807 default: {
17808 // If we get here, then we have a pin operator followed by something
17809 // not understood. We'll create a missing node and return that.
17810 pm_parser_err_token(parser, &operator, PM_ERR_PATTERN_EXPRESSION_AFTER_PIN);
17811 pm_node_t *variable = UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &operator), PM_TOKEN_LENGTH(&operator)));
17812 return UP(pm_pinned_variable_node_create(parser, &operator, variable));
17813 }
17814 }
17815 }
17816 case PM_TOKEN_UCOLON_COLON: {
17817 pm_token_t delimiter = parser->current;
17818 parser_lex(parser);
17819
17820 expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT);
17821 pm_constant_path_node_t *node = pm_constant_path_node_create(parser, NULL, &delimiter, &parser->previous);
17822
17823 return parse_pattern_constant_path(parser, captures, UP(node), (uint16_t) (depth + 1));
17824 }
17825 case PM_TOKEN_CONSTANT: {
17826 pm_token_t constant = parser->current;
17827 parser_lex(parser);
17828
17829 pm_node_t *node = UP(pm_constant_read_node_create(parser, &constant));
17830 return parse_pattern_constant_path(parser, captures, node, (uint16_t) (depth + 1));
17831 }
17832 default:
17833 pm_parser_err_current(parser, diag_id);
17834 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_END(parser, &parser->previous), 0));
17835 }
17836}
17837
17845static pm_visit_result_t
17846parse_pattern_alternation_error_each(const pm_node_t *node, void *data) {
17847 pm_parser_t *parser = (pm_parser_t *) data;
17848
17849 switch (PM_NODE_TYPE(node)) {
17850 case PM_LOCAL_VARIABLE_TARGET_NODE:
17851 // Underscore-prefixed names are not captures, see
17852 // parse_pattern_capture.
17853 if (peek_at(parser, parser->start + PM_NODE_START(node)) != '_') {
17854 pm_parser_err(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE);
17855 }
17856 return PM_VISIT_SKIP;
17857 case PM_ARRAY_PATTERN_NODE:
17858 case PM_ASSOC_NODE:
17859 case PM_ASSOC_SPLAT_NODE:
17860 case PM_CAPTURE_PATTERN_NODE:
17861 case PM_FIND_PATTERN_NODE:
17862 case PM_HASH_PATTERN_NODE:
17863 case PM_IMPLICIT_NODE:
17864 case PM_PARENTHESES_NODE:
17865 case PM_SPLAT_NODE:
17866 return PM_VISIT_DESCEND;
17867 default:
17868 return PM_VISIT_SKIP;
17869 }
17870}
17871
17877static void
17878parse_pattern_alternation_error(pm_parser_t *parser, const pm_node_t *node) {
17879 pm_visit_node(node, parse_pattern_alternation_error_each, parser);
17880}
17881
17886static pm_node_t *
17887parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_set_t *captures, pm_node_t *first_node, pm_diagnostic_id_t diag_id, uint16_t depth) {
17888 pm_node_t *node = first_node;
17889 bool alternation = false;
17890
17891 while ((node == NULL) || (alternation = accept1(parser, PM_TOKEN_PIPE))) {
17892 if (alternation && !PM_NODE_TYPE_P(node, PM_ALTERNATION_PATTERN_NODE) && captures->size) {
17893 parse_pattern_alternation_error(parser, node);
17894 }
17895
17896 switch (parser->current.type) {
17897 case PM_TOKEN_IDENTIFIER:
17898 case PM_TOKEN_BRACKET_LEFT_ARRAY:
17899 case PM_TOKEN_BRACE_LEFT_HASH:
17900 case PM_TOKEN_CARET:
17901 case PM_TOKEN_CONSTANT:
17902 case PM_TOKEN_UCOLON_COLON:
17903 case PM_TOKEN_UDOT_DOT:
17904 case PM_TOKEN_UDOT_DOT_DOT:
17905 case PM_CASE_PRIMITIVE: {
17906 if (!alternation) {
17907 node = parse_pattern_primitive(parser, captures, diag_id, (uint16_t) (depth + 1));
17908 } else {
17909 pm_token_t operator = parser->previous;
17910 pm_node_t *right = parse_pattern_primitive(parser, captures, PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE, (uint16_t) (depth + 1));
17911
17912 if (captures->size) parse_pattern_alternation_error(parser, right);
17913 node = UP(pm_alternation_pattern_node_create(parser, node, right, &operator));
17914 }
17915
17916 break;
17917 }
17918 case PM_TOKEN_PARENTHESIS_LEFT_GROUPING:
17919 case PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES: {
17920 pm_token_t operator = parser->previous;
17921 pm_token_t opening = parser->current;
17922 parser_lex(parser);
17923
17924 pm_node_t *body = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN, (uint16_t) (depth + 1));
17925 accept1(parser, PM_TOKEN_NEWLINE);
17926 expect1_opening(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN, &opening);
17927 pm_node_t *right = UP(pm_parentheses_node_create(parser, &opening, body, &parser->previous, 0));
17928
17929 if (!alternation) {
17930 node = right;
17931 } else {
17932 if (captures->size) parse_pattern_alternation_error(parser, right);
17933 node = UP(pm_alternation_pattern_node_create(parser, node, right, &operator));
17934 }
17935
17936 break;
17937 }
17938 default: {
17939 pm_parser_err_current(parser, diag_id);
17940 pm_node_t *right = UP(pm_error_recovery_node_create(parser, PM_TOKEN_END(parser, &parser->previous), 0));
17941
17942 if (!alternation) {
17943 node = right;
17944 } else {
17945 if (captures->size) parse_pattern_alternation_error(parser, right);
17946 node = UP(pm_alternation_pattern_node_create(parser, node, right, &parser->previous));
17947 }
17948
17949 break;
17950 }
17951 }
17952 }
17953
17954 // If we have an =>, then we are assigning this pattern to a variable.
17955 // In this case we should create an assignment node.
17956 while (accept1(parser, PM_TOKEN_EQUAL_GREATER)) {
17957 pm_token_t operator = parser->previous;
17958 expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_PATTERN_IDENT_AFTER_HROCKET);
17959
17960 pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous);
17961 int depth;
17962
17963 if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) {
17964 pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0);
17965 }
17966
17967 pm_location_t previous_loc = TOK2LOC(parser, &parser->previous);
17968 parse_pattern_capture(parser, captures, constant_id, &previous_loc);
17969 pm_local_variable_target_node_t *target = pm_local_variable_target_node_create(
17970 parser,
17971 &previous_loc,
17972 constant_id,
17973 (uint32_t) (depth == -1 ? 0 : depth)
17974 );
17975
17976 node = UP(pm_capture_pattern_node_create(parser, node, target, &operator));
17977 }
17978
17979 return node;
17980}
17981
17985static pm_node_t *
17986parse_pattern(pm_parser_t *parser, pm_constant_id_set_t *captures, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
17987 /* Nested patterns recurse back into parse_pattern without passing through
17988 * parse_expression, so this cycle needs its own depth check. */
17989 if (PRISM_UNLIKELY(depth >= PRISM_DEPTH_MAXIMUM || pm_parser_stack_exhausted(parser))) {
17990 return parse_resource_recovery(parser, depth);
17991 }
17992
17993 pm_node_t *node = NULL;
17994
17995 bool leading_rest = false;
17996 bool trailing_rest = false;
17997
17998 switch (parser->current.type) {
17999 case PM_TOKEN_LABEL: {
18000 parser_lex(parser);
18001 pm_node_t *key = UP(pm_symbol_node_label_create(parser, &parser->previous));
18002 node = UP(parse_pattern_hash(parser, captures, key, (uint16_t) (depth + 1)));
18003
18004 if (!(flags & PM_PARSE_PATTERN_TOP)) {
18005 pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_IMPLICIT);
18006 }
18007
18008 return node;
18009 }
18010 case PM_TOKEN_USTAR_STAR: {
18011 node = parse_pattern_keyword_rest(parser, captures);
18012 node = UP(parse_pattern_hash(parser, captures, node, (uint16_t) (depth + 1)));
18013
18014 if (!(flags & PM_PARSE_PATTERN_TOP)) {
18015 pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_IMPLICIT);
18016 }
18017
18018 return node;
18019 }
18020 case PM_TOKEN_STRING_BEGIN: {
18021 // We need special handling for string beginnings because they could
18022 // be dynamic symbols leading to hash patterns.
18023 node = parse_pattern_primitive(parser, captures, diag_id, (uint16_t) (depth + 1));
18024
18025 if (pm_symbol_node_label_p(parser, node)) {
18026 node = UP(parse_pattern_hash(parser, captures, node, (uint16_t) (depth + 1)));
18027
18028 if (!(flags & PM_PARSE_PATTERN_TOP)) {
18029 pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_IMPLICIT);
18030 }
18031
18032 return node;
18033 }
18034
18035 node = parse_pattern_primitives(parser, captures, node, diag_id, (uint16_t) (depth + 1));
18036 break;
18037 }
18038 case PM_TOKEN_USTAR: {
18039 if (flags & (PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI)) {
18040 parser_lex(parser);
18041 node = UP(parse_pattern_rest(parser, captures));
18042 leading_rest = true;
18043 break;
18044 }
18045 }
18047 default:
18048 node = parse_pattern_primitives(parser, captures, NULL, diag_id, (uint16_t) (depth + 1));
18049 break;
18050 }
18051
18052 // If we got a dynamic label symbol, then we need to treat it like the
18053 // beginning of a hash pattern.
18054 if (pm_symbol_node_label_p(parser, node)) {
18055 return UP(parse_pattern_hash(parser, captures, node, (uint16_t) (depth + 1)));
18056 }
18057
18058 if ((flags & PM_PARSE_PATTERN_MULTI) && match1(parser, PM_TOKEN_COMMA)) {
18059 // If we have a comma, then we are now parsing either an array pattern
18060 // or a find pattern. We need to parse all of the patterns, put them
18061 // into a big list, and then determine which type of node we have.
18062 pm_node_list_t nodes = { 0 };
18063 pm_node_list_open_append(&parser->node_chunks, &nodes, node);
18064
18065 // Gather up all of the patterns into the list.
18066 while (accept1(parser, PM_TOKEN_COMMA)) {
18067 /*
18068 * A trailing comma ends the pattern when the next token cannot
18069 * begin another pattern element, leaving the token for the
18070 * enclosing context to accept or reject.
18071 */
18072 if (!token_begins_pattern_p(parser->current.type)) {
18073 // A trailing comma forms an implicit rest pattern (`[a,]` is
18074 // `[a, *]`). If a rest pattern has already been parsed, then
18075 // this is a second rest, which is not allowed (e.g. `[a, *b,]`
18076 // or `x => a, *b,`).
18077 if (trailing_rest) {
18078 pm_parser_err_previous(parser, PM_ERR_PATTERN_REST);
18079 }
18080
18081 node = UP(pm_implicit_rest_node_create(parser, &parser->previous));
18082 pm_node_list_open_append(&parser->node_chunks, &nodes, node);
18083 trailing_rest = true;
18084 break;
18085 }
18086
18087 if (accept1(parser, PM_TOKEN_USTAR)) {
18088 node = UP(parse_pattern_rest(parser, captures));
18089
18090 // If we have already parsed a splat pattern, then this is an
18091 // error. We will continue to parse the rest of the patterns,
18092 // but we will indicate it as an error.
18093 if (trailing_rest) {
18094 pm_parser_err_previous(parser, PM_ERR_PATTERN_REST);
18095 }
18096
18097 trailing_rest = true;
18098 } else {
18099 node = parse_pattern_primitives(parser, captures, NULL, PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA, (uint16_t) (depth + 1));
18100 }
18101
18102 pm_node_list_open_append(&parser->node_chunks, &nodes, node);
18103 }
18104
18105 pm_node_list_finalize(&parser->node_chunks, parser->arena, &nodes);
18106
18107 // If the first pattern and the last pattern are rest patterns, then we
18108 // will call this a find pattern, regardless of how many rest patterns
18109 // are in between because we know we already added the appropriate
18110 // errors. Otherwise we will create an array pattern.
18111 if (leading_rest && PM_NODE_TYPE_P(nodes.nodes[nodes.size - 1], PM_SPLAT_NODE)) {
18112 node = UP(pm_find_pattern_node_create(parser, &nodes));
18113
18114 if (nodes.size == 2) {
18115 pm_parser_err_node(parser, node, PM_ERR_PATTERN_FIND_MISSING_INNER);
18116 }
18117 } else {
18118 node = UP(pm_array_pattern_node_node_list_create(parser, &nodes));
18119
18120 if (leading_rest && trailing_rest) {
18121 pm_parser_err_node(parser, node, PM_ERR_PATTERN_ARRAY_MULTIPLE_RESTS);
18122 }
18123 }
18124 } else if (leading_rest) {
18125 // Otherwise, if we parsed a single splat pattern, then we know we have
18126 // an array pattern, so we can go ahead and create that node.
18127 node = UP(pm_array_pattern_node_rest_create(parser, node));
18128 }
18129
18130 return node;
18131}
18132
18138static PRISM_INLINE void
18139parse_negative_numeric(pm_node_t *node) {
18140 switch (PM_NODE_TYPE(node)) {
18141 case PM_INTEGER_NODE: {
18142 pm_integer_node_t *cast = (pm_integer_node_t *) node;
18143 cast->base.location.start--;
18144 cast->base.location.length++;
18145 cast->value.negative = true;
18146 break;
18147 }
18148 case PM_FLOAT_NODE: {
18149 pm_float_node_t *cast = (pm_float_node_t *) node;
18150 cast->base.location.start--;
18151 cast->base.location.length++;
18152 cast->value = -cast->value;
18153 break;
18154 }
18155 case PM_RATIONAL_NODE: {
18156 pm_rational_node_t *cast = (pm_rational_node_t *) node;
18157 cast->base.location.start--;
18158 cast->base.location.length++;
18159 cast->numerator.negative = true;
18160 break;
18161 }
18162 case PM_IMAGINARY_NODE:
18163 node->location.start--;
18164 node->location.length++;
18165 parse_negative_numeric(((pm_imaginary_node_t *) node)->numeric);
18166 break;
18167 default:
18168 PRISM_ASSERT(false && "unreachable");
18169 break;
18170 }
18171}
18172
18178static void
18179pm_parser_err_prefix(pm_parser_t *parser, pm_diagnostic_id_t diag_id) {
18180 switch (diag_id) {
18181 case PM_ERR_HASH_KEY: {
18182 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, diag_id, pm_token_str(parser->previous.type));
18183 break;
18184 }
18185 case PM_ERR_HASH_VALUE:
18186 case PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR: {
18187 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, pm_token_str(parser->current.type));
18188 break;
18189 }
18190 case PM_ERR_UNARY_RECEIVER: {
18191 const char *human = (parser->current.type == PM_TOKEN_EOF ? "end-of-input" : pm_token_str(parser->current.type));
18192 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, diag_id, human, parser->previous.start[0]);
18193 break;
18194 }
18195 case PM_ERR_UNARY_DISALLOWED:
18196 case PM_ERR_EXPECT_ARGUMENT: {
18197 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, pm_token_str(parser->current.type));
18198 break;
18199 }
18200 default:
18201 pm_parser_err_previous(parser, diag_id);
18202 break;
18203 }
18204}
18205
18209static void
18210ensure_valid_retry(pm_parser_t *parser, const pm_node_t *node) {
18211#define CONTEXT_NONE 0
18212#define CONTEXT_THROUGH_ENSURE 1
18213#define CONTEXT_THROUGH_ELSE 2
18214
18215 pm_context_node_t *context_node = parser->current_context;
18216 int context = CONTEXT_NONE;
18217
18218 while (context_node != NULL) {
18219 switch (context_node->context) {
18220 case PM_CONTEXT_BEGIN_RESCUE:
18221 case PM_CONTEXT_BLOCK_RESCUE:
18222 case PM_CONTEXT_CLASS_RESCUE:
18223 case PM_CONTEXT_DEF_RESCUE:
18224 case PM_CONTEXT_LAMBDA_RESCUE:
18225 case PM_CONTEXT_MODULE_RESCUE:
18226 case PM_CONTEXT_SCLASS_RESCUE:
18227 case PM_CONTEXT_DEFINED:
18228 case PM_CONTEXT_RESCUE_MODIFIER:
18229 // These are the good cases. We're allowed to have a retry here.
18230 return;
18231 case PM_CONTEXT_CLASS:
18232 case PM_CONTEXT_DEF:
18233 case PM_CONTEXT_DEF_PARAMS:
18234 case PM_CONTEXT_MAIN:
18235 case PM_CONTEXT_MODULE:
18236 case PM_CONTEXT_PREEXE:
18237 case PM_CONTEXT_SCLASS:
18238 // These are the bad cases. We're not allowed to have a retry in
18239 // these contexts.
18240 if (context == CONTEXT_NONE) {
18241 pm_parser_err_node(parser, node, PM_ERR_INVALID_RETRY_WITHOUT_RESCUE);
18242 } else if (context == CONTEXT_THROUGH_ENSURE) {
18243 pm_parser_err_node(parser, node, PM_ERR_INVALID_RETRY_AFTER_ENSURE);
18244 } else if (context == CONTEXT_THROUGH_ELSE) {
18245 pm_parser_err_node(parser, node, PM_ERR_INVALID_RETRY_AFTER_ELSE);
18246 }
18247 return;
18248 case PM_CONTEXT_BEGIN_ELSE:
18249 case PM_CONTEXT_BLOCK_ELSE:
18250 case PM_CONTEXT_CLASS_ELSE:
18251 case PM_CONTEXT_DEF_ELSE:
18252 case PM_CONTEXT_LAMBDA_ELSE:
18253 case PM_CONTEXT_MODULE_ELSE:
18254 case PM_CONTEXT_SCLASS_ELSE:
18255 // These are also bad cases, but with a more specific error
18256 // message indicating the else.
18257 context = CONTEXT_THROUGH_ELSE;
18258 break;
18259 case PM_CONTEXT_BEGIN_ENSURE:
18260 case PM_CONTEXT_BLOCK_ENSURE:
18261 case PM_CONTEXT_CLASS_ENSURE:
18262 case PM_CONTEXT_DEF_ENSURE:
18263 case PM_CONTEXT_LAMBDA_ENSURE:
18264 case PM_CONTEXT_MODULE_ENSURE:
18265 case PM_CONTEXT_SCLASS_ENSURE:
18266 // These are also bad cases, but with a more specific error
18267 // message indicating the ensure.
18268 context = CONTEXT_THROUGH_ENSURE;
18269 break;
18270 case PM_CONTEXT_NONE:
18271 case PM_CONTEXT_MAXIMUM:
18272 // This case should never happen.
18273 PRISM_ASSERT(false && "unreachable");
18274 break;
18275 case PM_CONTEXT_BEGIN:
18276 case PM_CONTEXT_BLOCK_BRACES:
18277 case PM_CONTEXT_BLOCK_KEYWORDS:
18278 case PM_CONTEXT_BLOCK_PARAMETERS:
18279 case PM_CONTEXT_CASE_IN:
18280 case PM_CONTEXT_CASE_WHEN:
18281 case PM_CONTEXT_DEFAULT_PARAMS:
18282 case PM_CONTEXT_ELSE:
18283 case PM_CONTEXT_ELSIF:
18284 case PM_CONTEXT_EMBEXPR:
18285 case PM_CONTEXT_FOR_INDEX:
18286 case PM_CONTEXT_FOR:
18287 case PM_CONTEXT_IF:
18288 case PM_CONTEXT_LAMBDA_BRACES:
18289 case PM_CONTEXT_LAMBDA_DO_END:
18290 case PM_CONTEXT_LOOP_PREDICATE:
18291 case PM_CONTEXT_MULTI_TARGET:
18292 case PM_CONTEXT_PARENS:
18293 case PM_CONTEXT_POSTEXE:
18294 case PM_CONTEXT_PREDICATE:
18295 case PM_CONTEXT_TERNARY:
18296 case PM_CONTEXT_UNLESS:
18297 case PM_CONTEXT_UNTIL:
18298 case PM_CONTEXT_WHILE:
18299 // In these contexts we should continue walking up the list of
18300 // contexts.
18301 break;
18302 }
18303
18304 context_node = context_node->prev;
18305 }
18306
18307#undef CONTEXT_NONE
18308#undef CONTEXT_ENSURE
18309#undef CONTEXT_ELSE
18310}
18311
18315static void
18316ensure_valid_yield(pm_parser_t *parser, const pm_yield_node_t *node) {
18317 pm_context_node_t *context_node = parser->current_context;
18318
18319 while (context_node != NULL) {
18320 switch (context_node->context) {
18321 case PM_CONTEXT_DEF:
18322 case PM_CONTEXT_DEF_PARAMS:
18323 case PM_CONTEXT_DEFINED:
18324 case PM_CONTEXT_DEF_ENSURE:
18325 case PM_CONTEXT_DEF_RESCUE:
18326 case PM_CONTEXT_DEF_ELSE:
18327 // These are the good cases. We're allowed to have a block exit
18328 // in these contexts.
18329 return;
18330 case PM_CONTEXT_CLASS:
18331 case PM_CONTEXT_CLASS_ENSURE:
18332 case PM_CONTEXT_CLASS_RESCUE:
18333 case PM_CONTEXT_CLASS_ELSE:
18334 case PM_CONTEXT_MAIN:
18335 case PM_CONTEXT_MODULE:
18336 case PM_CONTEXT_MODULE_ENSURE:
18337 case PM_CONTEXT_MODULE_RESCUE:
18338 case PM_CONTEXT_MODULE_ELSE:
18339 case PM_CONTEXT_SCLASS:
18340 case PM_CONTEXT_SCLASS_RESCUE:
18341 case PM_CONTEXT_SCLASS_ENSURE:
18342 case PM_CONTEXT_SCLASS_ELSE:
18343 // These are the bad cases. We're not allowed to have a retry in
18344 // these contexts.
18345 pm_parser_err_node(parser, UP(node), PM_ERR_INVALID_YIELD);
18346 return;
18347 case PM_CONTEXT_NONE:
18348 case PM_CONTEXT_MAXIMUM:
18349 // This case should never happen.
18350 PRISM_ASSERT(false && "unreachable");
18351 break;
18352 case PM_CONTEXT_BEGIN:
18353 case PM_CONTEXT_BEGIN_ELSE:
18354 case PM_CONTEXT_BEGIN_ENSURE:
18355 case PM_CONTEXT_BEGIN_RESCUE:
18356 case PM_CONTEXT_BLOCK_BRACES:
18357 case PM_CONTEXT_BLOCK_KEYWORDS:
18358 case PM_CONTEXT_BLOCK_ELSE:
18359 case PM_CONTEXT_BLOCK_ENSURE:
18360 case PM_CONTEXT_BLOCK_PARAMETERS:
18361 case PM_CONTEXT_BLOCK_RESCUE:
18362 case PM_CONTEXT_CASE_IN:
18363 case PM_CONTEXT_CASE_WHEN:
18364 case PM_CONTEXT_DEFAULT_PARAMS:
18365 case PM_CONTEXT_ELSE:
18366 case PM_CONTEXT_ELSIF:
18367 case PM_CONTEXT_EMBEXPR:
18368 case PM_CONTEXT_FOR_INDEX:
18369 case PM_CONTEXT_FOR:
18370 case PM_CONTEXT_IF:
18371 case PM_CONTEXT_LAMBDA_BRACES:
18372 case PM_CONTEXT_LAMBDA_DO_END:
18373 case PM_CONTEXT_LAMBDA_ELSE:
18374 case PM_CONTEXT_LAMBDA_ENSURE:
18375 case PM_CONTEXT_LAMBDA_RESCUE:
18376 case PM_CONTEXT_LOOP_PREDICATE:
18377 case PM_CONTEXT_MULTI_TARGET:
18378 case PM_CONTEXT_PARENS:
18379 case PM_CONTEXT_POSTEXE:
18380 case PM_CONTEXT_PREDICATE:
18381 case PM_CONTEXT_PREEXE:
18382 case PM_CONTEXT_RESCUE_MODIFIER:
18383 case PM_CONTEXT_TERNARY:
18384 case PM_CONTEXT_UNLESS:
18385 case PM_CONTEXT_UNTIL:
18386 case PM_CONTEXT_WHILE:
18387 // In these contexts we should continue walking up the list of
18388 // contexts.
18389 break;
18390 }
18391
18392 context_node = context_node->prev;
18393 }
18394}
18395
18400static pm_node_t *
18401parse_case(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
18402 size_t opening_newline_index = token_newline_index(parser);
18403 parser_lex(parser);
18404
18405 pm_token_t case_keyword = parser->previous;
18406 pm_node_t *predicate = NULL;
18407
18408 pm_scratch_node_list_t current_block_exits = { 0 };
18409 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
18410
18411 if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) {
18412 while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON));
18413 predicate = NULL;
18414 } else if (match3(parser, PM_TOKEN_KEYWORD_WHEN, PM_TOKEN_KEYWORD_IN, PM_TOKEN_KEYWORD_END)) {
18415 predicate = NULL;
18416 } else if (!token_begins_expression_p(parser->current.type)) {
18417 predicate = NULL;
18418 } else {
18419 predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CASE_EXPRESSION_AFTER_CASE, (uint16_t) (depth + 1));
18420 while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON));
18421 }
18422
18423 if (match1(parser, PM_TOKEN_KEYWORD_END)) {
18424 parser_warn_indentation_mismatch(parser, opening_newline_index, &case_keyword, false, false);
18425 parser_lex(parser);
18426 pop_block_exits(parser, previous_block_exits);
18427 pm_parser_err_token(parser, &case_keyword, PM_ERR_CASE_MISSING_CONDITIONS);
18428
18429 pm_node_list_t conditions = { 0 };
18430 return UP(pm_case_node_create(parser, &case_keyword, predicate, &conditions, NULL, &parser->previous));
18431 }
18432
18433 /* The keyword after the predicate decides whether this is a case-when or a
18434 * case-in node, and so whether the conditions are when or in nodes. */
18435 pm_node_list_t conditions = { 0 };
18436 bool case_when = match1(parser, PM_TOKEN_KEYWORD_WHEN);
18437
18438 if (case_when) {
18439 pm_static_literals_t literals = { 0 };
18440
18441 /* At this point we've seen a when keyword, so we know this is a
18442 * case-when node. We will continue to parse the when nodes until we hit
18443 * the end of the list. */
18444 while (match1(parser, PM_TOKEN_KEYWORD_WHEN)) {
18445 parser_warn_indentation_mismatch(parser, opening_newline_index, &case_keyword, false, true);
18446 parser_lex(parser);
18447
18448 pm_token_t when_keyword = parser->previous;
18449 pm_node_list_t when_conditions = { 0 };
18450 pm_token_t then_keyword = { 0 };
18451 pm_statements_node_t *statements = NULL;
18452
18453 do {
18454 if (accept1(parser, PM_TOKEN_USTAR)) {
18455 pm_token_t operator = parser->previous;
18456 pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_STAR, (uint16_t) (depth + 1));
18457
18458 pm_splat_node_t *splat_node = pm_splat_node_create(parser, &operator, expression);
18459 pm_node_list_open_append(&parser->node_chunks, &when_conditions, UP(splat_node));
18460
18461 if (PM_NODE_TYPE_P(expression, PM_ERROR_RECOVERY_NODE)) break;
18462 } else {
18463 pm_node_t *condition = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_CASE_EXPRESSION_AFTER_WHEN, (uint16_t) (depth + 1));
18464 pm_node_list_open_append(&parser->node_chunks, &when_conditions, condition);
18465
18466 /* If we found a missing node, then this is a syntax error
18467 * and we should stop looping. */
18468 if (PM_NODE_TYPE_P(condition, PM_ERROR_RECOVERY_NODE)) break;
18469
18470 /* If this is a string node, then we need to mark it as
18471 * frozen because when clause strings are frozen. */
18472 if (PM_NODE_TYPE_P(condition, PM_STRING_NODE)) {
18473 pm_node_flag_set(condition, PM_STRING_FLAGS_FROZEN | PM_NODE_FLAG_STATIC_LITERAL);
18474 } else if (PM_NODE_TYPE_P(condition, PM_SOURCE_FILE_NODE) && parser->version < PM_OPTIONS_VERSION_CRUBY_4_1) {
18475 pm_node_flag_set(condition, PM_NODE_FLAG_STATIC_LITERAL);
18476 }
18477
18478 pm_when_clause_static_literals_add(parser, &literals, condition);
18479 }
18480 } while (accept1(parser, PM_TOKEN_COMMA));
18481
18482 if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) {
18483 if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) {
18484 then_keyword = parser->previous;
18485 }
18486 } else {
18487 expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_WHEN_DELIMITER);
18488 then_keyword = parser->previous;
18489 }
18490
18491 if (!match3(parser, PM_TOKEN_KEYWORD_WHEN, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
18492 statements = parse_statements(parser, PM_CONTEXT_CASE_WHEN, (uint16_t) (depth + 1));
18493 }
18494
18495 pm_when_node_t *when_node = pm_when_node_create(parser, &when_keyword, &when_conditions, NTOK2PTR(then_keyword), statements);
18496 pm_node_list_open_append(&parser->node_chunks, &conditions, UP(when_node));
18497 }
18498
18499 pm_static_literals_free(&literals);
18500 } else {
18501 /* If this is a case-match node (i.e., it is a pattern matching case
18502 * statement) then we must have a predicate. */
18503 if (predicate == NULL) {
18504 pm_parser_err_token(parser, &case_keyword, PM_ERR_CASE_MATCH_MISSING_PREDICATE);
18505 }
18506
18507 /* At this point we expect that we're parsing a case-in node. We will
18508 * continue to parse the in nodes until we hit the end of the list. */
18509 while (match1(parser, PM_TOKEN_KEYWORD_IN)) {
18510 parser_warn_indentation_mismatch(parser, opening_newline_index, &case_keyword, false, true);
18511
18512 bool previous_pattern_matching_newlines = parser->pattern_matching_newlines;
18513 parser->pattern_matching_newlines = true;
18514
18515 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
18516 parser->command_start = false;
18517 parser_lex(parser);
18518
18519 pm_token_t in_keyword = parser->previous;
18520 pm_constant_id_set_t captures = { 0 };
18521 pm_node_t *pattern;
18522 pm_token_t then_keyword = { 0 };
18523 pm_statements_node_t *statements = NULL;
18524
18525 pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_IN, (uint16_t) (depth + 1));
18526 parser->pattern_matching_newlines = previous_pattern_matching_newlines;
18527
18528 /* Since we're in the top-level of the case-in node we need to check
18529 * for guard clauses in the form of `if` or `unless` statements. */
18530 if (accept1(parser, PM_TOKEN_KEYWORD_IF_MODIFIER)) {
18531 pm_token_t keyword = parser->previous;
18532 pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_IF_PREDICATE, (uint16_t) (depth + 1));
18533 pattern = UP(pm_if_node_modifier_create(parser, pattern, &keyword, predicate));
18534 } else if (accept1(parser, PM_TOKEN_KEYWORD_UNLESS_MODIFIER)) {
18535 pm_token_t keyword = parser->previous;
18536 pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_UNLESS_PREDICATE, (uint16_t) (depth + 1));
18537 pattern = UP(pm_unless_node_modifier_create(parser, pattern, &keyword, predicate));
18538 }
18539
18540 if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) {
18541 if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) {
18542 then_keyword = parser->previous;
18543 }
18544 } else {
18545 expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_IN_DELIMITER);
18546 then_keyword = parser->previous;
18547 }
18548
18549 if (!match3(parser, PM_TOKEN_KEYWORD_IN, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
18550 statements = parse_statements(parser, PM_CONTEXT_CASE_IN, (uint16_t) (depth + 1));
18551 }
18552
18553 pm_node_t *condition = UP(pm_in_node_create(parser, pattern, statements, &in_keyword, NTOK2PTR(then_keyword)));
18554 pm_node_list_open_append(&parser->node_chunks, &conditions, condition);
18555 }
18556 }
18557
18558 /* If we didn't parse any conditions (in or when) then we need to indicate
18559 * that we have an error. */
18560 if (conditions.size == 0) {
18561 pm_parser_err_token(parser, &case_keyword, PM_ERR_CASE_MISSING_CONDITIONS);
18562 }
18563
18564 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
18565 pm_else_node_t *else_clause = NULL;
18566
18567 if (accept1(parser, PM_TOKEN_KEYWORD_ELSE)) {
18568 pm_token_t else_keyword = parser->previous;
18569
18570 if (!match1(parser, PM_TOKEN_KEYWORD_END)) {
18571 else_clause = pm_else_node_create(parser, &else_keyword, parse_statements(parser, PM_CONTEXT_ELSE, (uint16_t) (depth + 1)), &parser->current);
18572 } else {
18573 else_clause = pm_else_node_create(parser, &else_keyword, NULL, &parser->current);
18574 }
18575 }
18576
18577 parser_warn_indentation_mismatch(parser, opening_newline_index, &case_keyword, false, false);
18578 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CASE_TERM, &case_keyword);
18579
18580 pm_node_t *node;
18581 if (case_when) {
18582 node = UP(pm_case_node_create(parser, &case_keyword, predicate, &conditions, else_clause, &parser->previous));
18583 } else {
18584 node = UP(pm_case_match_node_create(parser, &case_keyword, predicate, &conditions, else_clause, &parser->previous));
18585 }
18586
18587 pop_block_exits(parser, previous_block_exits);
18588 return node;
18589}
18590
18595static pm_node_t *
18596parse_class(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
18597 size_t opening_newline_index = token_newline_index(parser);
18598 parser_lex(parser);
18599
18600 pm_token_t class_keyword = parser->previous;
18601
18602 pm_scratch_node_list_t current_block_exits = { 0 };
18603 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
18604
18605 if (accept1(parser, PM_TOKEN_LESS_LESS)) {
18606 pm_token_t operator = parser->previous;
18607 pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS, (uint16_t) (depth + 1));
18608
18609 /* The header uses the enclosing loop context. The body starts a new one. */
18610 pm_do_loop_stack_push(parser, false);
18611 pm_parser_scope_push(parser, true);
18612 if (!match2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) {
18613 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_SINGLETON_CLASS_DELIMITER, pm_token_str(parser->current.type));
18614 }
18615
18616 pm_node_t *statements = NULL;
18617 if (!match4(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
18618 pm_accepts_block_stack_push(parser, true);
18619 statements = UP(parse_statements(parser, PM_CONTEXT_SCLASS, (uint16_t) (depth + 1)));
18620 pm_accepts_block_stack_pop(parser);
18621 }
18622
18623 if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
18624 PRISM_ASSERT(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
18625 statements = UP(parse_rescues_implicit_begin(parser, opening_newline_index, &class_keyword, class_keyword.start, (pm_statements_node_t *) statements, PM_RESCUES_SCLASS, (uint16_t) (depth + 1)));
18626 } else {
18627 parser_warn_indentation_mismatch(parser, opening_newline_index, &class_keyword, false, false);
18628 }
18629
18630 pm_do_loop_stack_pop(parser);
18631 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CLASS_TERM, &class_keyword);
18632
18633 pm_constant_id_list_t locals;
18634 pm_locals_order(parser, &locals, false);
18635
18636 pm_parser_scope_pop(parser);
18637
18638 flush_block_exits(parser, previous_block_exits);
18639 return UP(pm_singleton_class_node_create(parser, &locals, &class_keyword, &operator, expression, statements, &parser->previous));
18640 }
18641
18642 pm_node_t *constant_path = parse_expression(parser, PM_BINDING_POWER_INDEX, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_CLASS_NAME, (uint16_t) (depth + 1));
18643 pm_token_t name = parser->previous;
18644 if (name.type != PM_TOKEN_CONSTANT) {
18645 pm_parser_err_token(parser, &name, PM_ERR_CLASS_NAME);
18646 }
18647
18648 pm_token_t inheritance_operator = { 0 };
18649 pm_node_t *superclass;
18650
18651 if (match1(parser, PM_TOKEN_LESS)) {
18652 inheritance_operator = parser->current;
18653 lex_state_set(parser, PM_LEX_STATE_BEG);
18654
18655 parser->command_start = true;
18656 parser_lex(parser);
18657
18658 superclass = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CLASS_SUPERCLASS, (uint16_t) (depth + 1));
18659 } else {
18660 superclass = NULL;
18661 }
18662
18663 pm_do_loop_stack_push(parser, false);
18664 pm_parser_scope_push(parser, true);
18665
18666 if (inheritance_operator.start != NULL) {
18667 expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CLASS_UNEXPECTED_END);
18668 } else {
18669 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
18670 }
18671 pm_node_t *statements = NULL;
18672
18673 if (!match4(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
18674 pm_accepts_block_stack_push(parser, true);
18675 statements = UP(parse_statements(parser, PM_CONTEXT_CLASS, (uint16_t) (depth + 1)));
18676 pm_accepts_block_stack_pop(parser);
18677 }
18678
18679 if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
18680 PRISM_ASSERT(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
18681 statements = UP(parse_rescues_implicit_begin(parser, opening_newline_index, &class_keyword, class_keyword.start, (pm_statements_node_t *) statements, PM_RESCUES_CLASS, (uint16_t) (depth + 1)));
18682 } else {
18683 parser_warn_indentation_mismatch(parser, opening_newline_index, &class_keyword, false, false);
18684 }
18685
18686 pm_do_loop_stack_pop(parser);
18687 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CLASS_TERM, &class_keyword);
18688
18689 if (context_def_p(parser)) {
18690 pm_parser_err_token(parser, &class_keyword, PM_ERR_CLASS_IN_METHOD);
18691 }
18692
18693 pm_constant_id_list_t locals;
18694 pm_locals_order(parser, &locals, false);
18695
18696 pm_parser_scope_pop(parser);
18697
18698 if (!PM_NODE_TYPE_P(constant_path, PM_CONSTANT_PATH_NODE) && !(PM_NODE_TYPE_P(constant_path, PM_CONSTANT_READ_NODE))) {
18699 pm_parser_err_node(parser, constant_path, PM_ERR_CLASS_NAME);
18700 if (!PM_NODE_TYPE_P(constant_path, PM_ERROR_RECOVERY_NODE)) {
18701 constant_path = UP(pm_error_recovery_node_create_unexpected(parser, constant_path));
18702 }
18703 }
18704
18705 pop_block_exits(parser, previous_block_exits);
18706 return UP(pm_class_node_create(parser, &locals, &class_keyword, constant_path, &name, NTOK2PTR(inheritance_operator), superclass, statements, &parser->previous));
18707}
18708
18712static pm_node_t *
18713parse_def(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
18714 pm_scratch_node_list_t current_block_exits = { 0 };
18715 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
18716
18717 pm_token_t def_keyword = parser->current;
18718 size_t opening_newline_index = token_newline_index(parser);
18719
18720 pm_node_t *receiver = NULL;
18721 pm_token_t operator = { 0 };
18722 pm_token_t name;
18723
18724 /* This context is necessary for lexing `...` in a bare params correctly. It
18725 * must be pushed before lexing the first param, so it is here. */
18726 context_push(parser, PM_CONTEXT_DEF_PARAMS);
18727 parser_lex(parser);
18728
18729 /* This will be false if the method name is not a valid identifier but could
18730 * be followed by an operator. */
18731 bool valid_name = true;
18732
18733 switch (parser->current.type) {
18734 case PM_CASE_OPERATOR:
18735 pm_parser_scope_push(parser, true);
18736 lex_state_set(parser, PM_LEX_STATE_ENDFN);
18737 parser_lex(parser);
18738
18739 name = parser->previous;
18740 break;
18741 case PM_TOKEN_IDENTIFIER: {
18742 parser_lex(parser);
18743
18744 if (match2(parser, PM_TOKEN_DOT, PM_TOKEN_COLON_COLON)) {
18745 if (parser->previous.end[-1] == '=') {
18746 pm_parser_err_previous(parser, PM_ERR_DEF_RECEIVER_SETTER);
18747 }
18748 receiver = parse_variable_call(parser);
18749
18750 pm_parser_scope_push(parser, true);
18751 lex_state_set(parser, PM_LEX_STATE_FNAME);
18752 parser_lex(parser);
18753
18754 operator = parser->previous;
18755 name = parse_method_definition_name(parser);
18756 } else {
18757 pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous));
18758 pm_parser_scope_push(parser, true);
18759
18760 name = parser->previous;
18761 }
18762
18763 break;
18764 }
18765 case PM_TOKEN_INSTANCE_VARIABLE:
18766 case PM_TOKEN_CLASS_VARIABLE:
18767 case PM_TOKEN_GLOBAL_VARIABLE:
18768 valid_name = false;
18770 case PM_TOKEN_CONSTANT:
18771 case PM_TOKEN_KEYWORD_NIL:
18772 case PM_TOKEN_KEYWORD_SELF:
18773 case PM_TOKEN_KEYWORD_TRUE:
18774 case PM_TOKEN_KEYWORD_FALSE:
18775 case PM_TOKEN_KEYWORD___FILE__:
18776 case PM_TOKEN_KEYWORD___LINE__:
18777 case PM_TOKEN_KEYWORD___ENCODING__: {
18778 pm_parser_scope_push(parser, true);
18779 parser_lex(parser);
18780
18781 pm_token_t identifier = parser->previous;
18782
18783 if (match2(parser, PM_TOKEN_DOT, PM_TOKEN_COLON_COLON)) {
18784 lex_state_set(parser, PM_LEX_STATE_FNAME);
18785 parser_lex(parser);
18786 operator = parser->previous;
18787
18788 switch (identifier.type) {
18789 case PM_TOKEN_CONSTANT:
18790 receiver = UP(pm_constant_read_node_create(parser, &identifier));
18791 break;
18792 case PM_TOKEN_INSTANCE_VARIABLE:
18793 receiver = UP(pm_instance_variable_read_node_create(parser, &identifier));
18794 break;
18795 case PM_TOKEN_CLASS_VARIABLE:
18796 receiver = UP(pm_class_variable_read_node_create(parser, &identifier));
18797 break;
18798 case PM_TOKEN_GLOBAL_VARIABLE:
18799 receiver = UP(pm_global_variable_read_node_create(parser, &identifier));
18800 break;
18801 case PM_TOKEN_KEYWORD_NIL:
18802 receiver = UP(pm_nil_node_create(parser, &identifier));
18803 break;
18804 case PM_TOKEN_KEYWORD_SELF:
18805 receiver = UP(pm_self_node_create(parser, &identifier));
18806 break;
18807 case PM_TOKEN_KEYWORD_TRUE:
18808 receiver = UP(pm_true_node_create(parser, &identifier));
18809 break;
18810 case PM_TOKEN_KEYWORD_FALSE:
18811 receiver = UP(pm_false_node_create(parser, &identifier));
18812 break;
18813 case PM_TOKEN_KEYWORD___FILE__:
18814 receiver = UP(pm_source_file_node_create(parser, &identifier));
18815 break;
18816 case PM_TOKEN_KEYWORD___LINE__:
18817 receiver = UP(pm_source_line_node_create(parser, &identifier));
18818 break;
18819 case PM_TOKEN_KEYWORD___ENCODING__:
18820 receiver = UP(pm_source_encoding_node_create(parser, &identifier));
18821 break;
18822 default:
18823 break;
18824 }
18825
18826 name = parse_method_definition_name(parser);
18827 } else {
18828 if (!valid_name) {
18829 PM_PARSER_ERR_TOKEN_FORMAT(parser, &identifier, PM_ERR_DEF_NAME, pm_token_str(identifier.type));
18830 }
18831
18832 name = identifier;
18833 }
18834 break;
18835 }
18836 case PM_TOKEN_PARENTHESIS_LEFT: {
18837 /* The current context is `PM_CONTEXT_DEF_PARAMS`, however the inner
18838 * expression of this parenthesis should not be processed under this
18839 * context. Thus, the context is popped here. */
18840 context_pop(parser);
18841 lex_state_set(parser, PM_LEX_STATE_BEG);
18842 parser_lex(parser);
18843
18844 pm_token_t lparen = parser->previous;
18845 pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_DEF_RECEIVER, (uint16_t) (depth + 1));
18846
18847 accept1(parser, PM_TOKEN_NEWLINE);
18848 expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
18849 pm_token_t rparen = parser->previous;
18850
18851 lex_state_set(parser, PM_LEX_STATE_FNAME);
18852 expect2(parser, PM_TOKEN_DOT, PM_TOKEN_COLON_COLON, PM_ERR_DEF_RECEIVER_TERM);
18853
18854 operator = parser->previous;
18855 receiver = UP(pm_parentheses_node_create(parser, &lparen, expression, &rparen, 0));
18856
18857 /* To push `PM_CONTEXT_DEF_PARAMS` again is for the same reason as
18858 * described the above. */
18859 pm_parser_scope_push(parser, true);
18860 context_push(parser, PM_CONTEXT_DEF_PARAMS);
18861 name = parse_method_definition_name(parser);
18862 break;
18863 }
18864 default:
18865 pm_parser_scope_push(parser, true);
18866 name = parse_method_definition_name(parser);
18867 break;
18868 }
18869
18870 pm_token_t lparen = { 0 };
18871 pm_token_t rparen = { 0 };
18872 pm_parameters_node_t *params;
18873
18874 bool accept_endless_def = true;
18875 switch (parser->current.type) {
18876 case PM_TOKEN_PARENTHESIS_LEFT: {
18877 parser_lex(parser);
18878 lparen = parser->previous;
18879
18880 if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
18881 params = NULL;
18882 } else {
18883 /* https://bugs.ruby-lang.org/issues/19107 */
18884 bool allow_trailing_comma = parser->version >= PM_OPTIONS_VERSION_CRUBY_4_1;
18885 params = parse_parameters(
18886 parser,
18887 PM_BINDING_POWER_DEFINED,
18888 true,
18889 allow_trailing_comma,
18890 true,
18891 true,
18892 false,
18893 PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
18894 (uint16_t) (depth + 1)
18895 );
18896 }
18897
18898 lex_state_set(parser, PM_LEX_STATE_BEG);
18899 parser->command_start = true;
18900
18901 context_pop(parser);
18902 if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
18903 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_str(parser->current.type));
18904 parser->previous.start = parser->previous.end;
18905 parser->previous.type = 0;
18906 }
18907
18908 rparen = parser->previous;
18909 break;
18910 }
18911 case PM_CASE_PARAMETER: {
18912 /* If we're about to lex a label, we need to add the label state to
18913 * make sure the next newline is ignored. */
18914 if (parser->current.type == PM_TOKEN_LABEL) {
18915 lex_state_set(parser, parser->lex_state | PM_LEX_STATE_LABEL);
18916 }
18917
18918 params = parse_parameters(
18919 parser,
18920 PM_BINDING_POWER_DEFINED,
18921 false,
18922 false,
18923 true,
18924 true,
18925 false,
18926 PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
18927 (uint16_t) (depth + 1)
18928 );
18929
18930 /* Reject `def * = 1` and similar. We have to specifically check for
18931 * them because they create ambiguity with optional arguments. */
18932 accept_endless_def = false;
18933
18934 context_pop(parser);
18935 break;
18936 }
18937 default: {
18938 params = NULL;
18939 context_pop(parser);
18940 break;
18941 }
18942 }
18943
18944 pm_node_t *statements = NULL;
18945 pm_token_t equal = { 0 };
18946 pm_token_t end_keyword = { 0 };
18947
18948 if (accept1(parser, PM_TOKEN_EQUAL)) {
18949 if (token_is_setter_name(&name)) {
18950 pm_parser_err_token(parser, &name, PM_ERR_DEF_ENDLESS_SETTER);
18951 }
18952 if (!accept_endless_def) {
18953 pm_parser_err_previous(parser, PM_ERR_DEF_ENDLESS_PARAMETERS);
18954 }
18955 if (
18956 parser->current_context->context == PM_CONTEXT_DEFAULT_PARAMS &&
18957 parser->current_context->prev->context == PM_CONTEXT_BLOCK_PARAMETERS
18958 ) {
18959 PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_START(parser, &def_keyword), PM_TOKENS_LENGTH(&def_keyword, &parser->previous), PM_ERR_UNEXPECTED_PARAMETER_DEFAULT_VALUE, "endless method definition");
18960 }
18961 equal = parser->previous;
18962
18963 context_push(parser, PM_CONTEXT_DEF);
18964 pm_do_loop_stack_push(parser, false);
18965
18966 /* The body accepts a command call (`def f = foo 1`) only where the
18967 * caller accepts an endless method with a command body. */
18968 uint8_t allow_flags = (flags & PM_PARSE_ACCEPTS_COMMAND_DEF) ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0;
18969
18970 /* Inside a def body, we push true onto the accepts_block_stack so that
18971 * `do` is lexed as PM_TOKEN_KEYWORD_DO (which can only start a block
18972 * for primary-level constructs, not commands). During command argument
18973 * parsing, the stack is pushed to false, causing `do` to be lexed as
18974 * PM_TOKEN_KEYWORD_DO_BLOCK, which is not consumed inside the endless
18975 * def body and instead left for the outer context. A method definition
18976 * opens a fresh context all the way through its rescue modifier, so
18977 * this frame spans the rescue modifier value as well: the `do` in
18978 * `baz def f = a rescue z do end` lexes as a plain keyword that
18979 * attaches to `z` rather than to `baz`. */
18980 pm_accepts_block_stack_push(parser, true);
18981 pm_node_t *statement = parse_expression(parser, PM_BINDING_POWER_DEFINED + 1, allow_flags | PM_PARSE_IN_ENDLESS_DEF, PM_ERR_DEF_ENDLESS, (uint16_t) (depth + 1));
18982
18983 /* Any number of rescue modifiers chain onto the body within the method
18984 * definition itself, associating to the left: `def f = a rescue b
18985 * rescue c` defines a method whose body is `(a rescue b) rescue c`,
18986 * rather than a rescue modifier guarding the definition. */
18987 while (accept1(parser, PM_TOKEN_KEYWORD_RESCUE_MODIFIER)) {
18988 context_push(parser, PM_CONTEXT_RESCUE_MODIFIER);
18989
18990 pm_token_t rescue_keyword = parser->previous;
18991
18992 /* In the Ruby grammar, the rescue value of an endless method
18993 * command excludes and/or and in/=>. */
18994 pm_node_t *value = parse_expression(parser, PM_BINDING_POWER_MATCH + 1, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
18995 context_pop(parser);
18996
18997 statement = UP(pm_rescue_modifier_node_create(parser, statement, &rescue_keyword, value));
18998 }
18999
19000 /* A plain `do` still belongs to the endless method context. A
19001 * PM_TOKEN_KEYWORD_DO_BLOCK can attach to an enclosing command. */
19002 if (accept1(parser, PM_TOKEN_KEYWORD_DO)) {
19003 pm_block_node_t *block = parse_block(parser, (uint16_t) (depth + 1));
19004 pm_parser_err_node(parser, UP(block), PM_ERR_DEF_ENDLESS_DO_BLOCK);
19005 }
19006
19007 pm_accepts_block_stack_pop(parser);
19008
19009 /* A nested endless def whose body is a command call (e.g.,
19010 * `def f = def g = foo bar`) is a command write and cannot appear
19011 * as a def body. */
19012 if (PM_NODE_TYPE_P(statement, PM_DEF_NODE) && pm_command_call_value_p(parser, statement)) {
19013 PM_PARSER_ERR_NODE_FORMAT(parser, statement, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->current.type));
19014 }
19015
19016 statements = UP(pm_statements_node_single_create(parser, statement, false));
19017 pm_do_loop_stack_pop(parser);
19018 context_pop(parser);
19019 } else {
19020 if (lparen.start == NULL) {
19021 lex_state_set(parser, PM_LEX_STATE_BEG);
19022 parser->command_start = true;
19023 expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_DEF_PARAMS_TERM);
19024 } else {
19025 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
19026 }
19027
19028 pm_accepts_block_stack_push(parser, true);
19029 pm_do_loop_stack_push(parser, false);
19030
19031 if (!match4(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
19032 pm_accepts_block_stack_push(parser, true);
19033 statements = UP(parse_statements(parser, PM_CONTEXT_DEF, (uint16_t) (depth + 1)));
19034 pm_accepts_block_stack_pop(parser);
19035 }
19036
19037 if (match3(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_ELSE)) {
19038 PRISM_ASSERT(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
19039 context_push(parser, PM_CONTEXT_DEF);
19040 statements = UP(parse_rescues_implicit_begin(parser, opening_newline_index, &def_keyword, def_keyword.start, (pm_statements_node_t *) statements, PM_RESCUES_DEF, (uint16_t) (depth + 1)));
19041 context_pop(parser);
19042 } else {
19043 parser_warn_indentation_mismatch(parser, opening_newline_index, &def_keyword, false, false);
19044 }
19045
19046 pm_accepts_block_stack_pop(parser);
19047 pm_do_loop_stack_pop(parser);
19048
19049 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_DEF_TERM, &def_keyword);
19050 end_keyword = parser->previous;
19051 }
19052
19053 pm_constant_id_list_t locals;
19054 pm_locals_order(parser, &locals, false);
19055 pm_parser_scope_pop(parser);
19056
19057 /* If the final character is `@` as is the case when defining methods to
19058 * override the unary operators, we should ignore the @ in the same way we
19059 * do for symbols. */
19060 pm_constant_id_t name_id = pm_parser_constant_id_raw(parser, name.start, parse_operator_symbol_name(&name));
19061
19062 flush_block_exits(parser, previous_block_exits);
19063
19064 return UP(pm_def_node_create(
19065 parser,
19066 name_id,
19067 &name,
19068 receiver,
19069 params,
19070 statements,
19071 &locals,
19072 &def_keyword,
19073 NTOK2PTR(operator),
19074 NTOK2PTR(lparen),
19075 NTOK2PTR(rparen),
19076 NTOK2PTR(equal),
19077 NTOK2PTR(end_keyword)
19078 ));
19079}
19080
19084static pm_node_t *
19085parse_module(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
19086 pm_scratch_node_list_t current_block_exits = { 0 };
19087 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
19088
19089 size_t opening_newline_index = token_newline_index(parser);
19090 parser_lex(parser);
19091 pm_token_t module_keyword = parser->previous;
19092
19093 pm_node_t *constant_path = parse_expression(parser, PM_BINDING_POWER_INDEX, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_MODULE_NAME, (uint16_t) (depth + 1));
19094 pm_token_t name;
19095
19096 /* If we can recover from a syntax error that occurred while parsing the
19097 * name of the module, then we'll handle that here. */
19098 if (PM_NODE_TYPE_P(constant_path, PM_ERROR_RECOVERY_NODE)) {
19099 pop_block_exits(parser, previous_block_exits);
19100
19101 pm_token_t missing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
19102 return UP(pm_module_node_create(parser, NULL, &module_keyword, constant_path, &missing, NULL, &missing));
19103 }
19104
19105 while (accept1(parser, PM_TOKEN_COLON_COLON)) {
19106 pm_token_t double_colon = parser->previous;
19107
19108 expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT);
19109 constant_path = UP(pm_constant_path_node_create(parser, constant_path, &double_colon, &parser->previous));
19110 }
19111
19112 /* Here we retrieve the name of the module. If it wasn't a constant, then
19113 * it's possible that `module foo` was passed, which is a syntax error. We
19114 * handle that here as well. */
19115 name = parser->previous;
19116 if (name.type != PM_TOKEN_CONSTANT) {
19117 pm_parser_err_token(parser, &name, PM_ERR_MODULE_NAME);
19118 }
19119
19120 if (!PM_NODE_TYPE_P(constant_path, PM_CONSTANT_READ_NODE) && !PM_NODE_TYPE_P(constant_path, PM_CONSTANT_PATH_NODE) && !PM_NODE_TYPE_P(constant_path, PM_ERROR_RECOVERY_NODE)) {
19121 if (name.type == PM_TOKEN_CONSTANT) pm_parser_err_node(parser, constant_path, PM_ERR_MODULE_NAME);
19122 constant_path = UP(pm_error_recovery_node_create_unexpected(parser, constant_path));
19123 }
19124
19125 pm_parser_scope_push(parser, true);
19126 accept2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE);
19127 pm_node_t *statements = NULL;
19128
19129 if (!match4(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
19130 pm_accepts_block_stack_push(parser, true);
19131 statements = UP(parse_statements(parser, PM_CONTEXT_MODULE, (uint16_t) (depth + 1)));
19132 pm_accepts_block_stack_pop(parser);
19133 }
19134
19135 if (match3(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_ELSE)) {
19136 PRISM_ASSERT(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
19137 statements = UP(parse_rescues_implicit_begin(parser, opening_newline_index, &module_keyword, module_keyword.start, (pm_statements_node_t *) statements, PM_RESCUES_MODULE, (uint16_t) (depth + 1)));
19138 } else {
19139 parser_warn_indentation_mismatch(parser, opening_newline_index, &module_keyword, false, false);
19140 }
19141
19142 pm_constant_id_list_t locals;
19143 pm_locals_order(parser, &locals, false);
19144
19145 pm_parser_scope_pop(parser);
19146 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_MODULE_TERM, &module_keyword);
19147
19148 if (context_def_p(parser)) {
19149 pm_parser_err_token(parser, &module_keyword, PM_ERR_MODULE_IN_METHOD);
19150 }
19151
19152 pop_block_exits(parser, previous_block_exits);
19153
19154 return UP(pm_module_node_create(parser, &locals, &module_keyword, constant_path, &name, statements, &parser->previous));
19155}
19156
19161static pm_node_t *
19162parse_parentheses(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
19163 pm_token_t opening = parser->current;
19164 pm_node_flags_t paren_flags = 0;
19165
19166 pm_scratch_node_list_t current_block_exits = { 0 };
19167 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
19168
19169 parser_lex(parser);
19170 while (true) {
19171 if (accept1(parser, PM_TOKEN_SEMICOLON)) {
19172 paren_flags |= PM_PARENTHESES_NODE_FLAGS_MULTIPLE_STATEMENTS;
19173 } else if (!accept1(parser, PM_TOKEN_NEWLINE)) {
19174 break;
19175 }
19176 }
19177
19178 /* If this is the end of the file or we match a right parenthesis, then we
19179 * have an empty parentheses node, and we can immediately return. */
19180 if (match2(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_EOF)) {
19181 /* A command argument group sets EXPR_ENDARG before its ')' is
19182 * consumed, even when the group is empty, so that a following '{' is
19183 * scanned as a block brace. */
19184 if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT) && opening.type == PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES) {
19185 lex_state_set(parser, PM_LEX_STATE_ENDARG);
19186 }
19187
19188 expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
19189 pop_block_exits(parser, previous_block_exits);
19190 return UP(pm_parentheses_node_create(parser, &opening, NULL, &parser->previous, paren_flags));
19191 }
19192
19193 /* Otherwise, we're going to parse the first statement in the list of
19194 * statements within the parentheses. */
19195 context_push(parser, PM_CONTEXT_PARENS);
19196 pm_node_t *statement = parse_expression(parser, PM_BINDING_POWER_STATEMENT, PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_COMMAND_DEF | PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_CANNOT_PARSE_EXPRESSION, (uint16_t) (depth + 1));
19197 context_pop(parser);
19198
19199 /* Determine if this statement is followed by a terminator. In the case of a
19200 * single statement, this is fine. But in the case of multiple statements
19201 * it's required. */
19202 bool terminator_found = false;
19203
19204 if (accept1(parser, PM_TOKEN_SEMICOLON)) {
19205 terminator_found = true;
19206 paren_flags |= PM_PARENTHESES_NODE_FLAGS_MULTIPLE_STATEMENTS;
19207 } else if (accept1(parser, PM_TOKEN_NEWLINE)) {
19208 terminator_found = true;
19209 }
19210
19211 if (terminator_found) {
19212 while (true) {
19213 if (accept1(parser, PM_TOKEN_SEMICOLON)) {
19214 paren_flags |= PM_PARENTHESES_NODE_FLAGS_MULTIPLE_STATEMENTS;
19215 } else if (!accept1(parser, PM_TOKEN_NEWLINE)) {
19216 break;
19217 }
19218 }
19219 }
19220
19221 /* If we hit a right parenthesis, then we're done parsing the parentheses
19222 * node, and we can check which kind of node we should return. */
19223 if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
19224 if (opening.type == PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES) {
19225 lex_state_set(parser, PM_LEX_STATE_ENDARG);
19226 }
19227
19228 parser_lex(parser);
19229 pop_block_exits(parser, previous_block_exits);
19230
19231 if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE) || PM_NODE_TYPE_P(statement, PM_SPLAT_NODE)) {
19232 /* If we have a single statement and are ending on a right
19233 * parenthesis, then we need to check if this is possibly a multiple
19234 * target node. */
19235 pm_multi_target_node_t *multi_target;
19236
19237 if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE) && ((pm_multi_target_node_t *) statement)->lparen_loc.length == 0) {
19238 multi_target = (pm_multi_target_node_t *) statement;
19239 multi_target->lparen_loc = TOK2LOC(parser, &opening);
19240 multi_target->rparen_loc = TOK2LOC(parser, &parser->previous);
19241 PM_NODE_START_SET_TOKEN(parser, multi_target, &opening);
19242 PM_NODE_LENGTH_SET_TOKEN(parser, multi_target, &parser->previous);
19243 } else {
19244 pm_multi_targets_t targets = { 0 };
19245 pm_multi_targets_append(parser, &targets, statement);
19246 multi_target = pm_multi_target_node_create(parser, &targets, &opening, &parser->previous);
19247 }
19248
19249 pm_node_t *result;
19250 if (match1(parser, PM_TOKEN_COMMA) && (binding_power == PM_BINDING_POWER_STATEMENT)) {
19251 result = parse_targets(parser, UP(multi_target), PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1));
19252 } else {
19253 result = UP(multi_target);
19254 }
19255
19256 if (context_p(parser, PM_CONTEXT_MULTI_TARGET)) {
19257 /* All set, this is explicitly allowed by the parent context. */
19258 } else if (context_p(parser, PM_CONTEXT_FOR_INDEX) && match2(parser, PM_TOKEN_KEYWORD_IN, PM_TOKEN_COMMA)) {
19259 /* All set, we're inside a for loop and we're parsing multiple
19260 * targets. A comma continues the index target list, as in
19261 * `for (a, b), c in ...`. */
19262 } else if (flags & PM_PARSE_ACCEPTS_STATEMENT) {
19263 /* The rescue-modifier value parser promotes this target on a
19264 * following `=` or comma. Reject any other binary operator that
19265 * would otherwise consume the target list (e.g. `(a, b) + c`). */
19266 if (pm_binding_powers[parser->current.type].binary && !match1(parser, PM_TOKEN_EQUAL)) {
19267 pm_parser_err_node(parser, result, PM_ERR_WRITE_TARGET_UNEXPECTED);
19268 }
19269 } else if (binding_power != PM_BINDING_POWER_STATEMENT) {
19270 /* Multi targets are not allowed when it's not a statement
19271 * level. */
19272 pm_parser_err_node(parser, result, PM_ERR_WRITE_TARGET_UNEXPECTED);
19273 } else if (context_p(parser, PM_CONTEXT_PARENS) && accept1(parser, PM_TOKEN_NEWLINE)) {
19274 if (!match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) pm_parser_err_node(parser, result, PM_ERR_WRITE_TARGET_UNEXPECTED);
19275 } else if (!match2(parser, PM_TOKEN_EQUAL, PM_TOKEN_PARENTHESIS_RIGHT)) {
19276 /* Multi targets must be followed by an equal sign in order to
19277 * be valid (or a right parenthesis if they are nested). */
19278 pm_parser_err_node(parser, result, PM_ERR_WRITE_TARGET_UNEXPECTED);
19279 }
19280
19281 return result;
19282 }
19283
19284 /* If we have a single statement and are ending on a right parenthesis
19285 * and we didn't return a multiple assignment node, then we can return a
19286 * regular parentheses node now. */
19287 pm_statements_node_t *statements = pm_statements_node_single_create(parser, statement, true);
19288 return UP(pm_parentheses_node_create(parser, &opening, UP(statements), &parser->previous, paren_flags));
19289 }
19290
19291 /* If we have more than one statement in the set of parentheses, then we are
19292 * going to parse all of them as a list of statements. We'll do that here.
19293 */
19294 context_push(parser, PM_CONTEXT_PARENS);
19295 paren_flags |= PM_PARENTHESES_NODE_FLAGS_MULTIPLE_STATEMENTS;
19296
19297 pm_statements_t body = { 0 };
19298 pm_statements_body_append(parser, &body, statement, true);
19299
19300 /* If we didn't find a terminator and we didn't find a right parenthesis,
19301 * then this is a syntax error. */
19302 if (!terminator_found && !match1(parser, PM_TOKEN_EOF)) {
19303 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->current.type));
19304 }
19305
19306 /* Parse each statement within the parentheses. */
19307 while (true) {
19308 pm_node_t *node = parse_expression(parser, PM_BINDING_POWER_STATEMENT, PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_COMMAND_DEF | PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_CANNOT_PARSE_EXPRESSION, (uint16_t) (depth + 1));
19309 pm_statements_body_append(parser, &body, node, true);
19310
19311 /* If we're recovering from a syntax error, then we need to stop parsing
19312 * the statements now. */
19313 if (parser->recovering) {
19314 /* If this is the level of context where the recovery has happened,
19315 * then we can mark the parser as done recovering. */
19316 if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) parser->recovering = false;
19317 break;
19318 }
19319
19320 /* If we couldn't parse an expression at all, then we need to bail out
19321 * of the loop. */
19322 if (PM_NODE_TYPE_P(node, PM_ERROR_RECOVERY_NODE)) break;
19323
19324 /* If we successfully parsed a statement, then we are going to need a
19325 * terminator to delimit them. */
19326 if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) {
19327 while (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON));
19328 if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) break;
19329 } else if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
19330 break;
19331 } else if (!match1(parser, PM_TOKEN_EOF)) {
19332 /* If we're at the end of the file, then we're going to add an error
19333 * after this for the ) anyway. */
19334 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->current.type));
19335 }
19336 }
19337
19338 context_pop(parser);
19339 expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
19340 pm_statements_node_t *statements = pm_statements_node_create(parser, &body);
19341
19342 /* When we're parsing multi targets, we allow them to be followed by a right
19343 * parenthesis if they are at the statement level. This is only possible if
19344 * they are the final statement in a parentheses. We need to explicitly
19345 * reject that here. */
19346 {
19347 pm_node_t *statement = statements->body.nodes[statements->body.size - 1];
19348
19349 if (PM_NODE_TYPE_P(statement, PM_SPLAT_NODE)) {
19350 pm_multi_targets_t targets = { 0 };
19351 pm_multi_targets_append(parser, &targets, statement);
19352 statement = UP(pm_multi_target_node_create(parser, &targets, NULL, NULL));
19353 statements->body.nodes[statements->body.size - 1] = statement;
19354 }
19355
19356 if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE)) {
19357 const uint8_t *offset = parser->start + PM_NODE_END(statement);
19358 pm_token_t operator = { .type = PM_TOKEN_EQUAL, .start = offset, .end = offset };
19359 pm_node_t *value = UP(pm_error_recovery_node_create(parser, PM_NODE_END(statement), 0));
19360
19361 statement = UP(pm_multi_write_node_create(parser, (pm_multi_target_node_t *) statement, &operator, value));
19362 statements->body.nodes[statements->body.size - 1] = statement;
19363
19364 pm_parser_err_node(parser, statement, PM_ERR_WRITE_TARGET_UNEXPECTED);
19365 }
19366 }
19367
19368 pop_block_exits(parser, previous_block_exits);
19369 pm_void_statements_check(parser, statements, true);
19370 return UP(pm_parentheses_node_create(parser, &opening, UP(statements), &parser->previous, paren_flags));
19371}
19372
19378static pm_node_t *
19379parse_splat(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
19380 pm_token_t operator = parser->previous;
19381 pm_node_t *name = NULL;
19382
19383 if (token_begins_expression_p(parser->current.type)) {
19384 name = parse_expression(parser, PM_BINDING_POWER_INDEX, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_STAR, (uint16_t) (depth + 1));
19385 }
19386
19387 return UP(pm_splat_node_create(parser, &operator, name));
19388}
19389
19390static pm_keyword_hash_node_t *
19391parse_array_keyword_hash(pm_parser_t *parser, pm_node_t *key, uint8_t flags, uint16_t depth) {
19392 pm_hash_elements_t elements = { 0 };
19393
19394 pm_static_literals_t hash_keys = { 0 };
19395 pm_hash_key_static_literals_add(parser, &hash_keys, key);
19396
19397 pm_token_t operator = { 0 };
19398 if (parser->previous.type == PM_TOKEN_EQUAL_GREATER) {
19399 operator = parser->previous;
19400 }
19401
19402 pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1));
19403 pm_node_t *assoc = UP(pm_assoc_node_create(parser, key, NTOK2PTR(operator), value));
19404 pm_hash_elements_append(parser, &elements, assoc);
19405
19406 if (accept1(parser, PM_TOKEN_COMMA) && !match1(parser, PM_TOKEN_BRACKET_RIGHT)) {
19407 parse_assocs(parser, &hash_keys, &elements, (uint16_t) (depth + 1));
19408 }
19409
19410 pm_static_literals_free(&hash_keys);
19411 return pm_keyword_hash_node_create(parser, &elements);
19412}
19413
19414static pm_array_node_t *
19415parse_array(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
19416 pm_token_t opening = parser->current;
19417 parser_lex(parser);
19418
19419 pm_array_elements_t elements = { 0 };
19420
19421 bool parsed_bare_hash = false;
19422 while (!match2(parser, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_EOF)) {
19423 bool accepted_newline = accept1(parser, PM_TOKEN_NEWLINE);
19424
19425 // Handle the case where we don't have a comma and we have a
19426 // newline followed by a right bracket.
19427 if (accepted_newline && match1(parser, PM_TOKEN_BRACKET_RIGHT)) {
19428 break;
19429 }
19430
19431 // Ensure that we have a comma between elements in the array.
19432 if (elements.nodes.size > 0) {
19433 if (accept1(parser, PM_TOKEN_COMMA)) {
19434 // If there was a comma but we also accepts a newline,
19435 // then this is a syntax error.
19436 if (accepted_newline) {
19437 pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
19438 }
19439 } else {
19440 // If there was no comma, then we need to add a syntax
19441 // error.
19442 PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_END(parser, &parser->previous), 0, PM_ERR_ARRAY_SEPARATOR, pm_token_str(parser->current.type));
19443 parser->previous.start = parser->previous.end;
19444 parser->previous.type = 0;
19445 }
19446 }
19447
19448 // If we have a right bracket immediately following a comma,
19449 // this is allowed since it's a trailing comma. In this case we
19450 // can break out of the loop.
19451 if (match1(parser, PM_TOKEN_BRACKET_RIGHT)) break;
19452
19453 pm_node_t *element;
19454
19455 if (accept1(parser, PM_TOKEN_USTAR)) {
19456 pm_token_t operator = parser->previous;
19457 pm_node_t *expression = NULL;
19458
19459 if (match3(parser, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_COMMA, PM_TOKEN_EOF)) {
19460 pm_parser_scope_forwarding_positionals_check(parser, &operator);
19461 } else {
19462 expression = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_ARRAY_EXPRESSION_AFTER_STAR, (uint16_t) (depth + 1));
19463 }
19464
19465 element = UP(pm_splat_node_create(parser, &operator, expression));
19466 } else if (match2(parser, PM_TOKEN_LABEL, PM_TOKEN_USTAR_STAR)) {
19467 if (parsed_bare_hash) {
19468 pm_parser_err_current(parser, PM_ERR_EXPRESSION_BARE_HASH);
19469 }
19470
19471 pm_hash_elements_t hash_elements = { 0 };
19472 pm_static_literals_t hash_keys = { 0 };
19473
19474 if (!match8(parser, PM_TOKEN_EOF, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_TOKEN_KEYWORD_DO_BLOCK, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_KEYWORD_DO, PM_TOKEN_PARENTHESIS_RIGHT)) {
19475 parse_assocs(parser, &hash_keys, &hash_elements, (uint16_t) (depth + 1));
19476 }
19477
19478 pm_static_literals_free(&hash_keys);
19479 element = UP(pm_keyword_hash_node_create(parser, &hash_elements));
19480 parsed_bare_hash = true;
19481 } else {
19482 element = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_LABEL), PM_ERR_ARRAY_EXPRESSION, (uint16_t) (depth + 1));
19483
19484 if (pm_symbol_node_label_p(parser, element) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) {
19485 if (parsed_bare_hash) {
19486 pm_parser_err_previous(parser, PM_ERR_EXPRESSION_BARE_HASH);
19487 }
19488
19489 element = UP(parse_array_keyword_hash(parser, element, flags, depth));
19490 parsed_bare_hash = true;
19491 }
19492 }
19493
19494 pm_array_elements_append(parser, &elements, element);
19495 if (PM_NODE_TYPE_P(element, PM_ERROR_RECOVERY_NODE)) break;
19496 }
19497
19498 accept1(parser, PM_TOKEN_NEWLINE);
19499
19500 if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) {
19501 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARRAY_TERM, pm_token_str(parser->current.type));
19502 const uint8_t *end = match1(parser, PM_TOKEN_EOF) ? parser->current.end : parser->previous.end;
19503 parser->previous.start = end;
19504 parser->previous.end = end;
19505 parser->previous.type = 0;
19506 }
19507
19508 return pm_array_node_create(parser, &opening, &elements, &parser->previous);
19509}
19510
19511static pm_hash_node_t *
19512parse_hash(pm_parser_t *parser, uint16_t depth) {
19513 pm_token_t opening = parser->current;
19514 parser_lex(parser);
19515
19516 pm_hash_elements_t elements = { 0 };
19517 if (!match2(parser, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_EOF)) {
19518 pm_static_literals_t hash_keys = { 0 };
19519 parse_assocs(parser, &hash_keys, &elements, (uint16_t) (depth + 1));
19520 pm_static_literals_free(&hash_keys);
19521
19522 accept1(parser, PM_TOKEN_NEWLINE);
19523 }
19524
19525 expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_HASH_TERM, &opening);
19526 return pm_hash_node_create(parser, &opening, &elements, &parser->previous);
19527}
19528
19529static pm_node_t *
19530parse_heredoc(pm_parser_t *parser, uint16_t depth) {
19531 if (PRISM_UNLIKELY(depth >= PRISM_DEPTH_MAXIMUM || pm_parser_stack_exhausted(parser))) {
19532 return parse_resource_recovery(parser, depth);
19533 }
19534
19535 PRISM_ASSERT(parser->lex_modes.current->mode == PM_LEX_HEREDOC);
19536 pm_heredoc_lex_mode_t lex_mode = parser->lex_modes.current->as.heredoc.base;
19537
19538 size_t common_whitespace = (size_t) -1;
19539 parser->lex_modes.current->as.heredoc.common_whitespace = &common_whitespace;
19540
19541 parser_lex(parser);
19542 pm_token_t opening = parser->previous;
19543
19544 pm_node_t *node;
19545 pm_node_t *part;
19546
19547 if (match2(parser, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) {
19548 // If we get here, then we have an empty heredoc. We'll create
19549 // an empty content token and return an empty string node.
19550 expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length);
19551 pm_token_t content = parse_strings_empty_content(parser->previous.start);
19552
19553 if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) {
19554 node = UP(pm_xstring_node_create_unescaped(parser, &opening, &content, &parser->previous, &PM_STRING_EMPTY));
19555 } else {
19556 node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &PM_STRING_EMPTY));
19557 }
19558
19559 PM_NODE_LENGTH_SET_TOKEN(parser, node, &opening);
19560 } else if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) == NULL) {
19561 // If we get here, then we tried to find something in the
19562 // heredoc but couldn't actually parse anything, so we'll just
19563 // return a missing node.
19564 //
19565 // parse_string_part handles its own errors, so there is no need
19566 // for us to add one here.
19567 node = UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)));
19568 } else if (PM_NODE_TYPE_P(part, PM_STRING_NODE) && match2(parser, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) {
19569 // If we get here, then the part that we parsed was plain string
19570 // content and we're at the end of the heredoc, so we can return
19571 // just a string node with the heredoc opening and closing as
19572 // its opening and closing.
19573 pm_node_flag_set(part, parse_unescaped_encoding(parser, parser->explicit_encoding));
19574 pm_string_node_t *cast = (pm_string_node_t *) part;
19575
19576 cast->opening_loc = TOK2LOC(parser, &opening);
19577 cast->closing_loc = TOK2LOC(parser, &parser->current);
19578 cast->base.location = cast->opening_loc;
19579
19580 if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) {
19581 PRISM_ASSERT(sizeof(pm_string_node_t) == sizeof(pm_x_string_node_t));
19582 cast->base.type = PM_X_STRING_NODE;
19583 }
19584
19585 if (lex_mode.indent == PM_HEREDOC_INDENT_TILDE && (common_whitespace != (size_t) -1) && (common_whitespace != 0)) {
19586 parse_heredoc_dedent_string(parser->arena, &cast->unescaped, common_whitespace);
19587 }
19588
19589 node = UP(cast);
19590 expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length);
19591 } else {
19592 // If we get here, then we have multiple parts in the heredoc,
19593 // so we'll need to create an interpolated string node to hold
19594 // them all.
19595 pm_node_list_t parts = { 0 };
19596 pm_node_list_open_append(&parser->node_chunks, &parts, part);
19597
19598 while (!match2(parser, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) {
19599 if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) != NULL) {
19600 pm_node_list_open_append(&parser->node_chunks, &parts, part);
19601 }
19602 }
19603
19604 // Now that we have all of the parts, create the correct type of
19605 // interpolated node.
19606 expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length);
19607
19608 if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) {
19609 /* The parts of a backtick heredoc are kept as they are, without
19610 * the processing that pm_interpolated_xstring_node_create does. */
19611 pm_node_list_finalize(&parser->node_chunks, parser->arena, &parts);
19612
19613 node = UP(pm_interpolated_x_string_node_new(
19614 parser->arena,
19615 ++parser->node_id,
19616 0,
19617 TOK2LOC(parser, &opening),
19618 TOK2LOC(parser, &opening),
19619 parts,
19620 TOK2LOC(parser, &parser->previous)
19621 ));
19622 } else {
19623 pm_interpolated_string_node_t *cast = pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous);
19624 cast->base.location = cast->opening_loc;
19625 node = UP(cast);
19626 }
19627
19628 // If this is a heredoc that is indented with a ~, then we need
19629 // to dedent each line by the common leading whitespace.
19630 if (lex_mode.indent == PM_HEREDOC_INDENT_TILDE && (common_whitespace != (size_t) -1) && (common_whitespace != 0)) {
19631 pm_node_list_t *nodes;
19632 if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) {
19633 nodes = &((pm_interpolated_x_string_node_t *) node)->parts;
19634 } else {
19635 nodes = &((pm_interpolated_string_node_t *) node)->parts;
19636 }
19637
19638 parse_heredoc_dedent(parser, nodes, common_whitespace);
19639 }
19640 }
19641
19642 /* If a missing terminator left this heredoc's lex mode on the stack, it
19643 * still points at our stack-local common_whitespace. Clear the pointer so
19644 * that subsequent lexing cannot read from this function's dead stack frame.
19645 */
19646 pm_lex_mode_t *whitespace_mode = parser->lex_modes.current;
19647 do {
19648 if (whitespace_mode->mode == PM_LEX_HEREDOC && whitespace_mode->as.heredoc.common_whitespace == &common_whitespace) {
19649 whitespace_mode->as.heredoc.common_whitespace = NULL;
19650 }
19651 whitespace_mode = whitespace_mode->prev;
19652 } while (whitespace_mode != NULL);
19653
19654 /* The body and terminator are out of source order. The caller resumes
19655 * after the opener, so missing tokens must be anchored there as well. */
19656 parser->previous = opening;
19657 return node;
19658}
19659
19660static pm_begin_node_t *
19661parse_begin(pm_parser_t *parser, uint16_t depth) {
19662 pm_token_t begin_keyword = parser->current;
19663 size_t opening_newline_index = token_newline_index(parser);
19664 parser_lex(parser);
19665
19666 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
19667
19668 pm_scratch_node_list_t current_block_exits = { 0 };
19669 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
19670 pm_statements_node_t *begin_statements = NULL;
19671
19672 if (!match4(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) {
19673 pm_accepts_block_stack_push(parser, true);
19674 begin_statements = parse_statements(parser, PM_CONTEXT_BEGIN, (uint16_t) (depth + 1));
19675 pm_accepts_block_stack_pop(parser);
19676 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
19677 }
19678
19679 pm_begin_node_t *node = pm_begin_node_create(parser, &begin_keyword, begin_statements);
19680 parse_rescues(parser, opening_newline_index, &begin_keyword, node, PM_RESCUES_BEGIN, (uint16_t) (depth + 1));
19681 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_BEGIN_TERM, &begin_keyword);
19682
19683 PM_NODE_LENGTH_SET_TOKEN(parser, node, &parser->previous);
19684 pm_begin_node_end_keyword_set(parser, node, &parser->previous);
19685 pop_block_exits(parser, previous_block_exits);
19686 return node;
19687}
19688
19689static pm_defined_node_t *
19690parse_defined(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
19691 pm_token_t keyword = parser->current;
19692 parser_lex(parser);
19693
19694 pm_token_t lparen = { 0 };
19695 pm_token_t rparen = { 0 };
19696 pm_node_t *expression;
19697
19698 context_push(parser, PM_CONTEXT_DEFINED);
19699 bool newline = accept1(parser, PM_TOKEN_NEWLINE);
19700
19701 if (!newline && accept2(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) {
19702 lparen = parser->previous;
19703 expression = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_DEFINED_EXPRESSION, (uint16_t) (depth + 1));
19704
19705 if (!parser->recovering) {
19706 accept1(parser, PM_TOKEN_NEWLINE);
19707 expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
19708 rparen = parser->previous;
19709 }
19710 } else {
19711 expression = parse_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_DEFINED_EXPRESSION, (uint16_t) (depth + 1));
19712 }
19713
19714 context_pop(parser);
19715 return pm_defined_node_create(parser, NTOK2PTR(lparen), expression, NTOK2PTR(rparen), &keyword);
19716}
19717
19718static pm_for_node_t *
19719parse_for(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
19720 pm_token_t for_keyword = parser->current;
19721 size_t opening_newline_index = token_newline_index(parser);
19722 parser_lex(parser);
19723
19724 pm_node_t *index;
19725 context_push(parser, PM_CONTEXT_FOR_INDEX);
19726
19727 // First, parse out the first index expression.
19728 if (accept1(parser, PM_TOKEN_USTAR)) {
19729 index = parse_splat(parser, flags, depth);
19730 } else if (token_begins_expression_p(parser->current.type)) {
19731 index = parse_expression(parser, PM_BINDING_POWER_INDEX, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA, (uint16_t) (depth + 1));
19732 } else {
19733 pm_parser_err_token(parser, &for_keyword, PM_ERR_FOR_INDEX);
19734 index = UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &for_keyword), PM_TOKEN_LENGTH(&for_keyword)));
19735 }
19736
19737 // Now, if there are multiple index expressions, parse them out.
19738 if (match1(parser, PM_TOKEN_COMMA)) {
19739 index = parse_targets(parser, index, PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1));
19740 } else {
19741 index = parse_target(parser, index, false, false);
19742 }
19743
19744 context_pop(parser);
19745 pm_do_loop_stack_push(parser, true);
19746
19747 expect1(parser, PM_TOKEN_KEYWORD_IN, PM_ERR_FOR_IN);
19748 pm_token_t in_keyword = parser->previous;
19749
19750 pm_node_t *collection = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_FOR_COLLECTION, (uint16_t) (depth + 1));
19751 pm_do_loop_stack_pop(parser);
19752
19753 pm_token_t do_keyword = { 0 };
19754 if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) {
19755 do_keyword = parser->previous;
19756 } else {
19757 if (!match2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE)) {
19758 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_FOR_DELIMITER, pm_token_str(parser->current.type));
19759 }
19760 }
19761
19762 pm_statements_node_t *statements = NULL;
19763 if (!match1(parser, PM_TOKEN_KEYWORD_END)) {
19764 statements = parse_statements(parser, PM_CONTEXT_FOR, (uint16_t) (depth + 1));
19765 }
19766
19767 parser_warn_indentation_mismatch(parser, opening_newline_index, &for_keyword, false, false);
19768 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_FOR_TERM, &for_keyword);
19769
19770 return pm_for_node_create(parser, index, collection, statements, &for_keyword, &in_keyword, NTOK2PTR(do_keyword), &parser->previous);
19771}
19772
19773static pm_lambda_node_t *
19774parse_lambda(pm_parser_t *parser, uint16_t depth) {
19775 int previous_lambda_enclosure_nesting = parser->lambda_enclosure_nesting;
19776 parser->lambda_enclosure_nesting = parser->enclosure_nesting;
19777
19778 pm_token_t operator = parser->current;
19779 size_t opening_newline_index = token_newline_index(parser);
19780 parser_lex(parser);
19781
19782 pm_parser_scope_push(parser, false);
19783 pm_block_parameters_node_t *block_parameters;
19784
19785 switch (parser->current.type) {
19786 case PM_TOKEN_PARENTHESIS_LEFT: {
19787 pm_token_t opening = parser->current;
19788 parser_lex(parser);
19789
19790 if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
19791 block_parameters = pm_block_parameters_node_create(parser, NULL, &opening, &((pm_node_list_t) { 0 }));
19792 } else {
19793 block_parameters = parse_block_parameters(parser, false, &opening, true, true, (uint16_t) (depth + 1));
19794 }
19795
19796 accept1(parser, PM_TOKEN_NEWLINE);
19797 expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
19798
19799 pm_block_parameters_node_closing_set(parser, block_parameters, &parser->previous);
19800 break;
19801 }
19802 case PM_CASE_PARAMETER: {
19803 block_parameters = parse_block_parameters(parser, false, NULL, true, false, (uint16_t) (depth + 1));
19804 break;
19805 }
19806 default: {
19807 block_parameters = NULL;
19808 break;
19809 }
19810 }
19811
19812 pm_token_t opening;
19813 pm_node_t *body = NULL;
19814
19815 if (accept1(parser, PM_TOKEN_LAMBDA_BEGIN)) {
19816 opening = parser->previous;
19817
19818 if (!match1(parser, PM_TOKEN_BRACE_RIGHT)) {
19819 body = UP(parse_statements(parser, PM_CONTEXT_LAMBDA_BRACES, (uint16_t) (depth + 1)));
19820 }
19821
19822 parser_warn_indentation_mismatch(parser, opening_newline_index, &operator, false, false);
19823
19824 /* Restore the enclosing lambda's nesting now that the body has been
19825 * parsed, so that the token following the closing `}` is lexed in the
19826 * enclosing context. During the body the nesting held this lambda's own
19827 * level, which every token inside the braces sits above. */
19828 parser->lambda_enclosure_nesting = previous_lambda_enclosure_nesting;
19829 expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_LAMBDA_TERM_BRACE, &opening);
19830 } else {
19831 /* A `-> { }` body is delimited by `{`/`}`, whose enclosure frame the
19832 * lexer manages. A `-> do end` body is delimited by keywords, so push
19833 * the frame here and pop it before `end`. The frame isolates the body
19834 * from an enclosing `while`/`until`/`for` condition, so a `do` inside
19835 * it opens a block rather than the loop body. The push must precede
19836 * consuming the `do`, which lexes the first token of the body. */
19837 pm_enclosure_frame_push(parser);
19838 expect1(parser, PM_TOKEN_KEYWORD_DO_LAMBDA, PM_ERR_LAMBDA_OPEN);
19839 opening = parser->previous;
19840
19841 /* The lexer cleared the nesting when it produced the `do`. If it was
19842 * missing entirely, clear it here so that the body is recovered the
19843 * same way it would have been parsed: no token within it sits at the
19844 * beginning of a lambda. */
19845 parser->lambda_enclosure_nesting = -1;
19846
19847 if (!match3(parser, PM_TOKEN_KEYWORD_END, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
19848 body = UP(parse_statements(parser, PM_CONTEXT_LAMBDA_DO_END, (uint16_t) (depth + 1)));
19849 }
19850
19851 if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
19852 PRISM_ASSERT(body == NULL || PM_NODE_TYPE_P(body, PM_STATEMENTS_NODE));
19853 body = UP(parse_rescues_implicit_begin(parser, opening_newline_index, &operator, opening.start, (pm_statements_node_t *) body, PM_RESCUES_LAMBDA, (uint16_t) (depth + 1)));
19854 } else {
19855 parser_warn_indentation_mismatch(parser, opening_newline_index, &operator, false, false);
19856 }
19857
19858 pm_enclosure_frame_pop(parser);
19859
19860 /* As with the brace branch above, restore the nesting before consuming
19861 * the closing `end`, which lexes the token that follows it. */
19862 parser->lambda_enclosure_nesting = previous_lambda_enclosure_nesting;
19863 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_LAMBDA_TERM_END, &operator);
19864 }
19865
19866 pm_constant_id_list_t locals;
19867 pm_locals_order(parser, &locals, pm_parser_scope_toplevel_p(parser));
19868 pm_node_t *parameters = parse_blocklike_parameters(parser, UP(block_parameters), &operator, &parser->previous);
19869
19870 pm_parser_scope_pop(parser);
19871 return pm_lambda_node_create(parser, &locals, &operator, &opening, &parser->previous, parameters, body);
19872}
19873
19874static pm_node_t *
19875parse_regexp(pm_parser_t *parser, uint16_t depth) {
19876 pm_token_t opening = parser->current;
19877 parser_lex(parser);
19878
19879 if (match1(parser, PM_TOKEN_REGEXP_END)) {
19880 // If we get here, then we have an end immediately after a start. In
19881 // that case we'll create an empty content token and return an
19882 // uninterpolated regular expression.
19883 pm_token_t content = (pm_token_t) {
19884 .type = PM_TOKEN_STRING_CONTENT,
19885 .start = parser->previous.end,
19886 .end = parser->previous.end
19887 };
19888
19889 parser_lex(parser);
19890
19891 pm_regular_expression_node_t *node = pm_regular_expression_node_create(parser, &opening, &content, &parser->previous);
19892 pm_regexp_offsets_t offsets = { 0 };
19893 pm_node_flag_set(UP(node), pm_regexp_parse(parser, node, &offsets));
19894 return UP(node);
19895 }
19896
19897 pm_node_list_t parts = { 0 };
19898
19899 if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
19900 // In this case we've hit string content so we know the regular
19901 // expression at least has something in it. We'll need to check if the
19902 // following token is the end (in which case we can return a plain
19903 // regular expression) or if it's not then it has interpolation.
19904 pm_string_t unescaped = parser->current_string;
19905 pm_regexp_offsets_t offsets = parser->current_regexp_offsets;
19906 pm_token_t content = parser->current;
19907 parser_lex(parser);
19908
19909 // If we hit an end, then we can create a regular expression
19910 // node without interpolation, which can be represented more
19911 // succinctly and more easily compiled.
19912 if (accept1(parser, PM_TOKEN_REGEXP_END)) {
19913 pm_regular_expression_node_t *node = (pm_regular_expression_node_t *) pm_regular_expression_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped);
19914
19915 /* A regular expression is validated as soon as it is parsed, even
19916 * when a =~ follows, because the =~ may turn out to apply to an
19917 * expression that contains it, as in `!/a/ =~ b`. */
19918 pm_node_flag_set(UP(node), pm_regexp_parse(parser, node, &offsets));
19919 return UP(node);
19920 }
19921
19922 // If we get here, then we have interpolation so we'll need to create
19923 // a regular expression node with interpolation.
19924 pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped));
19925 if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
19926 // This is extremely strange, but the first string part of a
19927 // regular expression will always be tagged as binary if we
19928 // are in a US-ASCII file, no matter its contents.
19929 pm_node_flag_set(part, PM_STRING_FLAGS_FORCED_BINARY_ENCODING);
19930 }
19931
19932 pm_node_list_open_append(&parser->node_chunks, &parts, part);
19933 }
19934
19935 // Now that we're here and we have interpolation, we'll parse all of the
19936 // parts into the list.
19937 pm_node_t *part;
19938 while (!match2(parser, PM_TOKEN_REGEXP_END, PM_TOKEN_EOF)) {
19939 if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) != NULL) {
19940 pm_node_list_open_append(&parser->node_chunks, &parts, part);
19941 }
19942 }
19943
19944 pm_token_t closing = parser->current;
19945 bool terminated = !match1(parser, PM_TOKEN_EOF);
19946
19947 if (terminated) {
19948 expect1(parser, PM_TOKEN_REGEXP_END, PM_ERR_REGEXP_TERM);
19949 } else {
19950 pm_parser_err_token(parser, &opening, PM_ERR_REGEXP_TERM);
19951 closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
19952 }
19953
19954 /* A regular expression that is not terminated is still validated, as the
19955 * content up to the end of the file, so its errors are reported too. */
19956 pm_interpolated_regular_expression_node_t *interpolated = pm_interpolated_regular_expression_node_create(parser, &opening, &parts, &closing);
19957 pm_regexp_parse_interpolated(parser, interpolated);
19958
19959 return UP(interpolated);
19960}
19961
19962static pm_node_t *
19963parse_alias(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
19964 if (binding_power != PM_BINDING_POWER_STATEMENT && !(flags & PM_PARSE_ACCEPTS_STATEMENT)) {
19965 pm_parser_err_current(parser, PM_ERR_STATEMENT_ALIAS);
19966 }
19967
19968 pm_token_t keyword = parser->current;
19969 parser_lex(parser);
19970
19971 pm_node_t *new_name = parse_alias_argument(parser, true, (uint16_t) (depth + 1));
19972 pm_node_t *old_name = parse_alias_argument(parser, false, (uint16_t) (depth + 1));
19973
19974 switch (PM_NODE_TYPE(new_name)) {
19975 case PM_BACK_REFERENCE_READ_NODE:
19976 case PM_NUMBERED_REFERENCE_READ_NODE:
19977 case PM_GLOBAL_VARIABLE_READ_NODE: {
19978 if (PM_NODE_TYPE_P(old_name, PM_BACK_REFERENCE_READ_NODE) || PM_NODE_TYPE_P(old_name, PM_NUMBERED_REFERENCE_READ_NODE) || PM_NODE_TYPE_P(old_name, PM_GLOBAL_VARIABLE_READ_NODE)) {
19979 if (PM_NODE_TYPE_P(old_name, PM_NUMBERED_REFERENCE_READ_NODE)) {
19980 pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT_NUMBERED_REFERENCE);
19981 }
19982 } else if (!PM_NODE_TYPE_P(old_name, PM_ERROR_RECOVERY_NODE)) {
19983 pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT);
19984 old_name = UP(pm_error_recovery_node_create_unexpected(parser, old_name));
19985 }
19986
19987 return UP(pm_alias_global_variable_node_create(parser, &keyword, new_name, old_name));
19988 }
19989 case PM_SYMBOL_NODE:
19990 case PM_INTERPOLATED_SYMBOL_NODE: {
19991 if (!PM_NODE_TYPE_P(old_name, PM_SYMBOL_NODE) && !PM_NODE_TYPE_P(old_name, PM_INTERPOLATED_SYMBOL_NODE) && !PM_NODE_TYPE_P(old_name, PM_ERROR_RECOVERY_NODE)) {
19992 pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT);
19993 old_name = UP(pm_error_recovery_node_create_unexpected(parser, old_name));
19994 }
19995 }
19997 default:
19998 return UP(pm_alias_method_node_create(parser, &keyword, new_name, old_name));
19999 }
20000}
20001
20002static pm_until_node_t *
20003parse_until(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
20004 pm_token_t keyword = parser->current;
20005 size_t opening_newline_index = token_newline_index(parser);
20006
20007 context_push(parser, PM_CONTEXT_LOOP_PREDICATE);
20008 pm_do_loop_stack_push(parser, true);
20009
20010 parser_lex(parser);
20011 pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_UNTIL_PREDICATE, (uint16_t) (depth + 1));
20012
20013 pm_do_loop_stack_pop(parser);
20014 context_pop(parser);
20015
20016 pm_token_t do_keyword = { 0 };
20017 if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) {
20018 do_keyword = parser->previous;
20019 } else {
20020 expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_UNTIL_PREDICATE);
20021 }
20022
20023 pm_statements_node_t *statements = NULL;
20024 if (!match1(parser, PM_TOKEN_KEYWORD_END)) {
20025 pm_accepts_block_stack_push(parser, true);
20026 statements = parse_statements(parser, PM_CONTEXT_UNTIL, (uint16_t) (depth + 1));
20027 pm_accepts_block_stack_pop(parser);
20028 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
20029 }
20030
20031 parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false);
20032 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_UNTIL_TERM, &keyword);
20033
20034 return pm_until_node_create(parser, &keyword, NTOK2PTR(do_keyword), &parser->previous, predicate, statements, 0);
20035}
20036
20037static pm_while_node_t *
20038parse_while(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
20039 pm_token_t keyword = parser->current;
20040 size_t opening_newline_index = token_newline_index(parser);
20041
20042 context_push(parser, PM_CONTEXT_LOOP_PREDICATE);
20043 pm_do_loop_stack_push(parser, true);
20044
20045 parser_lex(parser);
20046 pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_WHILE_PREDICATE, (uint16_t) (depth + 1));
20047
20048 pm_do_loop_stack_pop(parser);
20049 context_pop(parser);
20050
20051 pm_token_t do_keyword = { 0 };
20052 if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) {
20053 do_keyword = parser->previous;
20054 } else {
20055 expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_WHILE_PREDICATE);
20056 }
20057
20058 pm_statements_node_t *statements = NULL;
20059 if (!match1(parser, PM_TOKEN_KEYWORD_END)) {
20060 pm_accepts_block_stack_push(parser, true);
20061 statements = parse_statements(parser, PM_CONTEXT_WHILE, (uint16_t) (depth + 1));
20062 pm_accepts_block_stack_pop(parser);
20063 accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON);
20064 }
20065
20066 parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false);
20067 expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_WHILE_TERM, &keyword);
20068
20069 return pm_while_node_create(parser, &keyword, NTOK2PTR(do_keyword), &parser->previous, predicate, statements, 0);
20070}
20071
20072static pm_node_t *
20073parse_xstring(pm_parser_t *parser, uint16_t depth) {
20074 pm_token_t opening = parser->current;
20075 parser_lex(parser);
20076
20077 // When we get here, we don't know if this string is going to have
20078 // interpolation or not, even though it is allowed. Still, we want to be
20079 // able to return a string node without interpolation if we can since
20080 // it'll be faster.
20081 if (match1(parser, PM_TOKEN_STRING_END)) {
20082 // If we get here, then we have an end immediately after a start. In
20083 // that case we'll create an empty content token and return an
20084 // uninterpolated string.
20085 pm_token_t content = (pm_token_t) {
20086 .type = PM_TOKEN_STRING_CONTENT,
20087 .start = parser->previous.end,
20088 .end = parser->previous.end
20089 };
20090
20091 parser_lex(parser);
20092 return UP(pm_xstring_node_create(parser, &opening, &content, &parser->previous));
20093 }
20094
20095 pm_node_list_t parts = { 0 };
20096
20097 if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
20098 // In this case we've hit string content so we know the string
20099 // at least has something in it. We'll need to check if the
20100 // following token is the end (in which case we can return a
20101 // plain string) or if it's not then it has interpolation.
20102 pm_string_t unescaped = parser->current_string;
20103 pm_token_t content = parser->current;
20104 parser_lex(parser);
20105
20106 if (match1(parser, PM_TOKEN_STRING_END)) {
20107 pm_node_t *node = UP(pm_xstring_node_create_unescaped(parser, &opening, &content, &parser->current, &unescaped));
20108 pm_node_flag_set(node, parse_unescaped_encoding(parser, parser->explicit_encoding));
20109 parser_lex(parser);
20110 return node;
20111 }
20112
20113 // If we get here, then we have interpolation so we'll need to
20114 // create a string node with interpolation.
20115 pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped));
20116 pm_node_flag_set(part, parse_unescaped_encoding(parser, parser->explicit_encoding));
20117
20118 pm_node_list_open_append(&parser->node_chunks, &parts, part);
20119 }
20120
20121 pm_node_t *part;
20122 while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
20123 if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) != NULL) {
20124 pm_node_list_open_append(&parser->node_chunks, &parts, part);
20125 }
20126 }
20127
20128 pm_token_t closing = parser->current;
20129 if (match1(parser, PM_TOKEN_EOF)) {
20130 pm_parser_err_token(parser, &opening, PM_ERR_XSTRING_TERM);
20131 closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
20132 } else {
20133 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_XSTRING_TERM);
20134 }
20135
20136 return UP(pm_interpolated_xstring_node_create(parser, &opening, &parts, &closing));
20137}
20138
20139static pm_node_t *
20140parse_multi_target(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t binding_power, uint16_t depth) {
20141 if ((binding_power == PM_BINDING_POWER_STATEMENT) && match1(parser, PM_TOKEN_COMMA)) {
20142 /* If we get here, then we have a comma immediately following something
20143 * that could have been a target in a multiple assignment. */
20144 node = parse_targets_validate(parser, node, PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1));
20145 }
20146 return node;
20147}
20148
20149static pm_node_t *
20150parse_identifier(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20151 pm_token_t identifier = parser->current;
20152 parser_lex(parser);
20153
20154 bool followed = (
20155 ((flags & PM_PARSE_ACCEPTS_COMMAND_CALL) && token_begins_argument_p(parser->current.type)) ||
20156 (pm_accepts_block_stack_p(parser) && match1(parser, PM_TOKEN_KEYWORD_DO)) ||
20157 match1(parser, PM_TOKEN_BRACE_LEFT)
20158 );
20159
20160 /* Resolve numbered parameter names used as calls before registering an
20161 * implicit parameter or parsing the call's arguments and block. */
20162 pm_node_t *node;
20163 if (followed && pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &identifier), PM_TOKEN_LENGTH(&identifier))) {
20164 node = UP(pm_call_node_variable_call_create(parser, &identifier));
20165 } else {
20166 node = parse_variable_call(parser);
20167 }
20168
20169 if (PM_NODE_TYPE_P(node, PM_CALL_NODE)) {
20170 // If parse_variable_call returned with a call node, then we
20171 // know the identifier is not in the local table. In that case
20172 // we need to check if there are arguments following the
20173 // identifier.
20174 pm_call_node_t *call = (pm_call_node_t *) node;
20175 pm_arguments_t arguments = { 0 };
20176
20177 if (parse_arguments_list(parser, &arguments, true, flags, (uint16_t) (depth + 1))) {
20178 // Since we found arguments, we need to turn off the
20179 // variable call bit in the flags.
20180 pm_node_flag_unset(UP(call), PM_CALL_NODE_FLAGS_VARIABLE_CALL);
20181
20182 call->opening_loc = arguments.opening_loc;
20183 call->arguments = pm_arguments_node_create(parser, &arguments);
20184 call->closing_loc = arguments.closing_loc;
20185 call->block = arguments.block;
20186
20187 const pm_location_t *end = pm_arguments_end(&arguments);
20188 if (end == NULL) {
20189 PM_NODE_LENGTH_SET_LOCATION(call, &call->message_loc);
20190 } else {
20191 PM_NODE_LENGTH_SET_LOCATION(call, end);
20192 }
20193 }
20194 } else if (followed) {
20195 // Otherwise, we know the identifier is in the local table. This can
20196 // still be a method call if it is followed by arguments or a block, so
20197 // we need to check for that here.
20198 pm_arguments_t arguments = { 0 };
20199 parse_arguments_list(parser, &arguments, true, flags, (uint16_t) (depth + 1));
20200 pm_call_node_t *fcall = pm_call_node_fcall_create(parser, &identifier, &arguments);
20201
20202 if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) {
20203 // If we're about to convert an 'it' implicit local
20204 // variable read into a method call, we need to remove
20205 // it from the list of implicit local variables.
20206 pm_node_unreference(parser, node);
20207 } else {
20208 // Otherwise, we're about to convert a regular local
20209 // variable read into a method call, in which case we
20210 // need to indicate that this was not a read for the
20211 // purposes of warnings.
20212 PRISM_ASSERT(PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE));
20213
20214 pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node;
20215 PRISM_ASSERT(pm_parser_local_depth_constant_id(parser, cast->name) == (int) cast->depth);
20216 pm_locals_unread(parser, cast->name);
20217 }
20218
20219 return UP(fcall);
20220 }
20221
20222 return parse_multi_target(parser, node, binding_power, depth);
20223}
20224
20225static pm_array_node_t *
20226parse_uninterpolated_symbol_array(pm_parser_t *parser) {
20227 pm_token_t opening = parser->current;
20228 parser_lex(parser);
20229
20230 pm_array_elements_t elements = { 0 };
20231
20232 /* The element being parsed is either a single symbol node in current, or,
20233 * once it needs more than one part, the open list of parts of an
20234 * interpolated symbol. */
20235 pm_node_t *current = NULL;
20236 pm_node_list_t parts = { 0 };
20237
20238 while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
20239 accept1(parser, PM_TOKEN_WORDS_SEP);
20240 if (match1(parser, PM_TOKEN_STRING_END)) break;
20241
20242 // Interpolation is not possible but nested heredocs can still lead to
20243 // consecutive (disjoint) string tokens when the final newline is escaped.
20244 while (match1(parser, PM_TOKEN_STRING_CONTENT)) {
20245 // Record the string node, moving to interpolation if needed.
20246 if (parts.size > 0) {
20247 pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
20248 parser_lex(parser);
20249 pm_node_list_open_append(&parser->node_chunks, &parts, string);
20250 } else if (current == NULL) {
20251 current = UP(pm_symbol_node_create_current_string(parser, NULL, &parser->current, NULL));
20252 parser_lex(parser);
20253 } else {
20254 pm_symbol_node_t *cast = (pm_symbol_node_t *) current;
20255 pm_token_t content = { .type = PM_TOKEN_STRING_CONTENT, .start = parser->start + cast->content_loc.start, .end = parser->start + cast->content_loc.start + cast->content_loc.length };
20256 pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &cast->unescaped));
20257 pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
20258 parser_lex(parser);
20259
20260 pm_node_list_open_append(&parser->node_chunks, &parts, first_string);
20261 pm_node_list_open_append(&parser->node_chunks, &parts, second_string);
20262 current = NULL;
20263 }
20264 }
20265
20266 if (parts.size > 0) {
20267 pm_array_elements_append(parser, &elements, UP(pm_interpolated_symbol_node_create(parser, NULL, &parts, NULL)));
20268 parts = (pm_node_list_t) { 0 };
20269 } else if (current != NULL) {
20270 pm_array_elements_append(parser, &elements, current);
20271 current = NULL;
20272 } else {
20273 expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_I_LOWER_ELEMENT);
20274 }
20275 }
20276
20277 pm_token_t closing = parser->current;
20278 if (match1(parser, PM_TOKEN_EOF)) {
20279 pm_parser_err_token(parser, &opening, PM_ERR_LIST_I_LOWER_TERM);
20280 closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
20281 } else {
20282 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_LOWER_TERM);
20283 }
20284
20285 return pm_array_node_create(parser, &opening, &elements, &closing);
20286}
20287
20292static void
20293parse_interpolated_symbol_array_element(pm_parser_t *parser, pm_array_elements_t *elements, pm_node_t *element) {
20294 if (PM_NODE_TYPE_P(element, PM_SYMBOL_NODE) && !PM_NODE_FLAG_P(element, PM_SYMBOL_FLAGS_FORCED_BINARY_ENCODING | PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING)) {
20295 const pm_symbol_node_t *symbol = (const pm_symbol_node_t *) element;
20296 pm_token_t content = {
20297 .type = PM_TOKEN_STRING_CONTENT,
20298 .start = parser->start + symbol->content_loc.start,
20299 .end = parser->start + PM_LOCATION_END(&symbol->content_loc)
20300 };
20301
20302 if (PM_NODE_FLAG_P(element, PM_SYMBOL_FLAGS_FORCED_UTF8_ENCODING)) {
20303 parse_symbol_encoding_validate_utf8(parser, &content, &symbol->unescaped);
20304 } else {
20305 parse_symbol_encoding_validate_other(parser, &content, &symbol->unescaped);
20306 }
20307 }
20308
20309 pm_array_elements_append(parser, elements, element);
20310}
20311
20312static pm_array_node_t *
20313parse_interpolated_symbol_array(pm_parser_t *parser, uint16_t depth) {
20314 parser_lex(parser);
20315 pm_token_t opening = parser->previous;
20316 pm_array_elements_t elements = { 0 };
20317
20318 /* The element being parsed is either a single symbol node in current, or,
20319 * once it needs more than one part, the open list of parts of an
20320 * interpolated symbol. */
20321 pm_node_t *current = NULL;
20322 pm_node_list_t parts = { 0 };
20323
20324 while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
20325 switch (parser->current.type) {
20326 case PM_TOKEN_WORDS_SEP: {
20327 /* Reset the explicit encoding if we hit a separator since each
20328 * element can have its own encoding. */
20329 parser->explicit_encoding = NULL;
20330
20331 /* If we hit a separator after we've hit content, then we need
20332 * to append that content to the list and reset the element. */
20333 if (parts.size > 0) {
20334 pm_array_elements_append(parser, &elements, UP(pm_interpolated_symbol_node_create(parser, NULL, &parts, NULL)));
20335 parts = (pm_node_list_t) { 0 };
20336 } else if (current != NULL) {
20337 parse_interpolated_symbol_array_element(parser, &elements, current);
20338 current = NULL;
20339 }
20340
20341 parser_lex(parser);
20342 break;
20343 }
20344 case PM_TOKEN_STRING_CONTENT: {
20345 if (parts.size > 0) {
20346 /* If we hit string content and the element is already
20347 * interpolated, then we need to append the string content
20348 * to its parts. */
20349 pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
20350 parser_lex(parser);
20351
20352 pm_node_list_open_append(&parser->node_chunks, &parts, string);
20353 } else if (current == NULL) {
20354 /* If we hit content and there is no element yet, then this
20355 * is the first string content we've seen. In that case
20356 * we're going to create a new symbol node for it. */
20357 current = UP(pm_symbol_node_create_current_string(parser, NULL, &parser->current, NULL));
20358 parser_lex(parser);
20359 } else {
20360 /* If we hit string content and the element is a symbol
20361 * node, then we need to turn the element into the parts of
20362 * an interpolated symbol and add the string content to
20363 * them. */
20364 pm_symbol_node_t *cast = (pm_symbol_node_t *) current;
20365 pm_token_t content = {
20366 .type = PM_TOKEN_STRING_CONTENT,
20367 .start = parser->start + cast->content_loc.start,
20368 .end = parser->start + cast->content_loc.start + cast->content_loc.length
20369 };
20370
20371 pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &cast->unescaped));
20372 pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
20373 parser_lex(parser);
20374
20375 pm_node_list_open_append(&parser->node_chunks, &parts, first_string);
20376 pm_node_list_open_append(&parser->node_chunks, &parts, second_string);
20377 current = NULL;
20378 }
20379
20380 break;
20381 }
20382 case PM_TOKEN_EMBVAR:
20383 case PM_TOKEN_EMBEXPR_BEGIN: {
20384 /* If we hit an embedded variable or expression and the element
20385 * is a symbol node, then we'll turn the element into the parts
20386 * of an interpolated symbol, starting with its string. */
20387 if (current != NULL) {
20388 pm_node_list_open_append(&parser->node_chunks, &parts, UP(pm_symbol_node_to_string_node(parser, (pm_symbol_node_t *) current)));
20389 current = NULL;
20390 }
20391
20392 pm_node_t *part = parse_string_part(parser, (uint16_t) (depth + 1));
20393 pm_node_list_open_append(&parser->node_chunks, &parts, part);
20394 break;
20395 }
20396 default:
20397 expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_I_UPPER_ELEMENT);
20398 parse_discard_token(parser, depth);
20399 break;
20400 }
20401 }
20402
20403 /* If we have an element, then we need to append it to the list. */
20404 if (parts.size > 0) {
20405 pm_array_elements_append(parser, &elements, UP(pm_interpolated_symbol_node_create(parser, NULL, &parts, NULL)));
20406 } else if (current != NULL) {
20407 parse_interpolated_symbol_array_element(parser, &elements, current);
20408 }
20409
20410 pm_token_t closing = parser->current;
20411 if (match1(parser, PM_TOKEN_EOF)) {
20412 pm_parser_err_token(parser, &opening, PM_ERR_LIST_I_UPPER_TERM);
20413 closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
20414 } else {
20415 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_UPPER_TERM);
20416 }
20417
20418 return pm_array_node_create(parser, &opening, &elements, &closing);
20419}
20420
20421static pm_array_node_t *
20422parse_uninterpolated_string_array(pm_parser_t *parser) {
20423 pm_token_t opening = parser->current;
20424 parser_lex(parser);
20425
20426 pm_array_elements_t elements = { 0 };
20427
20428 /* The element being parsed is either a single string node in current, or,
20429 * once it needs more than one part, the open list of parts of an
20430 * interpolated string. */
20431 pm_node_t *current = NULL;
20432 pm_node_list_t parts = { 0 };
20433
20434 while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
20435 accept1(parser, PM_TOKEN_WORDS_SEP);
20436 if (match1(parser, PM_TOKEN_STRING_END)) break;
20437
20438 // Interpolation is not possible but nested heredocs can still lead to
20439 // consecutive (disjoint) string tokens when the final newline is escaped.
20440 while (match1(parser, PM_TOKEN_STRING_CONTENT)) {
20441 pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
20442
20443 // Record the string node, moving to interpolation if needed.
20444 if (parts.size > 0) {
20445 pm_node_list_open_append(&parser->node_chunks, &parts, string);
20446 } else if (current == NULL) {
20447 current = string;
20448 } else {
20449 pm_node_list_open_append(&parser->node_chunks, &parts, current);
20450 pm_node_list_open_append(&parser->node_chunks, &parts, string);
20451 current = NULL;
20452 }
20453 parser_lex(parser);
20454 }
20455
20456 if (parts.size > 0) {
20457 pm_array_elements_append(parser, &elements, UP(pm_interpolated_string_node_create(parser, NULL, &parts, NULL)));
20458 parts = (pm_node_list_t) { 0 };
20459 } else if (current != NULL) {
20460 pm_array_elements_append(parser, &elements, current);
20461 current = NULL;
20462 } else {
20463 expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_W_LOWER_ELEMENT);
20464 }
20465 }
20466
20467 pm_token_t closing = parser->current;
20468 if (match1(parser, PM_TOKEN_EOF)) {
20469 pm_parser_err_token(parser, &opening, PM_ERR_LIST_W_LOWER_TERM);
20470 closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
20471 } else {
20472 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_LOWER_TERM);
20473 }
20474
20475 return pm_array_node_create(parser, &opening, &elements, &closing);
20476}
20477
20478static pm_array_node_t *
20479parse_interpolated_string_array(pm_parser_t *parser, uint16_t depth) {
20480 pm_token_t opening = parser->current;
20481 parser_lex(parser);
20482
20483 pm_array_elements_t elements = { 0 };
20484
20485 /* The element being parsed is either a single string node in current, or,
20486 * once it needs more than one part, the open list of parts of an
20487 * interpolated string. */
20488 pm_node_t *current = NULL;
20489 pm_node_list_t parts = { 0 };
20490
20491 while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
20492 switch (parser->current.type) {
20493 case PM_TOKEN_WORDS_SEP: {
20494 /* Reset the explicit encoding if we hit a separator since each
20495 * element can have its own encoding. */
20496 parser->explicit_encoding = NULL;
20497
20498 /* If we hit a separator after we've hit content, then we need
20499 * to append that content to the list and reset the element. */
20500 if (parts.size > 0) {
20501 pm_array_elements_append(parser, &elements, UP(pm_interpolated_string_node_create(parser, NULL, &parts, NULL)));
20502 parts = (pm_node_list_t) { 0 };
20503 } else if (current != NULL) {
20504 pm_array_elements_append(parser, &elements, current);
20505 current = NULL;
20506 }
20507
20508 parser_lex(parser);
20509 break;
20510 }
20511 case PM_TOKEN_STRING_CONTENT: {
20512 pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL));
20513 pm_node_flag_set(string, parse_unescaped_encoding(parser, parser->explicit_encoding));
20514 parser_lex(parser);
20515
20516 if (parts.size > 0) {
20517 /* If we hit string content and the element is already
20518 * interpolated, then we need to append the string content
20519 * to its parts. */
20520 pm_node_list_open_append(&parser->node_chunks, &parts, string);
20521 } else if (current == NULL) {
20522 /* If we hit content and there is no element yet, then this
20523 * is the first string content we've seen, and it becomes
20524 * the element. */
20525 current = string;
20526 } else {
20527 /* If we hit string content and the element is a string
20528 * node, then we need to turn the element into the parts of
20529 * an interpolated string and add the string content to
20530 * them. */
20531 pm_node_list_open_append(&parser->node_chunks, &parts, current);
20532 pm_node_list_open_append(&parser->node_chunks, &parts, string);
20533 current = NULL;
20534 }
20535
20536 break;
20537 }
20538 case PM_TOKEN_EMBVAR:
20539 case PM_TOKEN_EMBEXPR_BEGIN: {
20540 /* If we hit an embedded variable or expression and the element
20541 * is a string node, then we'll turn the element into the parts
20542 * of an interpolated string, starting with that string. */
20543 if (current != NULL) {
20544 pm_node_list_open_append(&parser->node_chunks, &parts, current);
20545 current = NULL;
20546 }
20547
20548 pm_node_t *part = parse_string_part(parser, (uint16_t) (depth + 1));
20549 pm_node_list_open_append(&parser->node_chunks, &parts, part);
20550 break;
20551 }
20552 default:
20553 expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_W_UPPER_ELEMENT);
20554 parse_discard_token(parser, depth);
20555 break;
20556 }
20557 }
20558
20559 /* If we have an element, then we need to append it to the list. */
20560 if (parts.size > 0) {
20561 pm_array_elements_append(parser, &elements, UP(pm_interpolated_string_node_create(parser, NULL, &parts, NULL)));
20562 } else if (current != NULL) {
20563 pm_array_elements_append(parser, &elements, current);
20564 }
20565
20566 pm_token_t closing = parser->current;
20567 if (match1(parser, PM_TOKEN_EOF)) {
20568 pm_parser_err_token(parser, &opening, PM_ERR_LIST_W_UPPER_TERM);
20569 closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
20570 } else {
20571 expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_UPPER_TERM);
20572 }
20573
20574 return pm_array_node_create(parser, &opening, &elements, &closing);
20575}
20576
20577static pm_node_t *
20578parse_jump(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
20579 parser_lex(parser);
20580
20581 pm_token_t keyword = parser->previous;
20582 pm_arguments_t arguments = { 0 };
20583
20584 if (token_begins_expression_p(parser->current.type) || match2(parser, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR)) {
20585 pm_binding_power_t binding_power = pm_binding_powers[parser->current.type].left;
20586
20587 if (binding_power == PM_BINDING_POWER_UNSET || binding_power >= PM_BINDING_POWER_RANGE) {
20588 pm_token_t next = parser->current;
20589
20590 /* The arguments of a jump start a fresh command context, so a
20591 * `do` inside them binds to the nearest call within the
20592 * arguments rather than to an enclosing command call, as in
20593 * `check 1, case x when 1 then return bar do end end`. */
20594 pm_accepts_block_stack_push(parser, true);
20595 parse_arguments(parser, &arguments, false, PM_TOKEN_EOF, flags, (uint16_t) (depth + 1));
20596 pm_accepts_block_stack_pop(parser);
20597
20598 // Reject `foo && return bar`.
20599 if (!(flags & PM_PARSE_ACCEPTS_COMMAND_CALL) && arguments.list.size > 0) {
20600 PM_PARSER_ERR_TOKEN_FORMAT(parser, &next, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(next.type));
20601 }
20602
20603 // Reject a trailing comma, e.g. `return a,`. The arguments
20604 // parser silently accepts a trailing comma only when it is
20605 // immediately followed by the EOF terminator; in every other
20606 // case (e.g. `return a,;`) it reports the dangling comma
20607 // itself. We reject the accepted case here to stay in line
20608 // with the command call argument parsing above.
20609 if (parser->previous.type == PM_TOKEN_COMMA && match1(parser, PM_TOKEN_EOF)) {
20610 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_str(parser->current.type));
20611 }
20612 }
20613
20614 // It's possible that we've parsed a block argument through our
20615 // call to parse_arguments. If we found one, we should mark it
20616 // as invalid and destroy it, as we don't have a place for it.
20617 if (arguments.block != NULL) {
20618 pm_parser_err_node(parser, arguments.block, PM_ERR_UNEXPECTED_BLOCK_ARGUMENT);
20619 pm_node_unreference(parser, arguments.block);
20620 arguments.block = NULL;
20621 }
20622 }
20623
20624 switch (keyword.type) {
20625 case PM_TOKEN_KEYWORD_BREAK: {
20626 pm_node_t *node = UP(pm_break_node_create(parser, &keyword, pm_arguments_node_create(parser, &arguments)));
20627 if (!parser->partial_script) ensure_valid_jump(parser, node);
20628 return node;
20629 }
20630 case PM_TOKEN_KEYWORD_NEXT: {
20631 pm_node_t *node = UP(pm_next_node_create(parser, &keyword, pm_arguments_node_create(parser, &arguments)));
20632 if (!parser->partial_script) ensure_valid_jump(parser, node);
20633 return node;
20634 }
20635 case PM_TOKEN_KEYWORD_RETURN: {
20636 pm_node_t *node = UP(pm_return_node_create(parser, &keyword, pm_arguments_node_create(parser, &arguments)));
20637 ensure_valid_return(parser, node);
20638 return node;
20639 }
20640 default:
20641 PRISM_ASSERT(false && "unreachable");
20642 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)));
20643 }
20644}
20645
20646static pm_node_t *
20647parse_not(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20648 pm_token_t message = parser->current;
20649 parser_lex(parser);
20650
20651 pm_arguments_t arguments = { 0 };
20652 pm_node_t *receiver = NULL;
20653
20654 /* The `not` keyword without parentheses is only valid in contexts where it
20655 * would be parsed as an expression (i.e., at or below the `not` binding
20656 * power level). In other contexts (e.g., method arguments, array elements,
20657 * assignment right-hand sides), parentheses are required: `not(x)`. An
20658 * exception is made for endless def bodies, where `not` is valid as both
20659 * `arg` and `command` (e.g., `def f = not 1`, `def f = not foo bar`). */
20660 if (binding_power > PM_BINDING_POWER_NOT && !(flags & PM_PARSE_IN_ENDLESS_DEF) && !match1(parser, PM_TOKEN_PARENTHESIS_LEFT)) {
20661 if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES)) {
20662 pm_parser_err(parser, PM_TOKEN_END(parser, &parser->previous), 1, PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN);
20663 } else {
20664 accept1(parser, PM_TOKEN_NEWLINE);
20665 pm_parser_err_current(parser, PM_ERR_EXPECT_LPAREN_AFTER_NOT_OTHER);
20666 }
20667
20668 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_END(parser, &parser->previous), 0));
20669 }
20670
20671 bool newline = accept1(parser, PM_TOKEN_NEWLINE);
20672
20673 if (!newline && accept2(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) {
20674 pm_token_t lparen = parser->previous;
20675
20676 if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
20677 receiver = UP(pm_parentheses_node_create(parser, &lparen, NULL, &parser->previous, 0));
20678 } else {
20679 arguments.opening_loc = TOK2LOC(parser, &lparen);
20680 receiver = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_NOT_EXPRESSION, (uint16_t) (depth + 1));
20681
20682 if (!parser->recovering) {
20683 accept1(parser, PM_TOKEN_NEWLINE);
20684 expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
20685 arguments.closing_loc = TOK2LOC(parser, &parser->previous);
20686 }
20687 }
20688 } else {
20689 /* The operand of `not` may be a command call, except at the start of
20690 * an endless method body, where it may be one only if the body may
20691 * be: `def f = not foo 1` has a command body, so it is rejected where
20692 * that is not accepted (`if def f = not foo 1; end`). A nested `not`
20693 * is still at the start of the body, so the operand carries the
20694 * endless method context on (`def f = not not foo 1`). */
20695 uint8_t operand_flags = (flags & PM_PARSE_IN_ENDLESS_DEF)
20696 ? (uint8_t) (flags & (PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_IN_ENDLESS_DEF))
20697 : PM_PARSE_ACCEPTS_COMMAND_CALL;
20698 receiver = parse_expression(parser, PM_BINDING_POWER_NOT, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | operand_flags), PM_ERR_NOT_EXPRESSION, (uint16_t) (depth + 1));
20699 }
20700
20701 return UP(pm_call_node_not_create(parser, receiver, &message, &arguments));
20702}
20703
20704static pm_pre_execution_node_t *
20705parse_preexe(pm_parser_t *parser, pm_binding_power_t binding_power, uint16_t depth) {
20706 pm_scratch_node_list_t current_block_exits = { 0 };
20707 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
20708
20709 if (binding_power != PM_BINDING_POWER_STATEMENT) {
20710 pm_parser_err_current(parser, PM_ERR_STATEMENT_PREEXE_BEGIN);
20711 }
20712
20713 pm_token_t keyword = parser->current;
20714 parser_lex(parser);
20715
20716 expect1(parser, PM_TOKEN_BRACE_LEFT, PM_ERR_BEGIN_UPCASE_BRACE);
20717 pm_token_t opening = parser->previous;
20718 pm_statements_node_t *statements = parse_statements(parser, PM_CONTEXT_PREEXE, (uint16_t) (depth + 1));
20719
20720 expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_BEGIN_UPCASE_TERM, &opening);
20721 pm_context_t context = parser->current_context->context;
20722 if ((context != PM_CONTEXT_MAIN) && (context != PM_CONTEXT_PREEXE)) {
20723 pm_parser_err_token(parser, &keyword, PM_ERR_BEGIN_UPCASE_TOPLEVEL);
20724 }
20725
20726 flush_block_exits(parser, previous_block_exits);
20727 return pm_pre_execution_node_create(parser, &keyword, &opening, statements, &parser->previous);
20728}
20729
20730static pm_post_execution_node_t *
20731parse_postexe(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20732 if (binding_power != PM_BINDING_POWER_STATEMENT && !(flags & PM_PARSE_ACCEPTS_STATEMENT)) {
20733 pm_parser_err_current(parser, PM_ERR_STATEMENT_POSTEXE_END);
20734 }
20735
20736 pm_token_t keyword = parser->current;
20737 parser_lex(parser);
20738
20739 if (context_def_p(parser)) {
20740 pm_parser_warn_token(parser, &keyword, PM_WARN_END_IN_METHOD);
20741 }
20742
20743 expect1(parser, PM_TOKEN_BRACE_LEFT, PM_ERR_END_UPCASE_BRACE);
20744 pm_token_t opening = parser->previous;
20745 pm_statements_node_t *statements = parse_statements(parser, PM_CONTEXT_POSTEXE, (uint16_t) (depth + 1));
20746
20747 expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_END_UPCASE_TERM, &opening);
20748 return pm_post_execution_node_create(parser, &keyword, &opening, statements, &parser->previous);
20749}
20750
20751static pm_undef_node_t *
20752parse_undef(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20753 if (binding_power != PM_BINDING_POWER_STATEMENT && !(flags & PM_PARSE_ACCEPTS_STATEMENT)) {
20754 pm_parser_err_current(parser, PM_ERR_STATEMENT_UNDEF);
20755 }
20756
20757 parser_lex(parser);
20758 pm_token_t keyword = parser->previous;
20759
20760 pm_node_list_t names = { 0 };
20761 pm_node_t *name = parse_undef_argument(parser, (uint16_t) (depth + 1));
20762
20763 if (!PM_NODE_TYPE_P(name, PM_ERROR_RECOVERY_NODE)) {
20764 pm_node_list_open_append(&parser->node_chunks, &names, name);
20765
20766 while (match1(parser, PM_TOKEN_COMMA)) {
20767 lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM);
20768 parser_lex(parser);
20769
20770 name = parse_undef_argument(parser, (uint16_t) (depth + 1));
20771 if (PM_NODE_TYPE_P(name, PM_ERROR_RECOVERY_NODE)) {
20772 break;
20773 }
20774
20775 pm_node_list_open_append(&parser->node_chunks, &names, name);
20776 }
20777 }
20778
20779 return pm_undef_node_create(parser, &keyword, &names);
20780}
20781
20782static pm_yield_node_t *
20783parse_yield(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
20784 pm_token_t keyword = parser->current;
20785 parser_lex(parser);
20786
20787 pm_arguments_t arguments = { 0 };
20788 parse_arguments_list(parser, &arguments, false, flags, (uint16_t) (depth + 1));
20789
20790 /* It's possible that we've parsed a block argument through our call to
20791 * parse_arguments_list. If we found one, we should mark it as invalid and
20792 * destroy it, as we don't have a place for it on the yield node. */
20793 if (arguments.block != NULL) {
20794 pm_parser_err_node(parser, arguments.block, PM_ERR_UNEXPECTED_BLOCK_ARGUMENT);
20795 pm_node_unreference(parser, arguments.block);
20796 arguments.block = NULL;
20797 }
20798
20799 pm_yield_node_t *node = pm_yield_node_create(parser, &keyword, &arguments.opening_loc, pm_arguments_node_create(parser, &arguments), &arguments.closing_loc);
20800 if (!parser->parsing_eval && !parser->partial_script) ensure_valid_yield(parser, node);
20801
20802 return node;
20803}
20804
20805static pm_node_t *
20806parse_char(pm_parser_t *parser, uint16_t depth) {
20807 pm_node_t *node = UP(pm_string_node_create_current_string(
20808 parser,
20809 &(pm_token_t) {
20810 .type = PM_TOKEN_STRING_BEGIN,
20811 .start = parser->current.start,
20812 .end = parser->current.start + 1
20813 },
20814 &(pm_token_t) {
20815 .type = PM_TOKEN_STRING_CONTENT,
20816 .start = parser->current.start + 1,
20817 .end = parser->current.end
20818 },
20819 NULL
20820 ));
20821
20822 pm_node_flag_set(node, parse_unescaped_encoding(parser, parser->explicit_encoding));
20823
20824 /* Skip past the character literal here, since now we have handled
20825 * parser->explicit_encoding correctly. */
20826 parser_lex(parser);
20827
20828 /* Characters can be followed by strings in which case they are
20829 * automatically concatenated. */
20830 if (match1(parser, PM_TOKEN_STRING_BEGIN)) {
20831 return parse_strings(parser, node, false, (uint16_t) (depth + 1));
20832 }
20833
20834 return node;
20835}
20836
20837static pm_node_t *
20838parse_constant(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20839 pm_token_t constant = parser->current;
20840 parser_lex(parser);
20841
20842 /* If a constant is immediately followed by parentheses, then this is in
20843 * fact a method call, not a constant read. */
20844 if (
20845 match1(parser, PM_TOKEN_PARENTHESIS_LEFT) ||
20846 ((flags & PM_PARSE_ACCEPTS_COMMAND_CALL) && token_begins_argument_p(parser->current.type)) ||
20847 (pm_accepts_block_stack_p(parser) && match1(parser, PM_TOKEN_KEYWORD_DO)) ||
20848 match1(parser, PM_TOKEN_BRACE_LEFT)
20849 ) {
20850 pm_arguments_t arguments = { 0 };
20851 parse_arguments_list(parser, &arguments, true, flags, (uint16_t) (depth + 1));
20852 return UP(pm_call_node_fcall_create(parser, &constant, &arguments));
20853 }
20854
20855 return parse_multi_target(parser, UP(pm_constant_read_node_create(parser, &parser->previous)), binding_power, depth);
20856}
20857
20858static pm_node_t *
20859parse_constant_path_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, uint16_t depth) {
20860 pm_token_t delimiter = parser->current;
20861 parser_lex(parser);
20862
20863 expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT);
20864 return parse_multi_target(parser, UP(pm_constant_path_node_create(parser, NULL, &delimiter, &parser->previous)), binding_power, depth);
20865}
20866
20867static pm_call_node_t *
20868parse_unary_plus(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20869 if (binding_power > PM_BINDING_POWER_UNARY) {
20870 pm_parser_err_prefix(parser, PM_ERR_UNARY_DISALLOWED);
20871 }
20872 parser_lex(parser);
20873
20874 pm_token_t operator = parser->previous;
20875 pm_node_t *receiver = parse_expression(parser, pm_binding_powers[parser->previous.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_UNARY_RECEIVER, (uint16_t) (depth + 1));
20876 return pm_call_node_unary_create(parser, &operator, receiver, "+@");
20877}
20878
20879static pm_call_node_t *
20880parse_unary_minus(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20881 if (binding_power > PM_BINDING_POWER_UNARY) {
20882 pm_parser_err_prefix(parser, PM_ERR_UNARY_DISALLOWED);
20883 }
20884 parser_lex(parser);
20885
20886 pm_token_t operator = parser->previous;
20887 pm_node_t *receiver = parse_expression(parser, pm_binding_powers[parser->previous.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_UNARY_RECEIVER, (uint16_t) (depth + 1));
20888 return pm_call_node_unary_create(parser, &operator, receiver, "-@");
20889}
20890
20891static pm_node_t *
20892parse_unary_minus_number(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
20893 pm_token_t operator = parser->current;
20894 parser_lex(parser);
20895
20896 pm_node_t *node = parse_expression(parser, pm_binding_powers[parser->previous.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_UNARY_RECEIVER, (uint16_t) (depth + 1));
20897
20898 if (accept1(parser, PM_TOKEN_STAR_STAR)) {
20899 pm_token_t exponent_operator = parser->previous;
20900 pm_node_t *exponent = parse_expression(parser, pm_binding_powers[exponent_operator.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_ARGUMENT, (uint16_t) (depth + 1));
20901 node = UP(pm_call_node_binary_create(parser, node, &exponent_operator, exponent, 0));
20902 node = UP(pm_call_node_unary_create(parser, &operator, node, "-@"));
20903 } else {
20904 switch (PM_NODE_TYPE(node)) {
20905 case PM_INTEGER_NODE:
20906 case PM_FLOAT_NODE:
20907 case PM_RATIONAL_NODE:
20908 case PM_IMAGINARY_NODE:
20909 parse_negative_numeric(node);
20910 break;
20911 default:
20912 node = UP(pm_call_node_unary_create(parser, &operator, node, "-@"));
20913 break;
20914 }
20915 }
20916
20917 return node;
20918}
20919
20920static pm_node_t *
20921parse_unary_star(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
20922 parser_lex(parser);
20923
20924 /* * operators at the beginning of expressions are only valid in the context
20925 * of a multiple assignment. We enforce that here. We'll still lex past it
20926 * though and create an error recovery node. */
20927 if (binding_power != PM_BINDING_POWER_STATEMENT) {
20928 pm_parser_err_prefix(parser, diag_id);
20929 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)));
20930 }
20931
20932 pm_node_t *splat = parse_splat(parser, flags, depth);
20933
20934 if (match1(parser, PM_TOKEN_COMMA)) {
20935 return parse_targets_validate(parser, splat, PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1));
20936 } else {
20937 return parse_target_validate(parser, splat, true);
20938 }
20939}
20940
20941static pm_node_t *
20942parse_super(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
20943 pm_token_t keyword = parser->current;
20944 parser_lex(parser);
20945
20946 pm_arguments_t arguments = { 0 };
20947 parse_arguments_list(parser, &arguments, true, flags, (uint16_t) (depth + 1));
20948
20949 if (
20950 arguments.opening_loc.length == 0 &&
20951 arguments.list.size == 0 &&
20952 ((arguments.block == NULL) || PM_NODE_TYPE_P(arguments.block, PM_BLOCK_NODE))
20953 ) {
20954 return UP(pm_forwarding_super_node_create(parser, &keyword, &arguments));
20955 }
20956
20957 return UP(pm_super_node_create(parser, &keyword, &arguments));
20958}
20959
20960static pm_range_node_t *
20961parse_range(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
20962 pm_token_t operator = parser->current;
20963 parser_lex(parser);
20964
20965 pm_node_t *right = parse_expression(parser, pm_binding_powers[operator.type].left, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
20966
20967 /* Unary .. and ... are special because these are non-associative operators
20968 * that can also be unary operators. In this case we need to explicitly
20969 * reject code that has a .. or ... that follows this expression. */
20970 if (match2(parser, PM_TOKEN_DOT_DOT, PM_TOKEN_DOT_DOT_DOT)) {
20971 pm_parser_err_current(parser, PM_ERR_UNEXPECTED_RANGE_OPERATOR);
20972 }
20973
20974 return pm_range_node_create(parser, NULL, &operator, right);
20975}
20976
20977static pm_call_node_t *
20978parse_bang(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20979 if (binding_power > PM_BINDING_POWER_UNARY) {
20980 pm_parser_err_prefix(parser, PM_ERR_UNARY_DISALLOWED);
20981 }
20982
20983 parser_lex(parser);
20984
20985 pm_token_t operator = parser->previous;
20986 pm_node_t *receiver = parse_expression(parser, pm_binding_powers[parser->previous.type].right, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | (binding_power < PM_BINDING_POWER_MATCH ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0)), PM_ERR_UNARY_RECEIVER, (uint16_t) (depth + 1));
20987 pm_conditional_predicate(parser, receiver, PM_CONDITIONAL_PREDICATE_TYPE_NOT);
20988 return pm_call_node_unary_create(parser, &operator, receiver, "!");
20989}
20990
20991static pm_call_node_t *
20992parse_tilde(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
20993 if (binding_power > PM_BINDING_POWER_UNARY) {
20994 pm_parser_err_prefix(parser, PM_ERR_UNARY_DISALLOWED);
20995 }
20996
20997 parser_lex(parser);
20998
20999 pm_token_t operator = parser->previous;
21000 pm_node_t *receiver = parse_expression(parser, pm_binding_powers[parser->previous.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_UNARY_RECEIVER, (uint16_t) (depth + 1));
21001 return pm_call_node_unary_create(parser, &operator, receiver, "~");
21002}
21003
21007static PRISM_INLINE pm_node_t *
21008parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
21009 switch (parser->current.type) {
21010 case PM_TOKEN_BRACKET_LEFT_ARRAY:
21011 return UP(parse_array(parser, flags, depth));
21012 case PM_TOKEN_PARENTHESIS_LEFT_GROUPING:
21013 case PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES:
21014 return parse_parentheses(parser, binding_power, flags, depth);
21015 case PM_TOKEN_BRACE_LEFT_HASH:
21016 return UP(parse_hash(parser, depth));
21017 case PM_TOKEN_CHARACTER_LITERAL:
21018 return parse_char(parser, depth);
21019 case PM_TOKEN_CONSTANT:
21020 return parse_constant(parser, binding_power, flags, depth);
21021 case PM_TOKEN_UCOLON_COLON:
21022 return parse_constant_path_prefix(parser, binding_power, depth);
21023 case PM_TOKEN_UDOT_DOT:
21024 case PM_TOKEN_UDOT_DOT_DOT:
21025 return UP(parse_range(parser, flags, depth));
21026 case PM_TOKEN_FLOAT:
21027 parser_lex(parser);
21028 return UP(pm_float_node_create(parser, &parser->previous));
21029 case PM_TOKEN_FLOAT_IMAGINARY:
21030 parser_lex(parser);
21031 return UP(pm_float_node_imaginary_create(parser, &parser->previous));
21032 case PM_TOKEN_FLOAT_RATIONAL:
21033 parser_lex(parser);
21034 return UP(pm_float_node_rational_create(parser, &parser->previous));
21035 case PM_TOKEN_FLOAT_RATIONAL_IMAGINARY:
21036 parser_lex(parser);
21037 return UP(pm_float_node_rational_imaginary_create(parser, &parser->previous));
21038 case PM_TOKEN_IDENTIFIER:
21039 case PM_TOKEN_METHOD_NAME:
21040 return parse_identifier(parser, binding_power, flags, depth);
21041 case PM_TOKEN_HEREDOC_START: {
21042 pm_node_t *node = parse_heredoc(parser, depth);
21043 if (match1(parser, PM_TOKEN_STRING_BEGIN)) {
21044 return parse_strings(parser, node, false, (uint16_t) (depth + 1));
21045 }
21046 return node;
21047 }
21048 case PM_TOKEN_CLASS_VARIABLE:
21049 parser_lex(parser);
21050 return parse_multi_target(parser, UP(pm_class_variable_read_node_create(parser, &parser->previous)), binding_power, depth);
21051 case PM_TOKEN_NUMBERED_REFERENCE:
21052 parser_lex(parser);
21053 return parse_multi_target(parser, UP(pm_numbered_reference_read_node_create(parser, &parser->previous)), binding_power, depth);
21054 case PM_TOKEN_GLOBAL_VARIABLE:
21055 parser_lex(parser);
21056 return parse_multi_target(parser, UP(pm_global_variable_read_node_create(parser, &parser->previous)), binding_power, depth);
21057 case PM_TOKEN_BACK_REFERENCE:
21058 parser_lex(parser);
21059 return parse_multi_target(parser, UP(pm_back_reference_read_node_create(parser, &parser->previous)), binding_power, depth);
21060 case PM_TOKEN_INSTANCE_VARIABLE:
21061 parser_lex(parser);
21062 return parse_multi_target(parser, UP(pm_instance_variable_read_node_create(parser, &parser->previous)), binding_power, depth);
21063 case PM_TOKEN_INTEGER: {
21064 pm_node_flags_t base = parser->integer.base;
21065 parser_lex(parser);
21066 return UP(pm_integer_node_create(parser, base, &parser->previous));
21067 }
21068 case PM_TOKEN_INTEGER_IMAGINARY: {
21069 pm_node_flags_t base = parser->integer.base;
21070 parser_lex(parser);
21071 return UP(pm_integer_node_imaginary_create(parser, base, &parser->previous));
21072 }
21073 case PM_TOKEN_INTEGER_RATIONAL: {
21074 pm_node_flags_t base = parser->integer.base;
21075 parser_lex(parser);
21076 return UP(pm_integer_node_rational_create(parser, base, &parser->previous));
21077 }
21078 case PM_TOKEN_INTEGER_RATIONAL_IMAGINARY: {
21079 pm_node_flags_t base = parser->integer.base;
21080 parser_lex(parser);
21081 return UP(pm_integer_node_rational_imaginary_create(parser, base, &parser->previous));
21082 }
21083 case PM_TOKEN_KEYWORD___ENCODING__:
21084 parser_lex(parser);
21085 return UP(pm_source_encoding_node_create(parser, &parser->previous));
21086 case PM_TOKEN_KEYWORD___FILE__:
21087 parser_lex(parser);
21088 return UP(pm_source_file_node_create(parser, &parser->previous));
21089 case PM_TOKEN_KEYWORD___LINE__:
21090 parser_lex(parser);
21091 return UP(pm_source_line_node_create(parser, &parser->previous));
21092 case PM_TOKEN_KEYWORD_ALIAS:
21093 return parse_alias(parser, binding_power, flags, depth);
21094 case PM_TOKEN_KEYWORD_CASE:
21095 return parse_case(parser, flags, depth);
21096 case PM_TOKEN_KEYWORD_BEGIN:
21097 return UP(parse_begin(parser, depth));
21098 case PM_TOKEN_KEYWORD_BEGIN_UPCASE:
21099 return UP(parse_preexe(parser, binding_power, depth));
21100 case PM_TOKEN_KEYWORD_BREAK:
21101 case PM_TOKEN_KEYWORD_NEXT:
21102 case PM_TOKEN_KEYWORD_RETURN:
21103 return parse_jump(parser, flags, depth);
21104 case PM_TOKEN_KEYWORD_SUPER:
21105 return parse_super(parser, flags, depth);
21106 case PM_TOKEN_KEYWORD_YIELD:
21107 return UP(parse_yield(parser, flags, depth));
21108 case PM_TOKEN_KEYWORD_CLASS:
21109 return parse_class(parser, flags, depth);
21110 case PM_TOKEN_KEYWORD_DEF:
21111 return parse_def(parser, flags, depth);
21112 case PM_TOKEN_KEYWORD_DEFINED:
21113 return UP(parse_defined(parser, flags, depth));
21114 case PM_TOKEN_KEYWORD_END_UPCASE:
21115 return UP(parse_postexe(parser, binding_power, flags, depth));
21116 case PM_TOKEN_KEYWORD_FALSE:
21117 parser_lex(parser);
21118 return UP(pm_false_node_create(parser, &parser->previous));
21119 case PM_TOKEN_KEYWORD_FOR:
21120 return UP(parse_for(parser, flags, depth));
21121 case PM_TOKEN_KEYWORD_IF:
21122 if (parser_end_of_line_p(parser)) {
21123 PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_KEYWORD_EOL);
21124 }
21125
21126 size_t opening_newline_index = token_newline_index(parser);
21127 bool if_after_else = parser->previous.type == PM_TOKEN_KEYWORD_ELSE;
21128 parser_lex(parser);
21129
21130 return parse_conditional(parser, PM_CONTEXT_IF, opening_newline_index, if_after_else, (uint16_t) (depth + 1));
21131 case PM_TOKEN_KEYWORD_UNDEF:
21132 return UP(parse_undef(parser, binding_power, flags, depth));
21133 case PM_TOKEN_KEYWORD_NOT:
21134 return parse_not(parser, binding_power, flags, depth);
21135 case PM_TOKEN_KEYWORD_UNLESS: {
21136 size_t opening_newline_index = token_newline_index(parser);
21137 parser_lex(parser);
21138
21139 return parse_conditional(parser, PM_CONTEXT_UNLESS, opening_newline_index, false, (uint16_t) (depth + 1));
21140 }
21141 case PM_TOKEN_KEYWORD_MODULE:
21142 return parse_module(parser, flags, depth);
21143 case PM_TOKEN_KEYWORD_NIL:
21144 parser_lex(parser);
21145 return UP(pm_nil_node_create(parser, &parser->previous));
21146 case PM_TOKEN_KEYWORD_REDO: {
21147 parser_lex(parser);
21148
21149 pm_node_t *node = UP(pm_redo_node_create(parser, &parser->previous));
21150 if (!parser->partial_script) ensure_valid_jump(parser, node);
21151
21152 return node;
21153 }
21154 case PM_TOKEN_KEYWORD_RETRY: {
21155 parser_lex(parser);
21156
21157 pm_node_t *node = UP(pm_retry_node_create(parser, &parser->previous));
21158 ensure_valid_retry(parser, node);
21159
21160 return node;
21161 }
21162 case PM_TOKEN_KEYWORD_SELF:
21163 parser_lex(parser);
21164 return UP(pm_self_node_create(parser, &parser->previous));
21165 case PM_TOKEN_KEYWORD_TRUE:
21166 parser_lex(parser);
21167 return UP(pm_true_node_create(parser, &parser->previous));
21168 case PM_TOKEN_KEYWORD_UNTIL:
21169 return UP(parse_until(parser, flags, depth));
21170 case PM_TOKEN_KEYWORD_WHILE:
21171 return UP(parse_while(parser, flags, depth));
21172 case PM_TOKEN_PERCENT_LOWER_I:
21173 return UP(parse_uninterpolated_symbol_array(parser));
21174 case PM_TOKEN_PERCENT_UPPER_I:
21175 return UP(parse_interpolated_symbol_array(parser, depth));
21176 case PM_TOKEN_PERCENT_LOWER_W:
21177 return UP(parse_uninterpolated_string_array(parser));
21178 case PM_TOKEN_PERCENT_UPPER_W:
21179 return UP(parse_interpolated_string_array(parser, depth));
21180 case PM_TOKEN_REGEXP_BEGIN:
21181 return parse_regexp(parser, depth);
21182 case PM_TOKEN_XSTRING_BEGIN:
21183 case PM_TOKEN_PERCENT_LOWER_X:
21184 return parse_xstring(parser, depth);
21185 case PM_TOKEN_USTAR:
21186 return parse_unary_star(parser, binding_power, flags, diag_id, depth);
21187 case PM_TOKEN_BANG:
21188 return UP(parse_bang(parser, binding_power, flags, depth));
21189 case PM_TOKEN_TILDE:
21190 return UP(parse_tilde(parser, binding_power, flags, depth));
21191 case PM_TOKEN_UMINUS:
21192 return UP(parse_unary_minus(parser, binding_power, flags, depth));
21193 case PM_TOKEN_UMINUS_NUM:
21194 return UP(parse_unary_minus_number(parser, flags, depth));
21195 case PM_TOKEN_MINUS_GREATER:
21196 return UP(parse_lambda(parser, depth));
21197 case PM_TOKEN_UPLUS:
21198 return UP(parse_unary_plus(parser, binding_power, flags, depth));
21199 case PM_TOKEN_STRING_BEGIN:
21200 return parse_strings(parser, NULL, flags & PM_PARSE_ACCEPTS_LABEL, (uint16_t) (depth + 1));
21201 case PM_TOKEN_SYMBOL_BEGIN:
21202 return parse_symbol(parser, PM_LEX_STATE_END, (uint16_t) (depth + 1));
21203 default: {
21204 pm_context_t recoverable = context_recoverable(parser, &parser->current);
21205
21206 if (recoverable != PM_CONTEXT_NONE) {
21207 parser->recovering = true;
21208
21209 // If the given error is not the generic one, then we'll add it
21210 // here because it will provide more context in addition to the
21211 // recoverable error that we will also add.
21212 if (diag_id != PM_ERR_CANNOT_PARSE_EXPRESSION) {
21213 pm_parser_err_prefix(parser, diag_id);
21214 }
21215
21216 // If we get here, then we are assuming this token is closing a
21217 // parent context, so we'll indicate that to the user so that
21218 // they know how we behaved.
21219 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT, pm_token_str(parser->current.type), context_human(recoverable));
21220 } else if (diag_id == PM_ERR_CANNOT_PARSE_EXPRESSION) {
21221 // We're going to make a special case here, because "cannot
21222 // parse expression" is pretty generic, and we know here that we
21223 // have an unexpected token.
21224 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, pm_token_str(parser->current.type));
21225 } else {
21226 pm_parser_err_prefix(parser, diag_id);
21227 }
21228
21229 return UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)));
21230 }
21231 }
21232}
21233
21234static pm_node_t *
21235parse_rescue_modifier_value(pm_parser_t *parser, uint8_t flags, bool statement, uint16_t depth);
21236
21244static void
21245parse_rescue_modifier_terminator(pm_parser_t *parser, uint8_t flags, uint16_t depth) {
21246 if (pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_MODIFIER) {
21247 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->current.type));
21248 parser_lex(parser);
21249 parse_expression(parser, pm_binding_powers[parser->previous.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
21250 }
21251}
21252
21258static void
21259parse_write_value_comma_check(pm_parser_t *parser, const pm_node_t *value) {
21260 if (match1(parser, PM_TOKEN_COMMA) && pm_command_before_comma_p(parser, value)) {
21261 pm_parser_err_current(parser, PM_ERR_INVALID_COMMA);
21262 }
21263}
21264
21274static pm_node_t *
21275parse_assignment_value(pm_parser_t *parser, pm_binding_power_t previous_binding_power, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
21276 /* Where an operator write accepts a command call it also accepts an
21277 * endless method with a command body (`x += def f = foo 1`). */
21278 uint8_t command_flags = (previous_binding_power == PM_BINDING_POWER_ASSIGNMENT)
21279 ? (uint8_t) (flags & PM_PARSE_ACCEPTS_COMMAND_CALL)
21280 : (previous_binding_power < PM_BINDING_POWER_MATCH ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0);
21281 if (command_flags) command_flags |= PM_PARSE_ACCEPTS_COMMAND_DEF;
21282
21283 pm_node_t *value = parse_value_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | command_flags), diag_id, (uint16_t) (depth + 1));
21284
21285 /* An operator write never takes a multi-value right-hand side, so a comma
21286 * after a statement-level one is already reported by the statement that
21287 * contains it. */
21288 if (previous_binding_power != PM_BINDING_POWER_STATEMENT && !(flags & PM_PARSE_ACCEPTS_STATEMENT)) {
21289 parse_write_value_comma_check(parser, value);
21290 }
21291
21292 // Assignments whose value is a command call (e.g., a = b c) can only
21293 // be followed by modifiers (if/unless/while/until/rescue) and not by
21294 // operators with higher binding power. If we find one, emit an error
21295 // and skip the operator and its right-hand side.
21296 if (pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_MODIFIER && (pm_command_call_value_p(parser, value) || pm_block_call_p(value))) {
21297 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->current.type));
21298 parser_lex(parser);
21299 parse_expression(parser, pm_binding_powers[parser->previous.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
21300 }
21301
21302 // Contradicting binding powers, the right-hand-side value of the assignment
21303 // allows the `rescue` modifier.
21304 if (match1(parser, PM_TOKEN_KEYWORD_RESCUE_MODIFIER)) {
21305 context_push(parser, PM_CONTEXT_RESCUE_MODIFIER);
21306
21307 pm_token_t rescue = parser->current;
21308 parser_lex(parser);
21309
21310 // As in parse_assignment_values, the resbody is a `stmt` (permitting a
21311 // multiple assignment / command call) when the rescued value is itself a
21312 // command call, and a plain `arg` otherwise.
21313 bool statement_value = pm_command_call_value_p(parser, value) || pm_block_call_p(value);
21314 uint8_t rescue_flags = (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | (statement_value ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0));
21315
21316 pm_node_t *right = parse_rescue_modifier_value(parser, rescue_flags, statement_value, (uint16_t) (depth + 1));
21317 context_pop(parser);
21318
21319 // A pattern-match resbody is a statement, but here the rescue is nested
21320 // in an assignment value where parse_expression_terminator cannot see
21321 // it, so reject a trailing operator above the modifier level directly.
21322 if (PM_NODE_TYPE_P(right, PM_MATCH_REQUIRED_NODE) || PM_NODE_TYPE_P(right, PM_MATCH_PREDICATE_NODE)) {
21323 parse_rescue_modifier_terminator(parser, flags, depth);
21324 }
21325
21326 return UP(pm_rescue_modifier_node_create(parser, value, &rescue, right));
21327 }
21328
21329 return value;
21330}
21331
21336static void
21337parse_assignment_value_local(pm_parser_t *parser, const pm_node_t *node) {
21338 switch (PM_NODE_TYPE(node)) {
21339 case PM_BEGIN_NODE: {
21340 const pm_begin_node_t *cast = (const pm_begin_node_t *) node;
21341 if (cast->statements != NULL) parse_assignment_value_local(parser, (const pm_node_t *) cast->statements);
21342 break;
21343 }
21344 case PM_LOCAL_VARIABLE_WRITE_NODE: {
21345 const pm_local_variable_write_node_t *cast = (const pm_local_variable_write_node_t *) node;
21346 PRISM_ASSERT(pm_parser_local_depth_constant_id(parser, cast->name) == (int) cast->depth);
21347 pm_locals_read(parser, cast->name);
21348 break;
21349 }
21350 case PM_PARENTHESES_NODE: {
21351 const pm_parentheses_node_t *cast = (const pm_parentheses_node_t *) node;
21352 if (cast->body != NULL) parse_assignment_value_local(parser, cast->body);
21353 break;
21354 }
21355 case PM_STATEMENTS_NODE: {
21356 const pm_statements_node_t *cast = (const pm_statements_node_t *) node;
21357 const pm_node_t *statement;
21358
21359 PM_NODE_LIST_FOREACH(&cast->body, index, statement) {
21360 parse_assignment_value_local(parser, statement);
21361 }
21362 break;
21363 }
21364 default:
21365 break;
21366 }
21367}
21368
21381static pm_node_t *
21382parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding_power, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
21383 bool statement_level = (previous_binding_power == PM_BINDING_POWER_STATEMENT) || (flags & PM_PARSE_ACCEPTS_STATEMENT);
21384
21385 bool permitted = true;
21386 if (!statement_level && match1(parser, PM_TOKEN_USTAR)) permitted = false;
21387
21388 // A command call (e.g. `x = y z`) is permitted as the value when assigning
21389 // directly (carrying the caller's flag), or in any statement-level context
21390 // — which includes a rescue modifier value via the flag.
21391 uint8_t command_call_flag = (previous_binding_power == PM_BINDING_POWER_ASSIGNMENT)
21392 ? (uint8_t) (flags & PM_PARSE_ACCEPTS_COMMAND_CALL)
21393 : ((previous_binding_power < PM_BINDING_POWER_MODIFIER || statement_level) ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0);
21394
21395 /* Where a single-target write accepts a command call it also accepts an
21396 * endless method with a command body (`x = def f = foo 1`), but a
21397 * multi-write accepts only the command call. */
21398 if (command_call_flag && binding_power != PM_BINDING_POWER_MULTI_ASSIGNMENT + 1) {
21399 command_call_flag |= PM_PARSE_ACCEPTS_COMMAND_DEF;
21400 }
21401
21402 pm_node_t *value = parse_starred_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | command_call_flag), diag_id, (uint16_t) (depth + 1));
21403 if (!permitted) pm_parser_err_node(parser, value, PM_ERR_UNEXPECTED_MULTI_WRITE);
21404
21405 parse_assignment_value_local(parser, value);
21406 bool single_value = true;
21407
21408 parse_write_value_comma_check(parser, value);
21409
21410 // Block calls (command call + do block, e.g., `foo bar do end`) cannot
21411 // be followed by a comma to form a multi-value RHS because each element
21412 // of a multi-value assignment must be an `arg`, not a `block_call`.
21413 if (statement_level && !pm_block_call_p(value) && (PM_NODE_TYPE_P(value, PM_SPLAT_NODE) || match1(parser, PM_TOKEN_COMMA))) {
21414 single_value = false;
21415
21416 pm_array_elements_t elements = { 0 };
21417 pm_array_elements_append(parser, &elements, value);
21418
21419 while (accept1(parser, PM_TOKEN_COMMA)) {
21420 pm_node_t *element = parse_starred_expression(parser, binding_power, false, PM_ERR_ARRAY_ELEMENT, (uint16_t) (depth + 1));
21421
21422 pm_array_elements_append(parser, &elements, element);
21423 if (PM_NODE_TYPE_P(element, PM_ERROR_RECOVERY_NODE)) break;
21424
21425 parse_assignment_value_local(parser, element);
21426 }
21427
21428 value = UP(pm_array_node_create(parser, NULL, &elements, NULL));
21429 }
21430
21431 // Assignments whose value is a command call (e.g., a = b c) can only
21432 // be followed by modifiers (if/unless/while/until/rescue) and not by
21433 // operators with higher binding power. If we find one, emit an error
21434 // and skip the operator and its right-hand side.
21435 if (single_value && pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_MODIFIER && (pm_command_call_value_p(parser, value) || pm_block_call_p(value))) {
21436 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(parser->current.type));
21437 parser_lex(parser);
21438 parse_expression(parser, pm_binding_powers[parser->previous.type].right, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
21439 }
21440
21441 // Contradicting binding powers, the right-hand-side value of the assignment
21442 // allows the `rescue` modifier.
21443 bool multiple_assignment = (binding_power == (PM_BINDING_POWER_MULTI_ASSIGNMENT + 1));
21444 if ((single_value || multiple_assignment) && match1(parser, PM_TOKEN_KEYWORD_RESCUE_MODIFIER)) {
21445 bool command_value = pm_command_call_value_p(parser, value) || pm_block_call_p(value);
21446
21447 // A multiple assignment whose value is a command call (`x, y = foo
21448 // bar`) is a complete statement (parse.y: `mlhs '='
21449 // command_call_value`, which has no rescue), so a trailing `rescue`
21450 // modifies the whole assignment rather than the value. Leave it for the
21451 // statement-level rescue instead of binding it to the value here. For a
21452 // non-command value the rescue does bind to the value (parse.y:
21453 // `mlhs '=' mrhs_arg modifier_rescue stmt`).
21454 if (multiple_assignment && command_value) return value;
21455
21456 context_push(parser, PM_CONTEXT_RESCUE_MODIFIER);
21457
21458 pm_token_t rescue = parser->current;
21459 parser_lex(parser);
21460
21461 // The resbody is a `stmt` (parse.y: `command_rhs`/`mlhs '=' mrhs_arg`),
21462 // which permits a multiple assignment and a command call, when this is a
21463 // multiple assignment or the rescued value is itself a command call.
21464 // Otherwise it is a plain `arg` (parse.y: `arg_rhs`).
21465 bool statement_value = multiple_assignment || command_value;
21466 uint8_t rescue_flags = (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | (statement_value ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0));
21467
21468 pm_node_t *right = parse_rescue_modifier_value(parser, rescue_flags, statement_value, (uint16_t) (depth + 1));
21469 context_pop(parser);
21470
21471 // A pattern-match resbody is a statement, but here the rescue is nested
21472 // in an assignment value where parse_expression_terminator cannot see
21473 // it, so reject a trailing operator above the modifier level directly.
21474 if (PM_NODE_TYPE_P(right, PM_MATCH_REQUIRED_NODE) || PM_NODE_TYPE_P(right, PM_MATCH_PREDICATE_NODE)) {
21475 parse_rescue_modifier_terminator(parser, flags, depth);
21476 }
21477
21478 return UP(pm_rescue_modifier_node_create(parser, value, &rescue, right));
21479 }
21480
21481 return value;
21482}
21483
21496static pm_node_t *
21497parse_rescue_modifier_value(pm_parser_t *parser, uint8_t flags, bool statement, uint16_t depth) {
21498 if (statement) {
21499 pm_node_t *value;
21500 bool multiple;
21501
21502 if (match1(parser, PM_TOKEN_USTAR)) {
21503 // A leading splat can only begin a multiple assignment target list.
21504 parser_lex(parser);
21505 value = parse_splat(parser, flags, depth);
21506 multiple = true;
21507 } else {
21508 // The flag lets a single-target assignment take a multiple-value or
21509 // splat right-hand side (`b = c, d` / `b = *c`); a comma _before_ an
21510 // `=`, or a parenthesized target list (`(b, c), d = 1`), instead
21511 // promotes to a multiple assignment target list below.
21512 value = parse_expression(parser, pm_binding_powers[PM_TOKEN_KEYWORD_RESCUE_MODIFIER].right, flags | PM_PARSE_ACCEPTS_STATEMENT | PM_PARSE_ACCEPTS_COMMAND_DEF, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
21513 multiple = match1(parser, PM_TOKEN_COMMA) || PM_NODE_TYPE_P(value, PM_MULTI_TARGET_NODE);
21514 }
21515
21516 if (multiple) {
21517 pm_node_t *target = parse_targets_validate(parser, value, PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1));
21518
21519 // A promoted target list is only a valid rescue value as part of a
21520 // complete `targets = values`. parse_targets_validate already
21521 // reports a missing `=` for every terminator except `)` (which it
21522 // permits for an enclosing mlhs paren that does not apply here), so
21523 // reject that case.
21524 if (!match1(parser, PM_TOKEN_EQUAL)) {
21525 if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) pm_parser_err_node(parser, target, PM_ERR_WRITE_TARGET_UNEXPECTED);
21526 return target;
21527 }
21528
21529 pm_token_t operator = parser->current;
21530 parser_lex(parser);
21531
21532 pm_node_t *values = parse_assignment_values(parser, PM_BINDING_POWER_STATEMENT, PM_BINDING_POWER_MULTI_ASSIGNMENT + 1, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL, (uint16_t) (depth + 1));
21533 value = parse_write(parser, target, &operator, values);
21534 }
21535
21536 // Reject a trailing operator that cannot follow a statement resbody.
21537 // Pattern-match handlers are statements too, but for the bare statement
21538 // form they are reported by parse_expression_terminator instead (which
21539 // keeps its existing error-recovery), so they are excluded here.
21540 if (!PM_NODE_TYPE_P(value, PM_MATCH_REQUIRED_NODE) && !PM_NODE_TYPE_P(value, PM_MATCH_PREDICATE_NODE)) {
21541 parse_rescue_modifier_terminator(parser, flags, depth);
21542 }
21543
21544 return value;
21545 }
21546
21547 // Otherwise the resbody is a plain `arg` (parse.y: `arg modifier_rescue
21548 // arg`), parsed above the `and`/`or`/`not` level so those stay outside it.
21549 return parse_expression(parser, PM_BINDING_POWER_DEFINED, flags, PM_ERR_RESCUE_MODIFIER_VALUE, (uint16_t) (depth + 1));
21550}
21551
21559static void
21560parse_call_operator_write(pm_parser_t *parser, pm_call_node_t *call_node, const pm_token_t *operator) {
21561 if (call_node->arguments != NULL) {
21562 pm_parser_err_token(parser, operator, PM_ERR_OPERATOR_WRITE_ARGUMENTS);
21563 pm_node_unreference(parser, UP(call_node->arguments));
21564 call_node->arguments = NULL;
21565 }
21566
21567 if (call_node->block != NULL) {
21568 pm_parser_err_token(parser, operator, PM_ERR_OPERATOR_WRITE_BLOCK);
21569 pm_node_unreference(parser, UP(call_node->block));
21570 call_node->block = NULL;
21571 }
21572}
21573
21577static pm_node_t *
21578parse_match_write(pm_parser_t *parser, pm_call_node_t *call, const pm_regexp_match_t *match) {
21579 pm_node_list_t targets = { 0 };
21580 pm_constant_id_set_t names = { 0 };
21581
21582 for (size_t index = 0; index < match->size; index++) {
21583 const pm_regexp_capture_t *capture = &match->captures[index];
21584 const uint8_t *name = capture->name;
21585 size_t length = capture->length;
21586
21587 if (!pm_slice_is_valid_local(parser, name, name + length)) continue;
21588
21589 const uint8_t *start = parser->start + capture->location.start;
21590 const uint8_t *end = start + capture->location.length;
21592
21593 if (length == capture->location.length && (name == start || memcmp(name, start, length) == 0)) {
21594 id = pm_parser_constant_id_raw(parser, start, end);
21595 } else {
21596 uint8_t *copy = PM_ARENA_ALLOC_ARRAY(parser->arena, uint8_t, length);
21597 memcpy(copy, name, length);
21598 id = pm_parser_constant_id_owned(parser, copy, length);
21599 }
21600
21601 if (id == 0 || !pm_constant_id_set_insert(&parser->scratch_arena, &names, id)) continue;
21602
21603 int depth = pm_parser_local_depth_constant_id(parser, id);
21604 if (depth == -1) {
21605 if (pm_local_is_keyword((const char *) name, length)) continue;
21606 pm_parser_local_add(parser, id, start, end, 0);
21607 }
21608
21609 pm_node_t *target = UP(pm_local_variable_target_node_create(parser, &capture->location, id, depth == -1 ? 0 : (uint32_t) depth));
21610 pm_node_list_open_append(&parser->node_chunks, &targets, target);
21611 }
21612
21613 return targets.size == 0 ? UP(call) : UP(pm_match_write_node_create(parser, call, &targets));
21614}
21615
21616static pm_if_node_t *
21617parse_ternary(pm_parser_t *parser, pm_node_t *node, uint8_t flags, uint16_t depth) {
21618 context_push(parser, PM_CONTEXT_TERNARY);
21619 pm_scratch_node_list_t current_block_exits = { 0 };
21620 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
21621
21622 pm_token_t qmark = parser->current;
21623 parser_lex(parser);
21624
21625 pm_node_t *true_expression = parse_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_TERNARY_EXPRESSION_TRUE, (uint16_t) (depth + 1));
21626
21627 if (parser->recovering) {
21628 // If parsing the true expression of this ternary resulted in a syntax
21629 // error that we can recover from, then we're going to put missing nodes
21630 // and tokens into the remaining places. We want to be sure to do this
21631 // before the `expect` function call to make sure it doesn't
21632 // accidentally move past a ':' token that occurs after the syntax
21633 // error.
21634 pm_token_t colon = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
21635 pm_node_t *false_expression = UP(pm_error_recovery_node_create(parser, PM_TOKEN_START(parser, &colon), PM_TOKEN_LENGTH(&colon)));
21636
21637 context_pop(parser);
21638 pop_block_exits(parser, previous_block_exits);
21639 return pm_if_node_ternary_create(parser, node, &qmark, true_expression, &colon, false_expression);
21640 }
21641
21642 accept1(parser, PM_TOKEN_NEWLINE);
21643 expect1(parser, PM_TOKEN_COLON, PM_ERR_TERNARY_COLON);
21644
21645 pm_token_t colon = parser->previous;
21646 pm_node_t *false_expression = parse_expression(parser, PM_BINDING_POWER_DEFINED, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_TERNARY_EXPRESSION_FALSE, (uint16_t) (depth + 1));
21647
21648 context_pop(parser);
21649 pop_block_exits(parser, previous_block_exits);
21650 return pm_if_node_ternary_create(parser, node, &qmark, true_expression, &colon, false_expression);
21651}
21652
21653static pm_node_t *
21654parse_constant_path_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, uint8_t flags, uint16_t depth) {
21655 pm_token_t delimiter = parser->current;
21656 parser_lex(parser);
21657
21658 switch (parser->current.type) {
21659 case PM_TOKEN_CONSTANT: {
21660 parser_lex(parser);
21661 pm_node_t *path;
21662
21663 if (
21664 (parser->current.type == PM_TOKEN_PARENTHESIS_LEFT) ||
21665 ((flags & PM_PARSE_ACCEPTS_COMMAND_CALL) && token_begins_argument_p(parser->current.type))
21666 ) {
21667 /* If we have a constant immediately following a '::' operator,
21668 * then this can either be a constant path or a method call,
21669 * depending on what follows the constant.
21670 *
21671 * If we have parentheses, then this is a method call. That
21672 * would look like Foo::Bar(). */
21673 pm_token_t message = parser->previous;
21674 pm_arguments_t arguments = { 0 };
21675
21676 parse_arguments_list(parser, &arguments, true, flags, (uint16_t) (depth + 1));
21677 path = UP(pm_call_node_call_create(parser, node, &delimiter, &message, &arguments));
21678 } else {
21679 /* Otherwise, this is a constant path. That would look like
21680 * Foo::Bar. */
21681 path = UP(pm_constant_path_node_create(parser, node, &delimiter, &parser->previous));
21682 }
21683
21684 return parse_multi_target(parser, path, previous_binding_power, depth);
21685 }
21686 case PM_CASE_OPERATOR:
21687 case PM_CASE_KEYWORD:
21688 case PM_TOKEN_IDENTIFIER:
21689 case PM_TOKEN_METHOD_NAME: {
21690 parser_lex(parser);
21691 pm_token_t message = parser->previous;
21692
21693 /* If we have an identifier following a '::' operator, then it is
21694 * for sure a method call. */
21695 pm_arguments_t arguments = { 0 };
21696 parse_arguments_list(parser, &arguments, true, flags, (uint16_t) (depth + 1));
21697
21698 pm_call_node_t *call = pm_call_node_call_create(parser, node, &delimiter, &message, &arguments);
21699 return parse_multi_target(parser, UP(call), previous_binding_power, depth);
21700 }
21701 case PM_TOKEN_PARENTHESIS_LEFT: {
21702 /* If we have a parenthesis following a '::' operator, then it is the
21703 * method call shorthand. That would look like Foo::(bar). */
21704 pm_arguments_t arguments = { 0 };
21705 parse_arguments_list(parser, &arguments, true, false, (uint16_t) (depth + 1));
21706
21707 return UP(pm_call_node_shorthand_create(parser, node, &delimiter, &arguments));
21708 }
21709 default:
21710 expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT);
21711 return UP(pm_constant_path_node_create(parser, node, &delimiter, &parser->previous));
21712 }
21713}
21714
21715static pm_node_t *
21716parse_match(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
21717 /* Note that we _must_ parse the value before adding the local variables in
21718 * order to properly mirror the behavior of Ruby. For example,
21719 *
21720 * /(?<foo>bar)/ =~ foo
21721 *
21722 * In this case, `foo` should be a method call and not a local yet.
21723 */
21724 /* The right operand can contain another =~ with captures of its own. */
21725 const pm_regexp_match_t *match = parser->regexp_match;
21726 parser->regexp_match = NULL;
21727
21728 pm_token_t operator = parser->current;
21729 parser_lex(parser);
21730
21731 pm_node_t *argument = parse_expression(parser, binding_power, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
21732 pm_call_node_t *call = pm_call_node_binary_create(parser, node, &operator, argument, 0);
21733
21734 return match != NULL && match->receiver == node ? parse_match_write(parser, call, match) : UP(call);
21735}
21736
21737static pm_node_t *
21738parse_equal(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
21739 pm_token_t operator = parser->current;
21740
21741 switch (PM_NODE_TYPE(node)) {
21742 case PM_CALL_NODE: {
21743 // If we have no arguments to the call node and we need this
21744 // to be a target then this is either a method call or a
21745 // local variable write. This _must_ happen before the value
21746 // is parsed because it could be referenced in the value.
21747 pm_call_node_t *call_node = (pm_call_node_t *) node;
21748 if (PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) {
21749 pm_parser_local_add_location(parser, &call_node->message_loc, 0);
21750 }
21751 }
21753 case PM_CASE_WRITABLE: {
21754 // When we have `it = value`, we need to add `it` as a local
21755 // variable before parsing the value, in case the value
21756 // references the variable.
21757 if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) {
21758 pm_parser_local_add_location(parser, &node->location, 0);
21759 }
21760
21761 parser_lex(parser);
21762 pm_node_t *value = parse_assignment_values(parser, previous_binding_power, PM_NODE_TYPE_P(node, PM_MULTI_TARGET_NODE) ? PM_BINDING_POWER_MULTI_ASSIGNMENT + 1 : binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL, (uint16_t) (depth + 1));
21763
21764 if (PM_NODE_TYPE_P(node, PM_MULTI_TARGET_NODE) && previous_binding_power != PM_BINDING_POWER_STATEMENT && !(flags & PM_PARSE_ACCEPTS_STATEMENT)) {
21765 pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_MULTI_WRITE);
21766 }
21767
21768 return parse_write(parser, node, &operator, value);
21769 }
21770 case PM_SPLAT_NODE: {
21771 pm_multi_targets_t targets = { 0 };
21772 pm_multi_targets_append(parser, &targets, node);
21773 pm_multi_target_node_t *multi_target = pm_multi_target_node_create(parser, &targets, NULL, NULL);
21774
21775 parser_lex(parser);
21776 pm_node_t *value = parse_assignment_values(parser, previous_binding_power, PM_BINDING_POWER_MULTI_ASSIGNMENT + 1, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL, (uint16_t) (depth + 1));
21777 return parse_write(parser, UP(multi_target), &operator, value);
21778 }
21779 case PM_SOURCE_ENCODING_NODE:
21780 case PM_FALSE_NODE:
21781 case PM_SOURCE_FILE_NODE:
21782 case PM_SOURCE_LINE_NODE:
21783 case PM_NIL_NODE:
21784 case PM_SELF_NODE:
21785 case PM_TRUE_NODE: {
21786 // In these special cases, we have specific error messages
21787 // and we will replace them with local variable writes.
21788 parser_lex(parser);
21789 pm_node_t *value = parse_assignment_values(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL, (uint16_t) (depth + 1));
21790 return parse_unwriteable_write(parser, node, &operator, value);
21791 }
21792 default:
21793 // In this case we have an = sign, but we don't know what
21794 // it's for. We need to treat it as an error. We'll mark it
21795 // as an error and skip past it.
21796 parser_lex(parser);
21797 pm_parser_err_token(parser, &operator, PM_ERR_EXPRESSION_NOT_WRITABLE);
21798 return node;
21799 }
21800}
21801
21802static pm_node_t *
21803parse_and_equal(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
21804 pm_token_t operator = parser->current;
21805
21806 switch (PM_NODE_TYPE(node)) {
21807 case PM_BACK_REFERENCE_READ_NODE:
21808 case PM_NUMBERED_REFERENCE_READ_NODE:
21809 PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser, node, PM_ERR_WRITE_TARGET_READONLY);
21811 case PM_GLOBAL_VARIABLE_READ_NODE: {
21812 parser_lex(parser);
21813
21814 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21815 pm_node_t *result = UP(pm_global_variable_and_write_node_create(parser, node, &operator, value));
21816
21817 return result;
21818 }
21819 case PM_CLASS_VARIABLE_READ_NODE: {
21820 parser_lex(parser);
21821
21822 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21823 pm_node_t *result = UP(pm_class_variable_and_write_node_create(parser, (pm_class_variable_read_node_t *) node, &operator, value));
21824
21825 return result;
21826 }
21827 case PM_CONSTANT_PATH_NODE: {
21828 parser_lex(parser);
21829
21830 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21831 pm_node_t *write = UP(pm_constant_path_and_write_node_create(parser, (pm_constant_path_node_t *) node, &operator, value));
21832
21833 return parse_shareable_constant_write(parser, write);
21834 }
21835 case PM_CONSTANT_READ_NODE: {
21836 parser_lex(parser);
21837
21838 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21839 pm_node_t *write = UP(pm_constant_and_write_node_create(parser, (pm_constant_read_node_t *) node, &operator, value));
21840
21841 if (context_def_p(parser)) {
21842 pm_parser_err_node(parser, write, PM_ERR_WRITE_TARGET_IN_METHOD);
21843 }
21844
21845 return parse_shareable_constant_write(parser, write);
21846 }
21847 case PM_INSTANCE_VARIABLE_READ_NODE: {
21848 parser_lex(parser);
21849
21850 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21851 pm_node_t *result = UP(pm_instance_variable_and_write_node_create(parser, (pm_instance_variable_read_node_t *) node, &operator, value));
21852
21853 return result;
21854 }
21855 case PM_IT_LOCAL_VARIABLE_READ_NODE: {
21856 pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2);
21857 parser_lex(parser);
21858
21859 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21860 pm_node_t *result = UP(pm_local_variable_and_write_node_create(parser, node, &operator, value, name, 0));
21861
21862 pm_node_unreference(parser, node);
21863 return result;
21864 }
21865 case PM_LOCAL_VARIABLE_READ_NODE: {
21866 if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) {
21867 PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + node->location.start);
21868 pm_node_unreference(parser, node);
21869 }
21870
21871 pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node;
21872 parser_lex(parser);
21873
21874 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21875 pm_node_t *result = UP(pm_local_variable_and_write_node_create(parser, node, &operator, value, cast->name, cast->depth));
21876
21877 return result;
21878 }
21879 case PM_CALL_NODE: {
21880 pm_call_node_t *cast = (pm_call_node_t *) node;
21881
21882 // If we have a vcall (a method with no arguments and no
21883 // receiver that could have been a local variable) then we
21884 // will transform it into a local variable write.
21885 if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) {
21886 pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length);
21887 pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1);
21888 parser_lex(parser);
21889
21890 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21891 pm_node_t *result = UP(pm_local_variable_and_write_node_create(parser, UP(cast), &operator, value, constant_id, 0));
21892
21893 return result;
21894 }
21895
21896 // Move past the token here so that we have already added
21897 // the local variable by this point.
21898 parser_lex(parser);
21899
21900 // If there is no call operator and the message is "[]" then
21901 // this is an aref expression, and we can transform it into
21902 // an aset expression.
21903 if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_INDEX)) {
21904 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21905 return UP(pm_index_and_write_node_create(parser, cast, &operator, value));
21906 }
21907
21908 // If this node cannot be writable, then we have an error.
21909 if (pm_call_node_writable_p(parser, cast)) {
21910 parse_write_name(parser, &cast->name);
21911 } else {
21912 pm_parser_err_node(parser, node, PM_ERR_WRITE_TARGET_UNEXPECTED);
21913 }
21914
21915 parse_call_operator_write(parser, cast, &operator);
21916 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1));
21917 return UP(pm_call_and_write_node_create(parser, cast, &operator, value));
21918 }
21919 case PM_MULTI_WRITE_NODE: {
21920 parser_lex(parser);
21921
21922 pm_parser_err_token(parser, &operator, PM_ERR_AMPAMPEQ_MULTI_ASSIGN);
21923 return node;
21924 }
21925 default:
21926 parser_lex(parser);
21927
21928 // In this case we have an &&= sign, but we don't know what it's for.
21929 // We need to treat it as an error. For now, we'll mark it as an error
21930 // and just skip right past it.
21931 pm_parser_err_token(parser, &operator, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ);
21932 return node;
21933 }
21934}
21935
21936static pm_node_t *
21937parse_or_equal(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
21938 pm_token_t operator = parser->current;
21939
21940 switch (PM_NODE_TYPE(node)) {
21941 case PM_BACK_REFERENCE_READ_NODE:
21942 case PM_NUMBERED_REFERENCE_READ_NODE:
21943 PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser, node, PM_ERR_WRITE_TARGET_READONLY);
21945 case PM_GLOBAL_VARIABLE_READ_NODE: {
21946 parser_lex(parser);
21947
21948 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
21949 pm_node_t *result = UP(pm_global_variable_or_write_node_create(parser, node, &operator, value));
21950
21951 return result;
21952 }
21953 case PM_CLASS_VARIABLE_READ_NODE: {
21954 parser_lex(parser);
21955
21956 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
21957 pm_node_t *result = UP(pm_class_variable_or_write_node_create(parser, (pm_class_variable_read_node_t *) node, &operator, value));
21958
21959 return result;
21960 }
21961 case PM_CONSTANT_PATH_NODE: {
21962 parser_lex(parser);
21963
21964 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
21965 pm_node_t *write = UP(pm_constant_path_or_write_node_create(parser, (pm_constant_path_node_t *) node, &operator, value));
21966
21967 return parse_shareable_constant_write(parser, write);
21968 }
21969 case PM_CONSTANT_READ_NODE: {
21970 parser_lex(parser);
21971
21972 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
21973 pm_node_t *write = UP(pm_constant_or_write_node_create(parser, (pm_constant_read_node_t *) node, &operator, value));
21974
21975 if (context_def_p(parser)) {
21976 pm_parser_err_node(parser, write, PM_ERR_WRITE_TARGET_IN_METHOD);
21977 }
21978
21979 return parse_shareable_constant_write(parser, write);
21980 }
21981 case PM_INSTANCE_VARIABLE_READ_NODE: {
21982 parser_lex(parser);
21983
21984 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
21985 pm_node_t *result = UP(pm_instance_variable_or_write_node_create(parser, (pm_instance_variable_read_node_t *) node, &operator, value));
21986
21987 return result;
21988 }
21989 case PM_IT_LOCAL_VARIABLE_READ_NODE: {
21990 pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2);
21991 parser_lex(parser);
21992
21993 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
21994 pm_node_t *result = UP(pm_local_variable_or_write_node_create(parser, node, &operator, value, name, 0));
21995
21996 pm_node_unreference(parser, node);
21997 return result;
21998 }
21999 case PM_LOCAL_VARIABLE_READ_NODE: {
22000 if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) {
22001 PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(node));
22002 pm_node_unreference(parser, node);
22003 }
22004
22005 pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node;
22006 parser_lex(parser);
22007
22008 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
22009 pm_node_t *result = UP(pm_local_variable_or_write_node_create(parser, node, &operator, value, cast->name, cast->depth));
22010
22011 return result;
22012 }
22013 case PM_CALL_NODE: {
22014 pm_call_node_t *cast = (pm_call_node_t *) node;
22015
22016 // If we have a vcall (a method with no arguments and no
22017 // receiver that could have been a local variable) then we
22018 // will transform it into a local variable write.
22019 if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) {
22020 pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length);
22021 pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1);
22022 parser_lex(parser);
22023
22024 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
22025 pm_node_t *result = UP(pm_local_variable_or_write_node_create(parser, UP(cast), &operator, value, constant_id, 0));
22026
22027 return result;
22028 }
22029
22030 // Move past the token here so that we have already added
22031 // the local variable by this point.
22032 parser_lex(parser);
22033
22034 // If there is no call operator and the message is "[]" then
22035 // this is an aref expression, and we can transform it into
22036 // an aset expression.
22037 if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_INDEX)) {
22038 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
22039 return UP(pm_index_or_write_node_create(parser, cast, &operator, value));
22040 }
22041
22042 // If this node cannot be writable, then we have an error.
22043 if (pm_call_node_writable_p(parser, cast)) {
22044 parse_write_name(parser, &cast->name);
22045 } else {
22046 pm_parser_err_node(parser, node, PM_ERR_WRITE_TARGET_UNEXPECTED);
22047 }
22048
22049 parse_call_operator_write(parser, cast, &operator);
22050 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1));
22051 return UP(pm_call_or_write_node_create(parser, cast, &operator, value));
22052 }
22053 case PM_MULTI_WRITE_NODE: {
22054 parser_lex(parser);
22055 pm_parser_err_token(parser, &operator, PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN);
22056 return node;
22057 }
22058 default:
22059 parser_lex(parser);
22060
22061 // In this case we have an ||= sign, but we don't know what it's for.
22062 // We need to treat it as an error. For now, we'll mark it as an error
22063 // and just skip right past it.
22064 pm_parser_err_token(parser, &operator, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ);
22065 return node;
22066 }
22067}
22068
22069static pm_node_t *
22070parse_operator_equal(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
22071 pm_token_t operator = parser->current;
22072
22073 switch (PM_NODE_TYPE(node)) {
22074 case PM_BACK_REFERENCE_READ_NODE:
22075 case PM_NUMBERED_REFERENCE_READ_NODE:
22076 PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser, node, PM_ERR_WRITE_TARGET_READONLY);
22078 case PM_GLOBAL_VARIABLE_READ_NODE: {
22079 parser_lex(parser);
22080
22081 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22082 pm_node_t *result = UP(pm_global_variable_operator_write_node_create(parser, node, &operator, value));
22083
22084 return result;
22085 }
22086 case PM_CLASS_VARIABLE_READ_NODE: {
22087 parser_lex(parser);
22088
22089 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22090 pm_node_t *result = UP(pm_class_variable_operator_write_node_create(parser, (pm_class_variable_read_node_t *) node, &operator, value));
22091
22092 return result;
22093 }
22094 case PM_CONSTANT_PATH_NODE: {
22095 parser_lex(parser);
22096
22097 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22098 pm_node_t *write = UP(pm_constant_path_operator_write_node_create(parser, (pm_constant_path_node_t *) node, &operator, value));
22099
22100 return parse_shareable_constant_write(parser, write);
22101 }
22102 case PM_CONSTANT_READ_NODE: {
22103 parser_lex(parser);
22104
22105 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22106 pm_node_t *write = UP(pm_constant_operator_write_node_create(parser, (pm_constant_read_node_t *) node, &operator, value));
22107
22108 if (context_def_p(parser)) {
22109 pm_parser_err_node(parser, write, PM_ERR_WRITE_TARGET_IN_METHOD);
22110 }
22111
22112 return parse_shareable_constant_write(parser, write);
22113 }
22114 case PM_INSTANCE_VARIABLE_READ_NODE: {
22115 parser_lex(parser);
22116
22117 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22118 pm_node_t *result = UP(pm_instance_variable_operator_write_node_create(parser, (pm_instance_variable_read_node_t *) node, &operator, value));
22119
22120 return result;
22121 }
22122 case PM_IT_LOCAL_VARIABLE_READ_NODE: {
22123 pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2);
22124 parser_lex(parser);
22125
22126 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22127 pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, node, &operator, value, name, 0));
22128
22129 pm_node_unreference(parser, node);
22130 return result;
22131 }
22132 case PM_LOCAL_VARIABLE_READ_NODE: {
22133 if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) {
22134 PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(node));
22135 pm_node_unreference(parser, node);
22136 }
22137
22138 pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node;
22139 parser_lex(parser);
22140
22141 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22142 pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, node, &operator, value, cast->name, cast->depth));
22143
22144 return result;
22145 }
22146 case PM_CALL_NODE: {
22147 pm_call_node_t *cast = (pm_call_node_t *) node;
22148
22149 /* Register a new local before lexing the first RHS token. */
22150 if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) {
22151 pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length);
22152 pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1);
22153 parser_lex(parser);
22154
22155 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22156 pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, UP(cast), &operator, value, constant_id, 0));
22157
22158 return result;
22159 }
22160
22161 parser_lex(parser);
22162
22163 // If there is no call operator and the message is "[]" then
22164 // this is an aref expression, and we can transform it into
22165 // an aset expression.
22166 if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_INDEX)) {
22167 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22168 return UP(pm_index_operator_write_node_create(parser, cast, &operator, value));
22169 }
22170
22171 // If this node cannot be writable, then we have an error.
22172 if (pm_call_node_writable_p(parser, cast)) {
22173 parse_write_name(parser, &cast->name);
22174 } else {
22175 pm_parser_err_node(parser, node, PM_ERR_WRITE_TARGET_UNEXPECTED);
22176 }
22177
22178 parse_call_operator_write(parser, cast, &operator);
22179 pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, flags, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22180 return UP(pm_call_operator_write_node_create(parser, cast, &operator, value));
22181 }
22182 case PM_MULTI_WRITE_NODE: {
22183 parser_lex(parser);
22184 pm_parser_err_token(parser, &operator, PM_ERR_OPERATOR_MULTI_ASSIGN);
22185 return node;
22186 }
22187 default:
22188 parser_lex(parser);
22189
22190 // In this case we have an operator but we don't know what it's for.
22191 // We need to treat it as an error. For now, we'll mark it as an error
22192 // and just skip right past it.
22193 PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, pm_token_str(parser->current.type));
22194 return node;
22195 }
22196}
22197
22198static pm_node_t *
22199parse_call(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, uint8_t flags, uint16_t depth) {
22200 pm_token_t operator = parser->current;
22201 parser_lex(parser);
22202
22203 pm_arguments_t arguments = { 0 };
22204
22205 // This if statement handles the foo.() syntax.
22206 if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT)) {
22207 parse_arguments_list(parser, &arguments, true, false, (uint16_t) (depth + 1));
22208 return UP(pm_call_node_shorthand_create(parser, node, &operator, &arguments));
22209 }
22210
22211 switch (PM_NODE_TYPE(node)) {
22212 case PM_RESCUE_MODIFIER_NODE: {
22213 pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node;
22214 if (PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_REQUIRED_NODE)) {
22215 PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(operator.type));
22216 }
22217 break;
22218 }
22219 case PM_AND_NODE: {
22220 pm_and_node_t *cast = (pm_and_node_t *) node;
22221 if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) {
22222 PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(operator.type));
22223 }
22224 break;
22225 }
22226 case PM_OR_NODE: {
22227 pm_or_node_t *cast = (pm_or_node_t *) node;
22228 if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) {
22229 PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(operator.type));
22230 }
22231 break;
22232 }
22233 default:
22234 break;
22235 }
22236
22237 pm_token_t message;
22238
22239 switch (parser->current.type) {
22240 case PM_CASE_OPERATOR:
22241 case PM_CASE_KEYWORD:
22242 case PM_TOKEN_CONSTANT:
22243 case PM_TOKEN_IDENTIFIER:
22244 case PM_TOKEN_METHOD_NAME: {
22245 parser_lex(parser);
22246 message = parser->previous;
22247 break;
22248 }
22249 default: {
22250 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_MESSAGE, pm_token_str(parser->current.type));
22251 message = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end };
22252 }
22253 }
22254
22255 parse_arguments_list(parser, &arguments, true, flags, (uint16_t) (depth + 1));
22256 pm_call_node_t *call = pm_call_node_call_create(parser, node, &operator, &message, &arguments);
22257
22258 if (
22259 (previous_binding_power == PM_BINDING_POWER_STATEMENT) &&
22260 arguments.list.size == 0 &&
22261 arguments.opening_loc.length == 0 &&
22262 match1(parser, PM_TOKEN_COMMA)
22263 ) {
22264 return parse_targets_validate(parser, UP(call), PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1));
22265 } else {
22266 return UP(call);
22267 }
22268}
22269
22270static pm_node_t *
22271parse_aref(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, uint8_t flags, uint16_t depth) {
22272 parser_lex(parser);
22273
22274 pm_arguments_t arguments = { 0 };
22275 arguments.opening_loc = TOK2LOC(parser, &parser->previous);
22276
22277 if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) {
22278 parse_arguments(parser, &arguments, false, PM_TOKEN_BRACKET_RIGHT, parse_delimited_arguments_flags(parser, flags, true), (uint16_t) (depth + 1));
22279 expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_EXPECT_RBRACKET);
22280 }
22281
22282 arguments.closing_loc = TOK2LOC(parser, &parser->previous);
22283
22284 // If we have a comma after the closing bracket then this is a multiple
22285 // assignment and we should parse the targets.
22286 if (previous_binding_power == PM_BINDING_POWER_STATEMENT && match1(parser, PM_TOKEN_COMMA)) {
22287 pm_call_node_t *aref = pm_call_node_aref_create(parser, node, &arguments);
22288 return parse_targets_validate(parser, UP(aref), PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1));
22289 }
22290
22291 // If we're at the end of the arguments, we can now check if there is a
22292 // block node that starts with a {. If there is, then we can parse it and
22293 // add it to the arguments.
22294 pm_block_node_t *block = NULL;
22295 if (accept1(parser, PM_TOKEN_BRACE_LEFT)) {
22296 block = parse_block(parser, (uint16_t) (depth + 1));
22297 pm_arguments_validate_block(parser, &arguments, block);
22298 } else if (pm_accepts_block_stack_p(parser) && accept1(parser, PM_TOKEN_KEYWORD_DO)) {
22299 block = parse_block(parser, (uint16_t) (depth + 1));
22300 }
22301
22302 if (block != NULL) {
22303 if (arguments.block != NULL) {
22304 pm_parser_err_node(parser, UP(block), PM_ERR_ARGUMENT_AFTER_BLOCK);
22305 parse_arguments_append(parser, &arguments, arguments.block);
22306 }
22307
22308 arguments.block = UP(block);
22309 }
22310
22311 return UP(pm_call_node_aref_create(parser, node, &arguments));
22312}
22313
22314static PRISM_INLINE pm_node_t *
22315parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t previous_binding_power, pm_binding_power_t binding_power, uint8_t flags, uint16_t depth) {
22316 pm_token_t token = parser->current;
22317
22318 switch (token.type) {
22319 case PM_TOKEN_EQUAL:
22320 return parse_equal(parser, node, previous_binding_power, binding_power, flags, depth);
22321 case PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL:
22322 return parse_and_equal(parser, node, previous_binding_power, binding_power, flags, depth);
22323 case PM_TOKEN_PIPE_PIPE_EQUAL:
22324 return parse_or_equal(parser, node, previous_binding_power, binding_power, flags, depth);
22325 case PM_TOKEN_AMPERSAND_EQUAL:
22326 case PM_TOKEN_CARET_EQUAL:
22327 case PM_TOKEN_GREATER_GREATER_EQUAL:
22328 case PM_TOKEN_LESS_LESS_EQUAL:
22329 case PM_TOKEN_MINUS_EQUAL:
22330 case PM_TOKEN_PERCENT_EQUAL:
22331 case PM_TOKEN_PIPE_EQUAL:
22332 case PM_TOKEN_PLUS_EQUAL:
22333 case PM_TOKEN_SLASH_EQUAL:
22334 case PM_TOKEN_STAR_EQUAL:
22335 case PM_TOKEN_STAR_STAR_EQUAL:
22336 return parse_operator_equal(parser, node, previous_binding_power, binding_power, flags, depth);
22337 case PM_TOKEN_AMPERSAND_AMPERSAND:
22338 case PM_TOKEN_KEYWORD_AND: {
22339 parser_lex(parser);
22340
22341 pm_node_t *right = parse_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | (parser->previous.type == PM_TOKEN_KEYWORD_AND ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0)), PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22342 return UP(pm_and_node_create(parser, node, &token, right));
22343 }
22344 case PM_TOKEN_KEYWORD_OR:
22345 case PM_TOKEN_PIPE_PIPE: {
22346 parser_lex(parser);
22347
22348 pm_node_t *right = parse_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | (parser->previous.type == PM_TOKEN_KEYWORD_OR ? PM_PARSE_ACCEPTS_COMMAND_CALL : 0)), PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22349 return UP(pm_or_node_create(parser, node, &token, right));
22350 }
22351 case PM_TOKEN_EQUAL_TILDE:
22352 return parse_match(parser, node, binding_power, flags, depth);
22353 case PM_TOKEN_UAMPERSAND:
22354 case PM_TOKEN_USTAR:
22355 case PM_TOKEN_USTAR_STAR:
22356 // The only times this will occur are when we are in an error state,
22357 // but we'll put them in here so that errors can propagate.
22358 case PM_TOKEN_BANG_EQUAL:
22359 case PM_TOKEN_BANG_TILDE:
22360 case PM_TOKEN_EQUAL_EQUAL:
22361 case PM_TOKEN_EQUAL_EQUAL_EQUAL:
22362 case PM_TOKEN_LESS_EQUAL_GREATER:
22363 case PM_TOKEN_CARET:
22364 case PM_TOKEN_PIPE:
22365 case PM_TOKEN_AMPERSAND:
22366 case PM_TOKEN_GREATER_GREATER:
22367 case PM_TOKEN_LESS_LESS:
22368 case PM_TOKEN_MINUS:
22369 case PM_TOKEN_PLUS:
22370 case PM_TOKEN_PERCENT:
22371 case PM_TOKEN_SLASH:
22372 case PM_TOKEN_STAR:
22373 case PM_TOKEN_STAR_STAR: {
22374 parser_lex(parser);
22375 pm_token_t operator = parser->previous;
22376 switch (PM_NODE_TYPE(node)) {
22377 case PM_RESCUE_MODIFIER_NODE: {
22378 pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node;
22379 if (PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_REQUIRED_NODE)) {
22380 PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(operator.type));
22381 }
22382 break;
22383 }
22384 case PM_AND_NODE: {
22385 pm_and_node_t *cast = (pm_and_node_t *) node;
22386 if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) {
22387 PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(operator.type));
22388 }
22389 break;
22390 }
22391 case PM_OR_NODE: {
22392 pm_or_node_t *cast = (pm_or_node_t *) node;
22393 if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) {
22394 PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_str(operator.type));
22395 }
22396 break;
22397 }
22398 default:
22399 break;
22400 }
22401
22402 pm_node_t *argument = parse_expression(parser, binding_power, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22403 return UP(pm_call_node_binary_create(parser, node, &token, argument, 0));
22404 }
22405 case PM_TOKEN_GREATER:
22406 case PM_TOKEN_GREATER_EQUAL:
22407 case PM_TOKEN_LESS:
22408 case PM_TOKEN_LESS_EQUAL: {
22409 if (PM_NODE_TYPE_P(node, PM_CALL_NODE) && PM_NODE_FLAG_P(node, PM_CALL_NODE_FLAGS_COMPARISON)) {
22410 PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_COMPARISON_AFTER_COMPARISON);
22411 }
22412
22413 parser_lex(parser);
22414 pm_node_t *argument = parse_expression(parser, binding_power, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22415 return UP(pm_call_node_binary_create(parser, node, &token, argument, PM_CALL_NODE_FLAGS_COMPARISON));
22416 }
22417 case PM_TOKEN_AMPERSAND_DOT:
22418 case PM_TOKEN_DOT:
22419 return parse_call(parser, node, previous_binding_power, flags, depth);
22420 case PM_TOKEN_DOT_DOT:
22421 case PM_TOKEN_DOT_DOT_DOT: {
22422 parser_lex(parser);
22423
22424 pm_node_t *right = NULL;
22425 if (token_begins_expression_p(parser->current.type)) {
22426 right = parse_expression(parser, binding_power, flags & PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1));
22427 }
22428
22429 return UP(pm_range_node_create(parser, node, &token, right));
22430 }
22431 case PM_TOKEN_KEYWORD_IF_MODIFIER: {
22432 pm_token_t keyword = parser->current;
22433 parser_lex(parser);
22434
22435 pm_node_t *predicate = parse_value_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_IF_PREDICATE, (uint16_t) (depth + 1));
22436 return UP(pm_if_node_modifier_create(parser, node, &keyword, predicate));
22437 }
22438 case PM_TOKEN_KEYWORD_UNLESS_MODIFIER: {
22439 pm_token_t keyword = parser->current;
22440 parser_lex(parser);
22441
22442 pm_node_t *predicate = parse_value_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_UNLESS_PREDICATE, (uint16_t) (depth + 1));
22443 return UP(pm_unless_node_modifier_create(parser, node, &keyword, predicate));
22444 }
22445 case PM_TOKEN_KEYWORD_UNTIL_MODIFIER: {
22446 parser_lex(parser);
22447 pm_statements_node_t *statements = pm_statements_node_single_create(parser, node, true);
22448
22449 pm_node_t *predicate = parse_value_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_UNTIL_PREDICATE, (uint16_t) (depth + 1));
22450 return UP(pm_until_node_modifier_create(parser, &token, predicate, statements, PM_NODE_TYPE_P(node, PM_BEGIN_NODE) ? PM_LOOP_FLAGS_BEGIN_MODIFIER : 0));
22451 }
22452 case PM_TOKEN_KEYWORD_WHILE_MODIFIER: {
22453 parser_lex(parser);
22454 pm_statements_node_t *statements = pm_statements_node_single_create(parser, node, true);
22455
22456 pm_node_t *predicate = parse_value_expression(parser, binding_power, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), PM_ERR_CONDITIONAL_WHILE_PREDICATE, (uint16_t) (depth + 1));
22457 return UP(pm_while_node_modifier_create(parser, &token, predicate, statements, PM_NODE_TYPE_P(node, PM_BEGIN_NODE) ? PM_LOOP_FLAGS_BEGIN_MODIFIER : 0));
22458 }
22459 case PM_TOKEN_QUESTION_MARK:
22460 return UP(parse_ternary(parser, node, flags, depth));
22461 case PM_TOKEN_COLON_COLON:
22462 return parse_constant_path_infix(parser, node, previous_binding_power, flags, depth);
22463 case PM_TOKEN_KEYWORD_RESCUE_MODIFIER: {
22464 context_push(parser, PM_CONTEXT_RESCUE_MODIFIER);
22465 parser_lex(parser);
22466 accept1(parser, PM_TOKEN_NEWLINE);
22467
22468 pm_node_t *value = parse_rescue_modifier_value(parser, (uint8_t) ((flags & PM_PARSE_ACCEPTS_DO_BLOCK) | PM_PARSE_ACCEPTS_COMMAND_CALL), previous_binding_power == PM_BINDING_POWER_STATEMENT, (uint16_t) (depth + 1));
22469 context_pop(parser);
22470
22471 return UP(pm_rescue_modifier_node_create(parser, node, &token, value));
22472 }
22473 case PM_TOKEN_BRACKET_LEFT:
22474 return parse_aref(parser, node, previous_binding_power, flags, depth);
22475 case PM_TOKEN_KEYWORD_IN: {
22476 bool previous_pattern_matching_newlines = parser->pattern_matching_newlines;
22477 parser->pattern_matching_newlines = true;
22478
22479 pm_token_t operator = parser->current;
22480 parser->command_start = false;
22481 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
22482 parser_lex(parser);
22483
22484 pm_constant_id_set_t captures = { 0 };
22485 pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_IN, (uint16_t) (depth + 1));
22486
22487 parser->pattern_matching_newlines = previous_pattern_matching_newlines;
22488
22489 return UP(pm_match_predicate_node_create(parser, node, pattern, &operator));
22490 }
22491 case PM_TOKEN_EQUAL_GREATER: {
22492 bool previous_pattern_matching_newlines = parser->pattern_matching_newlines;
22493 parser->pattern_matching_newlines = true;
22494
22495 pm_token_t operator = parser->current;
22496 parser->command_start = false;
22497 lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
22498 parser_lex(parser);
22499
22500 pm_constant_id_set_t captures = { 0 };
22501 pm_node_t *pattern = parse_pattern(parser, &captures, PM_PARSE_PATTERN_TOP | PM_PARSE_PATTERN_MULTI, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET, (uint16_t) (depth + 1));
22502
22503 parser->pattern_matching_newlines = previous_pattern_matching_newlines;
22504
22505 return UP(pm_match_required_node_create(parser, node, pattern, &operator));
22506 }
22507 default:
22508 PRISM_ASSERT(false && "unreachable");
22509 return NULL;
22510 }
22511}
22512
22513#undef PM_PARSE_PATTERN_SINGLE
22514#undef PM_PARSE_PATTERN_TOP
22515#undef PM_PARSE_PATTERN_MULTI
22516
22529static bool
22530parse_expression_terminator(pm_parser_t *parser, pm_node_t *node) {
22531 pm_binding_power_t left = pm_binding_powers[parser->current.type].left;
22532
22533 switch (PM_NODE_TYPE(node)) {
22534 case PM_MULTI_WRITE_NODE:
22535 case PM_RETURN_NODE:
22536 case PM_BREAK_NODE:
22537 case PM_NEXT_NODE:
22538 return left > PM_BINDING_POWER_MODIFIER;
22539 case PM_CLASS_VARIABLE_WRITE_NODE:
22540 case PM_CONSTANT_PATH_WRITE_NODE:
22541 case PM_CONSTANT_WRITE_NODE:
22542 case PM_GLOBAL_VARIABLE_WRITE_NODE:
22543 case PM_INSTANCE_VARIABLE_WRITE_NODE:
22544 case PM_LOCAL_VARIABLE_WRITE_NODE:
22545 return PM_NODE_FLAG_P(node, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY) && left > PM_BINDING_POWER_MODIFIER;
22546 case PM_CALL_NODE: {
22547 // Calls with an implicit array on the right-hand side are
22548 // statements and can only be followed by modifiers.
22549 if (PM_NODE_FLAG_P(node, PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY)) {
22550 return left > PM_BINDING_POWER_MODIFIER;
22551 }
22552
22553 // Command-style calls (including block commands like
22554 // `foo bar do end`) can only be followed by composition
22555 // (and/or) and modifier (if/unless/etc.) operators.
22556 if (pm_command_call_value_p(parser, node)) {
22557 return left > PM_BINDING_POWER_COMPOSITION;
22558 }
22559
22560 // A block call (command with do-block, or any call chained
22561 // from one) can only be followed by call chaining (., ::,
22562 // &.), composition (and/or), and modifier operators.
22563 return left > PM_BINDING_POWER_COMPOSITION && left < PM_BINDING_POWER_CALL && pm_block_call_p(node);
22564 }
22565 case PM_SUPER_NODE:
22566 case PM_YIELD_NODE:
22567 // Command-style super/yield (without parens) can only be followed
22568 // by composition and modifier operators.
22569 if (pm_command_call_value_p(parser, node)) {
22570 return left > PM_BINDING_POWER_COMPOSITION;
22571 }
22572
22573 /* A super carrying a do-block is a block call, so it may also be
22574 * followed by call chaining (`.`, `::`, `&.`). */
22575 return left > PM_BINDING_POWER_COMPOSITION && left < PM_BINDING_POWER_CALL && pm_block_call_p(node);
22576 case PM_DEF_NODE:
22577 // An endless method whose body is a command-style call (e.g.,
22578 // `def f = foo bar`) is a command write and can only be
22579 // followed by modifiers.
22580 return left > PM_BINDING_POWER_MODIFIER && pm_command_call_value_p(parser, node);
22581 case PM_RESCUE_MODIFIER_NODE:
22582 // A rescue modifier whose handler is a pattern match (=> or in)
22583 // produces a statement and cannot be followed by operators above
22584 // the modifier level.
22585 if (left > PM_BINDING_POWER_MODIFIER) {
22586 pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node;
22587 pm_node_t *rescue_expression = cast->rescue_expression;
22588 return PM_NODE_TYPE_P(rescue_expression, PM_MATCH_REQUIRED_NODE) || PM_NODE_TYPE_P(rescue_expression, PM_MATCH_PREDICATE_NODE);
22589 }
22590 return false;
22591 default:
22592 return false;
22593 }
22594}
22595
22604static pm_node_t *
22605parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t flags, pm_diagnostic_id_t diag_id, uint16_t depth) {
22606 if (PRISM_UNLIKELY(depth >= PRISM_DEPTH_MAXIMUM || pm_parser_stack_exhausted(parser))) {
22607 return parse_resource_recovery(parser, depth);
22608 }
22609
22610 pm_node_t *node = parse_expression_prefix(parser, binding_power, flags, diag_id, depth);
22611
22612 // Some prefix nodes are statements and can only be followed by modifiers
22613 // (if/unless/while/until/rescue) or nothing at all. We check these cheaply
22614 // here before entering the infix loop.
22615 switch (PM_NODE_TYPE(node)) {
22616 case PM_ERROR_RECOVERY_NODE:
22617 return node;
22618 case PM_PRE_EXECUTION_NODE:
22619 return node;
22620 case PM_POST_EXECUTION_NODE:
22621 case PM_ALIAS_GLOBAL_VARIABLE_NODE:
22622 case PM_ALIAS_METHOD_NODE:
22623 case PM_UNDEF_NODE:
22624 if (pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_MODIFIER) {
22625 return node;
22626 }
22627 break;
22628 case PM_CALL_NODE:
22629 case PM_SUPER_NODE:
22630 case PM_YIELD_NODE:
22631 case PM_DEF_NODE:
22632 if (parse_expression_terminator(parser, node)) {
22633 return node;
22634 }
22635 break;
22636 case PM_SYMBOL_NODE:
22637 if (pm_symbol_node_label_p(parser, node)) {
22638 return node;
22639 }
22640 break;
22641 default:
22642 break;
22643 }
22644
22645 // Look and see if the next token can be parsed as an infix operator. If it
22646 // can, then we'll parse it using parse_expression_infix.
22647 pm_binding_powers_t current_binding_powers;
22648 pm_token_type_t current_token_type;
22649
22650 while (
22651 current_token_type = parser->current.type,
22652 current_binding_powers = pm_binding_powers[current_token_type],
22653 binding_power <= current_binding_powers.left &&
22654 current_binding_powers.binary
22655 ) {
22656 node = parse_expression_infix(parser, node, binding_power, current_binding_powers.right, flags, (uint16_t) (depth + 1));
22657 if (parse_expression_terminator(parser, node)) return node;
22658
22659 // If the operator is nonassoc and we should not be able to parse the
22660 // upcoming infix operator, break.
22661 if (current_binding_powers.nonassoc) {
22662 // If we are about to parse another non-associative operator at the
22663 // same precedence as the one we just parsed, then we need to add an
22664 // error. This covers chaining the same operator (`1 == 2 == 3`) as
22665 // well as different operators that share a precedence, since they
22666 // are equally non-associative with one another (`1 == 2 != 3`,
22667 // `1...2..3`).
22668 pm_binding_powers_t next_binding_powers = pm_binding_powers[parser->current.type];
22669 if (next_binding_powers.nonassoc && next_binding_powers.left == current_binding_powers.left) {
22670 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_str(parser->current.type), pm_token_str(current_token_type));
22671 break;
22672 }
22673
22674 // If this is an endless range, then we need to reject a couple of
22675 // additional operators because it violates the normal operator
22676 // precedence rules. Those patterns are:
22677 //
22678 // 1.. & 2
22679 // 1.. * 2
22680 //
22681 if (PM_NODE_TYPE_P(node, PM_RANGE_NODE) && ((pm_range_node_t *) node)->right == NULL) {
22682 if (match4(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_DOT, PM_TOKEN_AMPERSAND_DOT)) {
22683 PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_str(parser->current.type), pm_token_str(current_token_type));
22684 break;
22685 }
22686
22687 if (PM_BINDING_POWER_TERM <= next_binding_powers.left) {
22688 break;
22689 }
22690 } else if (current_binding_powers.left <= next_binding_powers.left) {
22691 break;
22692 }
22693 }
22694
22695 if (flags & PM_PARSE_ACCEPTS_COMMAND_CALL) {
22696 // A command-style method call is only accepted on method chains.
22697 // Thus, we check whether the parsed node can continue method chains.
22698 // The method chain can continue if the parsed node is one of the following five kinds:
22699 // (1) index access: foo[1]
22700 // (2) attribute access: foo.bar
22701 // (3) method call with parenthesis: foo.bar(1)
22702 // (4) method call with a block: foo.bar do end
22703 // (5) constant path: foo::Bar
22704 switch (node->type) {
22705 case PM_CALL_NODE: {
22706 pm_call_node_t *cast = (pm_call_node_t *)node;
22707 if (
22708 // (1) foo[1]
22709 !(
22710 cast->call_operator_loc.length == 0 &&
22711 cast->message_loc.length > 0 &&
22712 parser->start[cast->message_loc.start] == '[' &&
22713 parser->start[cast->message_loc.start + cast->message_loc.length - 1] == ']'
22714 ) &&
22715 // (2) foo.bar
22716 !(
22717 cast->call_operator_loc.length > 0 &&
22718 cast->arguments == NULL &&
22719 cast->block == NULL &&
22720 cast->opening_loc.length == 0
22721 ) &&
22722 // (3) foo.bar(1)
22723 !(
22724 cast->call_operator_loc.length > 0 &&
22725 cast->opening_loc.length > 0
22726 ) &&
22727 // (4) foo.bar do end
22728 !(
22729 cast->block != NULL && PM_NODE_TYPE_P(cast->block, PM_BLOCK_NODE)
22730 )
22731 ) {
22732 flags &= (uint8_t) ~(PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_COMMAND_DEF);
22733 }
22734 break;
22735 }
22736 // (5) foo::Bar
22737 case PM_CONSTANT_PATH_NODE:
22738 break;
22739 default:
22740 flags &= (uint8_t) ~(PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_COMMAND_DEF);
22741 break;
22742 }
22743 }
22744
22745 if (context_terminator(parser->current_context->context, &parser->current)) {
22746 pm_binding_powers_t next_binding_powers = pm_binding_powers[parser->current.type];
22747 if (
22748 !next_binding_powers.binary ||
22749 binding_power > next_binding_powers.left ||
22750 (PM_NODE_TYPE_P(node, PM_CALL_NODE) && pm_call_node_command_p((pm_call_node_t *) node))
22751 ) {
22752 return node;
22753 }
22754 }
22755 }
22756
22757 return node;
22758}
22759
22764static pm_statements_node_t *
22765wrap_statements(pm_parser_t *parser, pm_statements_node_t *statements) {
22766 pm_location_t location = statements == NULL ? (pm_location_t) { 0 } : statements->base.location;
22767
22768 if (PM_PARSER_COMMAND_LINE_OPTION_P(parser)) {
22769 if (statements == NULL) {
22770 statements = pm_statements_node_create(parser, &((pm_statements_t) { 0 }));
22771 }
22772
22773 pm_arguments_node_t *arguments = pm_arguments_node_single_create(
22774 parser,
22775 UP(pm_global_variable_read_node_synthesized_create(parser, pm_parser_constant_id_constant(parser, "$_", 2)))
22776 );
22777
22778 pm_statements_node_body_append(parser, statements, UP(pm_call_node_fcall_synthesized_create(
22779 parser,
22780 arguments,
22781 pm_parser_constant_id_constant(parser, "print", 5)
22782 )));
22783 }
22784
22785 if (PM_PARSER_COMMAND_LINE_OPTION_N(parser)) {
22786 if (PM_PARSER_COMMAND_LINE_OPTION_A(parser)) {
22787 if (statements == NULL) {
22788 statements = pm_statements_node_create(parser, &((pm_statements_t) { 0 }));
22789 }
22790
22791 pm_arguments_node_t *arguments = pm_arguments_node_single_create(
22792 parser,
22793 UP(pm_global_variable_read_node_synthesized_create(parser, pm_parser_constant_id_constant(parser, "$;", 2)))
22794 );
22795
22796 pm_global_variable_read_node_t *receiver = pm_global_variable_read_node_synthesized_create(parser, pm_parser_constant_id_constant(parser, "$_", 2));
22797 pm_call_node_t *call = pm_call_node_call_synthesized_create(parser, UP(receiver), "split", arguments, location);
22798
22799 pm_global_variable_write_node_t *write = pm_global_variable_write_node_synthesized_create(
22800 parser,
22801 pm_parser_constant_id_constant(parser, "$F", 2),
22802 UP(call)
22803 );
22804
22805 pm_statements_node_body_prepend(parser->arena, statements, UP(write));
22806 }
22807
22808 pm_arguments_t gets_arguments = { 0 };
22809 parse_arguments_append(
22810 parser,
22811 &gets_arguments,
22812 UP(pm_global_variable_read_node_synthesized_create(parser, pm_parser_constant_id_constant(parser, "$/", 2)))
22813 );
22814
22815 if (PM_PARSER_COMMAND_LINE_OPTION_L(parser)) {
22816 pm_hash_elements_t elements = { 0 };
22817 pm_hash_elements_append(parser, &elements, UP(pm_assoc_node_create(
22818 parser,
22819 UP(pm_symbol_node_synthesized_create(parser, "chomp")),
22820 NULL,
22821 UP(pm_true_node_synthesized_create(parser))
22822 )));
22823
22824 pm_keyword_hash_node_t *keywords = pm_keyword_hash_node_create(parser, &elements);
22825
22826 parse_arguments_append(parser, &gets_arguments, UP(keywords));
22827 gets_arguments.list_flags |= PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS;
22828 }
22829
22830 pm_arguments_node_t *arguments = pm_arguments_node_create(parser, &gets_arguments);
22831
22832 pm_statements_node_t *wrapped_statements = pm_statements_node_single_create(parser, UP(pm_while_node_synthesized_create(
22833 parser,
22834 UP(pm_call_node_fcall_synthesized_create(parser, arguments, pm_parser_constant_id_constant(parser, "gets", 4))),
22835 statements
22836 )), true);
22837
22838 statements = wrapped_statements;
22839 }
22840
22841 return statements;
22842}
22843
22847static pm_node_t *
22848parse_program(pm_parser_t *parser) {
22849 // If the current scope is NULL, then we want to push a new top level scope.
22850 // The current scope could exist in the event that we are parsing an eval
22851 // and the user has passed into scopes that already exist.
22852 if (parser->current_scope == NULL) {
22853 pm_parser_scope_push(parser, true);
22854 }
22855
22856 pm_scratch_node_list_t current_block_exits = { 0 };
22857 pm_scratch_node_list_t *previous_block_exits = push_block_exits(parser, &current_block_exits);
22858
22859 parser_lex(parser);
22860 pm_statements_node_t *statements = parse_statements(parser, PM_CONTEXT_MAIN, 0);
22861
22862 if (statements != NULL && !parser->parsing_eval) {
22863 // If we have statements, then the top-level statement should be
22864 // explicitly checked as well. We have to do this here because
22865 // everywhere else we check all but the last statement.
22866 PRISM_ASSERT(statements->body.size > 0);
22867 pm_void_statement_check_unwrap(parser, statements->body.nodes[statements->body.size - 1]);
22868 }
22869
22870 pm_constant_id_list_t locals;
22871 pm_locals_order(parser, &locals, true);
22872 pm_parser_scope_pop(parser);
22873
22874 // At the top level, see if we need to wrap the statements in a program
22875 // node with a while loop based on the options.
22876 if (parser->command_line & (PM_OPTIONS_COMMAND_LINE_P | PM_OPTIONS_COMMAND_LINE_N)) {
22877 statements = wrap_statements(parser, statements);
22878 } else {
22879 flush_block_exits(parser, previous_block_exits);
22880 }
22881
22882 // If this is an empty file, then we're still going to parse all of the
22883 // statements in order to gather up all of the comments and such. Here we'll
22884 // correct the location information.
22885 if (statements == NULL) {
22886 statements = pm_statements_node_create(parser, &((pm_statements_t) { 0 }));
22887 }
22888
22889 return UP(pm_program_node_create(parser, &locals, statements));
22890}
22891
22892/******************************************************************************/
22893/* External functions */
22894/******************************************************************************/
22895
22905static const char *
22906pm_strnstr(const char *big, const char *little, size_t big_length) {
22907 size_t little_length = strlen(little);
22908
22909 for (const char *max = big + big_length - little_length; big <= max; big++) {
22910 if (*big == *little && memcmp(big, little, little_length) == 0) return big;
22911 }
22912
22913 return NULL;
22914}
22915
22916#ifdef _WIN32
22917#define pm_parser_warn_shebang_carriage_return(parser, start, length) ((void) 0)
22918#else
22924static void
22925pm_parser_warn_shebang_carriage_return(pm_parser_t *parser, const uint8_t *start, size_t length) {
22926 if (length > 2 && start[length - 2] == '\r' && start[length - 1] == '\n') {
22927 pm_parser_warn(parser, U32(start - parser->start), U32(length), PM_WARN_SHEBANG_CARRIAGE_RETURN);
22928 }
22929}
22930#endif
22931
22936static void
22937pm_parser_init_shebang(pm_parser_t *parser, const pm_options_t *options, const char *engine, size_t length) {
22938 const char *switches = pm_strnstr(engine, " -", length);
22939 if (switches == NULL) return;
22940
22941 pm_options_t next_options = *options;
22942 options->shebang_callback(
22943 &next_options,
22944 (const uint8_t *) (switches + 1),
22945 length - ((size_t) (switches - engine)) - 1,
22946 options->shebang_callback_data
22947 );
22948
22949 size_t encoding_length;
22950 if ((encoding_length = pm_string_length(&next_options.encoding)) > 0) {
22951 const uint8_t *encoding_source = pm_string_source(&next_options.encoding);
22952 parser_lex_magic_comment_encoding_value(parser, encoding_source, encoding_source + encoding_length);
22953 }
22954
22955 parser->command_line = next_options.command_line;
22956 parser->frozen_string_literal = next_options.frozen_string_literal;
22957}
22958
22965#ifndef PM_CONSTANT_POOL_MIN_CAPACITY
22966#define PM_CONSTANT_POOL_MIN_CAPACITY 4
22967#endif
22968
22974static size_t
22975pm_size_sqrt(size_t value) {
22976 size_t result = 0;
22977 size_t bit = ((size_t) 1) << (sizeof(size_t) * 8 - 2);
22978
22979 while (bit > value) bit >>= 2;
22980
22981 while (bit != 0) {
22982 if (value >= result + bit) {
22983 value -= result + bit;
22984 result = (result >> 1) + bit;
22985 } else {
22986 result >>= 1;
22987 }
22988
22989 bit >>= 2;
22990 }
22991
22992 return result;
22993}
22994
22998void
22999pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options) {
23000 PRISM_ASSERT(arena != NULL);
23001 PRISM_ASSERT(source != NULL);
23002
23003 *parser = (pm_parser_t) {
23004 .arena = arena,
23005 .metadata_arena = { 0 },
23006 .node_id = 0,
23007 .lex_state = PM_LEX_STATE_BEG,
23008 .enclosure_nesting = 0,
23009 .lambda_enclosure_nesting = -1,
23010 .brace_nesting = 0,
23011 .do_loop_stack = { 0 },
23012 .accepts_block_stack = { 0 },
23013 .lex_modes = {
23014 .current = &parser->lex_modes.base,
23015 .base = { .mode = PM_LEX_DEFAULT },
23016 },
23017 .node_chunks = { .arena = &parser->scratch_arena },
23018 .start = source,
23019 .end = source + size,
23020 .previous = { .type = PM_TOKEN_EOF, .start = source, .end = source },
23021 .current = { .type = PM_TOKEN_EOF, .start = source, .end = source },
23022 .next_start = NULL,
23023 .heredoc_end = NULL,
23024 .data_loc = { 0 },
23025 .comment_list = { 0 },
23026 .magic_comment_list = { 0 },
23027 .warning_list = { 0 },
23028 .error_list = { 0 },
23029 .current_scope = NULL,
23030 .locals_size = 1,
23031 .current_context = NULL,
23032 .encoding = PM_ENCODING_UTF_8_ENTRY,
23033 .encoding_changed_callback = NULL,
23034 .encoding_comment_start = source,
23035 .lex_callback = { 0 },
23036 .filepath = PM_STRING_EMPTY,
23037 .constant_pool = { 0 },
23038 .line_offsets = { 0 },
23039 .integer = { 0 },
23040 .current_string = PM_STRING_EMPTY,
23041 .start_line = 1,
23042 .explicit_encoding = NULL,
23043 .command_line = 0,
23044 .parsing_eval = false,
23045 .partial_script = false,
23046 .attach_comments = false,
23047 .command_start = true,
23048 .recovering = false,
23049 .continuable = true,
23050 .encoding_locked = false,
23051 .encoding_changed = false,
23052 .pattern_matching_newlines = false,
23053 .in_keyword_arg = false,
23054 .current_block_exits = NULL,
23055 .stack_base = 0,
23056 .stack_budget = SIZE_MAX,
23057 .semantic_token_seen = false,
23058 .frozen_string_literal = PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET,
23059 .warn_mismatched_indentation = true
23060 };
23061
23062 /* Pre-size the AST arena based on input size to reduce the number of block
23063 * allocations (and the kernel page zeroing they trigger). The tree grows
23064 * linearly with the source. Measured across 16,460 stdlib and gem files,
23065 * it takes about 4.6 bytes per source byte at the median, from about 3.3
23066 * at the 25th percentile to about 6 at the 75th, depending on how much of
23067 * the source is comments and how much is literals. A source past the
23068 * reserve continues in further blocks. The reserve call is a no-op when
23069 * the capacity is at or below the default arena block size, so small
23070 * inputs don't waste an extra allocation. */
23071 if (size <= SIZE_MAX / 4) pm_arena_reserve(arena, size * 4);
23072
23073 /* The number of distinct names in a source grows with the square root of
23074 * its size, not with its size, because longer sources mostly reuse names.
23075 * Measured across 16,460 stdlib and gem files, a source holds about
23076 * sqrt(size) names at the median and 1.5 * sqrt(size) at the 90th
23077 * percentile. Starting the pool at twice that keeps nearly every source
23078 * below the load at which the pool grows, and each growth leaves the
23079 * outgrown buckets behind in the metadata arena. Ids are 30 bits wide,
23080 * which bounds the capacity. */
23081 size_t constant_estimate = pm_size_sqrt(size) * 2;
23082 if (constant_estimate > ((size_t) 1 << 30)) constant_estimate = (size_t) 1 << 30;
23083
23084 uint32_t constant_size = (uint32_t) constant_estimate;
23085 if (constant_size < PM_CONSTANT_POOL_MIN_CAPACITY) constant_size = PM_CONSTANT_POOL_MIN_CAPACITY;
23086
23087 /* Estimate the number of newlines from the size of the input. */
23088 size_t newline_size = size / 22;
23089 if (newline_size < 4) newline_size = 4;
23090
23091 /* Pre-size the metadata arena to hold the constant pool and the line
23092 * offsets at the sizes they start with, so that they share its first
23093 * block. Half of the input size on top covers what accumulates while
23094 * parsing, which is mostly comments. */
23095 if (size <= SIZE_MAX / 2) {
23096 pm_arena_reserve(&parser->metadata_arena, pm_constant_pool_init_size(constant_size) + newline_size * sizeof(uint32_t) + size / 2);
23097 }
23098
23099 pm_constant_pool_init(&parser->metadata_arena, &parser->constant_pool, constant_size);
23100 pm_line_offset_list_init(&parser->metadata_arena, &parser->line_offsets, newline_size);
23101
23102 // If options were provided to this parse, establish them here.
23103 if (options != NULL) {
23104 // filepath option
23105 if (pm_string_length(&options->filepath) > 0) {
23106 parser->filepath = options->filepath;
23107 }
23108
23109 // line option
23110 parser->start_line = options->line;
23111
23112 // encoding option
23113 size_t encoding_length = pm_string_length(&options->encoding);
23114 if (encoding_length > 0) {
23115 const uint8_t *encoding_source = pm_string_source(&options->encoding);
23116 parser_lex_magic_comment_encoding_value(parser, encoding_source, encoding_source + encoding_length);
23117 }
23118
23119 // encoding_locked option
23120 parser->encoding_locked = options->encoding_locked;
23121
23122 // frozen_string_literal option
23123 parser->frozen_string_literal = options->frozen_string_literal;
23124
23125 // command_line option
23126 parser->command_line = options->command_line;
23127
23128 // version option
23129 parser->version = options->version;
23130
23131 // partial_script
23132 parser->partial_script = options->partial_script;
23133
23134 // attach_comments
23135 parser->attach_comments = options->attach_comments;
23136
23137 // stack_budget
23138 parser->stack_budget = options->stack_budget;
23139
23140 // scopes option
23141 parser->parsing_eval = options->scopes_count > 0;
23142 if (parser->parsing_eval) parser->warn_mismatched_indentation = false;
23143
23144 for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
23145 const pm_options_scope_t *scope = pm_options_scope(options, scope_index);
23146 pm_parser_scope_push(parser, scope_index == 0);
23147
23148 // Scopes given from the outside are not allowed to have numbered
23149 // parameters.
23150 parser->current_scope->parameters = ((pm_scope_parameters_t) scope->forwarding) | PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED;
23151
23152 for (size_t local_index = 0; local_index < scope->locals_count; local_index++) {
23153 const pm_string_t *local = pm_options_scope_local(scope, local_index);
23154
23155 const uint8_t *source = pm_string_source(local);
23156 size_t length = pm_string_length(local);
23157
23158 uint8_t *allocated = PM_ARENA_ALLOC_ARRAY(&parser->metadata_arena, uint8_t, length);
23159 memcpy(allocated, source, length);
23160 pm_parser_local_add_owned(parser, allocated, length);
23161 }
23162 }
23163 }
23164
23165 // Now that we have established the user-provided options, check if
23166 // a version was given and parse as the latest version otherwise.
23167 if (parser->version == PM_OPTIONS_VERSION_UNSET) {
23168 parser->version = PM_OPTIONS_VERSION_LATEST;
23169 }
23170
23171 pm_accepts_block_stack_push(parser, true);
23172
23173 // Skip past the UTF-8 BOM if it exists.
23174 if (size >= 3 && source[0] == 0xef && source[1] == 0xbb && source[2] == 0xbf) {
23175 parser->current.end += 3;
23176 parser->encoding_comment_start += 3;
23177
23178 if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
23179 parser->encoding = PM_ENCODING_UTF_8_ENTRY;
23180 if (parser->encoding_changed_callback != NULL) parser->encoding_changed_callback(parser);
23181 }
23182 }
23183
23184 // If the -x command line flag is set, or the first shebang of the file does
23185 // not include "ruby", then we'll search for a shebang that does include
23186 // "ruby" and start parsing from there.
23187 bool search_shebang = PM_PARSER_COMMAND_LINE_OPTION_X(parser);
23188
23189 // If the first two bytes of the source are a shebang, then we will do a bit
23190 // of extra processing.
23191 //
23192 // First, we'll indicate that the encoding comment is at the end of the
23193 // shebang. This means that when a shebang is present the encoding comment
23194 // can begin on the second line.
23195 //
23196 // Second, we will check if the shebang includes "ruby". If it does, then we
23197 // we will start parsing from there. We will also potentially warning the
23198 // user if there is a carriage return at the end of the shebang. We will
23199 // also potentially call the shebang callback if this is the main script to
23200 // allow the caller to parse the shebang and find any command-line options.
23201 // If the shebang does not include "ruby" and this is the main script being
23202 // parsed, then we will start searching the file for a shebang that does
23203 // contain "ruby" as if -x were passed on the command line.
23204 const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end);
23205 size_t length = (size_t) ((newline != NULL ? newline : parser->end) - parser->current.end);
23206
23207 if (length >= 2 && parser->current.end[0] == '#' && parser->current.end[1] == '!') {
23208 parser->encoding_comment_start = newline != NULL ? newline + 1 : parser->end;
23209 const char *engine;
23210
23211 if ((engine = pm_strnstr((const char *) parser->start, "ruby", length)) != NULL) {
23212 if (newline != NULL) {
23213 if (options == NULL || options->main_script) {
23214 pm_parser_warn_shebang_carriage_return(parser, parser->start, length + 1);
23215 }
23216 }
23217
23218 if (options != NULL && options->main_script && options->shebang_callback != NULL) {
23219 pm_parser_init_shebang(parser, options, engine, length - ((size_t) (engine - (const char *) parser->start)));
23220 }
23221
23222 search_shebang = false;
23223 } else if (length > 2 && options != NULL && options->main_script && !parser->parsing_eval) {
23224 search_shebang = true;
23225 }
23226 }
23227
23228 // Here we're going to find the first shebang that includes "ruby" and start
23229 // parsing from there.
23230 if (search_shebang) {
23231 // If a shebang that includes "ruby" is not found, then we're going to a
23232 // a load error to the list of errors on the parser.
23233 bool found_shebang = false;
23234
23235 // This is going to point to the start of each line as we check it.
23236 // We'll maintain a moving window looking at each line at they come.
23237 const uint8_t *cursor = parser->start;
23238
23239 // The newline pointer points to the end of the current line that we're
23240 // considering. If it is NULL, then we're at the end of the file.
23241 const uint8_t *newline = next_newline(cursor, parser->end - cursor);
23242
23243 while (newline != NULL) {
23244 pm_line_offset_list_append(&parser->metadata_arena, &parser->line_offsets, U32(newline - parser->start + 1));
23245
23246 cursor = newline + 1;
23247 newline = next_newline(cursor, parser->end - cursor);
23248
23249 size_t length = (size_t) ((newline != NULL ? newline : parser->end) - cursor);
23250 if (length > 2 && cursor[0] == '#' && cursor[1] == '!') {
23251 const char *engine;
23252 if ((engine = pm_strnstr((const char *) cursor, "ruby", length)) != NULL) {
23253 found_shebang = true;
23254
23255 if (newline != NULL) {
23256 pm_parser_warn_shebang_carriage_return(parser, cursor, length + 1);
23257 parser->encoding_comment_start = newline + 1;
23258 }
23259
23260 if (options != NULL && options->shebang_callback != NULL) {
23261 pm_parser_init_shebang(parser, options, engine, length - ((size_t) (engine - (const char *) cursor)));
23262 }
23263
23264 break;
23265 }
23266 }
23267 }
23268
23269 if (found_shebang) {
23270 parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor };
23271 parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor };
23272 } else {
23273 pm_parser_err(parser, 0, 0, PM_ERR_SCRIPT_NOT_FOUND);
23274 pm_line_offset_list_clear(&parser->line_offsets);
23275 }
23276 }
23277
23278 // The encoding comment can start after any amount of inline whitespace, so
23279 // here we'll advance it to the first non-inline-whitespace character so
23280 // that it is ready for future comparisons.
23281 parser->encoding_comment_start += pm_strspn_inline_whitespace(parser->encoding_comment_start, parser->end - parser->encoding_comment_start);
23282}
23283
23292pm_parser_new(pm_arena_t *arena, const uint8_t *source, size_t size, const pm_options_t *options) {
23293 pm_parser_t *parser = (pm_parser_t *) xmalloc(sizeof(pm_parser_t));
23294 if (parser == NULL) PRISM_OOM();
23295
23296 pm_parser_init(arena, parser, source, size, options);
23297 return parser;
23298}
23299
23303void
23304pm_parser_cleanup(pm_parser_t *parser) {
23305 pm_arena_cleanup(&parser->metadata_arena);
23306 pm_arena_cleanup(&parser->scratch_arena);
23307}
23308
23312void
23314 pm_parser_cleanup(parser);
23315 xfree_sized(parser, sizeof(pm_parser_t));
23316}
23317
23323static bool
23324pm_parse_err_is_fatal(pm_diagnostic_id_t diag_id) {
23325 switch (diag_id) {
23326 case PM_ERR_ARRAY_EXPRESSION_AFTER_STAR:
23327 case PM_ERR_BEGIN_UPCASE_BRACE:
23328 case PM_ERR_CLASS_VARIABLE_BARE:
23329 case PM_ERR_END_UPCASE_BRACE:
23330 case PM_ERR_ESCAPE_INVALID_HEXADECIMAL:
23331 case PM_ERR_ESCAPE_INVALID_UNICODE_LIST:
23332 case PM_ERR_ESCAPE_INVALID_UNICODE_SHORT:
23333 case PM_ERR_EXPRESSION_NOT_WRITABLE:
23334 case PM_ERR_EXPRESSION_NOT_WRITABLE_SELF:
23335 case PM_ERR_GLOBAL_VARIABLE_BARE:
23336 case PM_ERR_HASH_KEY:
23337 case PM_ERR_HEREDOC_IDENTIFIER:
23338 case PM_ERR_INSTANCE_VARIABLE_BARE:
23339 case PM_ERR_INVALID_BLOCK_EXIT:
23340 case PM_ERR_INVALID_ENCODING_MAGIC_COMMENT:
23341 case PM_ERR_INVALID_FLOAT_EXPONENT:
23342 case PM_ERR_INVALID_NUMBER_BINARY:
23343 case PM_ERR_INVALID_NUMBER_DECIMAL:
23344 case PM_ERR_INVALID_NUMBER_HEXADECIMAL:
23345 case PM_ERR_INVALID_NUMBER_OCTAL:
23346 case PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING:
23347 case PM_ERR_NO_LOCAL_VARIABLE:
23348 case PM_ERR_PARAMETER_ORDER:
23349 case PM_ERR_STACK_EXHAUSTED:
23350 case PM_ERR_STATEMENT_UNDEF:
23351 case PM_ERR_VOID_EXPRESSION:
23352 return true;
23353 default:
23354 return false;
23355 }
23356}
23357
23391static void
23392pm_parse_continuable(pm_parser_t *parser) {
23393 // If there are no errors then there is nothing to continue.
23394 if (parser->error_list.size == 0) {
23395 parser->continuable = false;
23396 return;
23397 }
23398
23399 if (!parser->continuable) return;
23400
23401 size_t source_length = (size_t) (parser->end - parser->start);
23402
23403 // First pass: check if there are any non-stray, non-fatal errors.
23404 bool has_non_stray_error = false;
23405 for (pm_diagnostic_t *error = (pm_diagnostic_t *) parser->error_list.head; error != NULL; error = (pm_diagnostic_t *) error->node.next) {
23406 if (error->diag_id != PM_ERR_UNEXPECTED_TOKEN_IGNORE && error->diag_id != PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT && !pm_parse_err_is_fatal(error->diag_id)) {
23407 has_non_stray_error = true;
23408 break;
23409 }
23410 }
23411
23412 // Second pass: check each error. We track the minimum source position
23413 // among non-stray, non-fatal errors seen so far in list order, which
23414 // lets us detect cascade stray tokens.
23415 size_t non_stray_min_start = SIZE_MAX;
23416
23417 for (pm_diagnostic_t *error = (pm_diagnostic_t *) parser->error_list.head; error != NULL; error = (pm_diagnostic_t *) error->node.next) {
23418 size_t error_start = (size_t) error->location.start;
23419 size_t error_end = error_start + (size_t) error->location.length;
23420 bool at_eof = error_end >= source_length;
23421
23422 // Fatal errors are non-continuable unless they occur at EOF.
23423 if (pm_parse_err_is_fatal(error->diag_id) && !at_eof) {
23424 parser->continuable = false;
23425 return;
23426 }
23427
23428 // Track non-stray, non-fatal error positions in list order.
23429 if (error->diag_id != PM_ERR_UNEXPECTED_TOKEN_IGNORE &&
23430 error->diag_id != PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT) {
23431 if (error_start < non_stray_min_start) non_stray_min_start = error_start;
23432 continue;
23433 }
23434
23435 // This is a stray token. Determine if it is a cascade effect
23436 // of a preceding error or genuinely stray.
23437
23438 // Rule (a): a non-stray error was seen earlier in the list at a
23439 // strictly earlier position — this stray is a cascade effect.
23440 if (non_stray_min_start < error_start) continue;
23441
23442 // Rule (b): this stray is at EOF with valid code before it.
23443 // Single-byte stray tokens at EOF (like `\` for line continuation)
23444 // are likely truncated tokens. Multi-byte stray tokens (like the
23445 // keyword `end`) need additional evidence that they are cascade
23446 // effects (i.e. non-stray errors exist elsewhere).
23447 if (at_eof && error_start > 0) {
23448 // Exception: closing delimiters at EOF are genuinely stray.
23449 if (error->location.length == 1) {
23450 const uint8_t *byte = parser->start + error_start;
23451 if (*byte == ')' || *byte == ']' || *byte == '}') {
23452 parser->continuable = false;
23453 return;
23454 }
23455
23456 // Single-byte non-delimiter stray at EOF: cascade.
23457 continue;
23458 }
23459
23460 // Multi-byte stray at EOF: cascade only if there are
23461 // non-stray errors (evidence of a preceding parse failure).
23462 if (has_non_stray_error) continue;
23463 }
23464
23465 // Rule (c): a stray `=` at the start of a line could be the
23466 // beginning of an embedded document (`=begin`). The remaining
23467 // bytes after `=` parse as an identifier, so the error is not
23468 // at EOF, but the construct is genuinely incomplete.
23469 if (error->location.length == 1) {
23470 const uint8_t *byte = parser->start + error_start;
23471 if (*byte == '=' && (error_start == 0 || *(byte - 1) == '\n')) continue;
23472 }
23473
23474 // This stray token is genuinely non-continuable.
23475 parser->continuable = false;
23476 return;
23477 }
23478}
23479
23483pm_node_t *
23485 parser->stack_base = pm_stack_address();
23486 pm_node_t *node = parse_program(parser);
23487 pm_parse_continuable(parser);
23488 if (parser->attach_comments) pm_comments_attach(parser, node);
23489 pm_arena_cleanup(&parser->scratch_arena);
23490 return node;
23491}
23492
23499pm_node_t *
23500pm_parse_stream(pm_parser_t **parser, pm_arena_t *arena, pm_source_t *source, const pm_options_t *options) {
23501 bool eof = pm_source_stream_read(source);
23502
23503 pm_parser_t *tmp = pm_parser_new(arena, pm_source_source(source), pm_source_length(source), options);
23504 pm_node_t *node = pm_parse(tmp);
23505
23506 /* The stream reader stops at possible markers. Only the lexer knows if
23507 * the marker is real, even when the preceding source has syntax errors. */
23508 while (!eof && tmp->data_loc.length == 0) {
23509 eof = pm_source_stream_read(source);
23510
23511 pm_parser_free(tmp);
23512 pm_arena_cleanup(arena);
23513
23514 tmp = pm_parser_new(arena, pm_source_source(source), pm_source_length(source), options);
23515 node = pm_parse(tmp);
23516 }
23517
23518 *parser = tmp;
23519 return node;
23520}
23521
23522#undef PM_CASE_KEYWORD
23523#undef PM_CASE_OPERATOR
23524#undef PM_CASE_WRITABLE
23525#undef PM_STRING_EMPTY
23526
23527// We optionally support serializing to a binary string. For systems that don't
23528// want or need this functionality, it can be turned off with the
23529// PRISM_EXCLUDE_SERIALIZATION define.
23530#ifndef PRISM_EXCLUDE_SERIALIZATION
23531
23532static PRISM_INLINE void
23533pm_serialize_header(pm_buffer_t *buffer) {
23534 PM_BUFFER_APPEND_LITERAL(buffer, "PRISM");
23535 pm_buffer_append_byte(buffer, PRISM_VERSION_MAJOR);
23536 pm_buffer_append_byte(buffer, PRISM_VERSION_MINOR);
23537 pm_buffer_append_byte(buffer, PRISM_VERSION_PATCH);
23538 pm_buffer_append_byte(buffer, PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS ? 1 : 0);
23539}
23540
23544void
23545pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
23546 pm_serialize_header(buffer);
23547 pm_serialize_content(parser, node, buffer);
23548 pm_buffer_append_byte(buffer, '\0');
23549}
23550
23555void
23556pm_serialize_parse(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
23557 pm_options_t options = { 0 };
23558 pm_options_read(&options, data);
23559
23560 pm_arena_t arena = { 0 };
23561 pm_parser_t parser;
23562 pm_parser_init(&arena, &parser, source, size, &options);
23563
23564 pm_node_t *node = pm_parse(&parser);
23565
23566 pm_serialize_header(buffer);
23567 pm_serialize_content(&parser, node, buffer);
23568 pm_buffer_append_byte(buffer, '\0');
23569
23570 pm_parser_cleanup(&parser);
23571 pm_arena_cleanup(&arena);
23572 pm_options_cleanup(&options);
23573}
23574
23579void
23580pm_serialize_parse_stream(pm_buffer_t *buffer, pm_source_t *source, const char *data) {
23581 pm_arena_t arena = { 0 };
23582 pm_parser_t *parser;
23583 pm_options_t options = { 0 };
23584 pm_options_read(&options, data);
23585
23586 pm_node_t *node = pm_parse_stream(&parser, &arena, source, &options);
23587 pm_serialize_header(buffer);
23588 pm_serialize_content(parser, node, buffer);
23589 pm_buffer_append_byte(buffer, '\0');
23590
23591 pm_parser_free(parser);
23592 pm_arena_cleanup(&arena);
23593 pm_options_cleanup(&options);
23594}
23595
23604int8_t
23605pm_serialize_parse_errors_format(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data, pm_errors_format_type_t format_type) {
23606 pm_options_t options = { 0 };
23607 pm_options_read(&options, data);
23608
23609 pm_arena_t arena = { 0 };
23610 pm_parser_t parser;
23611 pm_parser_init(&arena, &parser, source, size, &options);
23612
23613 pm_parse(&parser);
23614
23615 int8_t result = -1;
23616 if (parser.error_list.size > 0) {
23617 const char *encoding_name = parser.encoding->name;
23618 pm_buffer_append_string(buffer, encoding_name, strlen(encoding_name));
23619 pm_buffer_append_byte(buffer, '\0');
23620
23621 result = (int8_t) pm_errors_format(&parser, buffer, format_type);
23622 }
23623
23624 pm_parser_cleanup(&parser);
23625 pm_arena_cleanup(&arena);
23626 pm_options_cleanup(&options);
23627
23628 return result;
23629}
23630
23634void
23635pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
23636 pm_options_t options = { 0 };
23637 pm_options_read(&options, data);
23638
23639 pm_arena_t arena = { 0 };
23640 pm_parser_t parser;
23641 pm_parser_init(&arena, &parser, source, size, &options);
23642
23643 pm_parse(&parser);
23644 pm_serialize_header(buffer);
23645 pm_serialize_encoding(parser.encoding, buffer);
23646 pm_buffer_append_varsint(buffer, parser.start_line);
23647 pm_buffer_append_uint64(buffer, pm_parser_checksum(&parser));
23648 pm_serialize_line_offset_list(&parser.line_offsets, buffer);
23649 pm_serialize_comment_list(&parser.comment_list, buffer);
23650
23651 pm_parser_cleanup(&parser);
23652 pm_arena_cleanup(&arena);
23653 pm_options_cleanup(&options);
23654}
23655
23656#endif
#define PRISM_ALIGNOF
Get the alignment requirement of a type.
Definition align.h:15
#define PRISM_UNLIKELY(x)
The compiler should predicate that this branch will not be taken.
Definition branch.h:17
pm_comment_type_t
This is the type of a comment that we've found while parsing.
Definition comments.h:22
uint32_t pm_constant_id_t
A constant id is a unique identifier for a constant in the constant pool.
pm_errors_format_type_t
The type of formatting to use when formatting errors.
A header file that defines macros to exclude certain features of the prism library.
#define PRISM_FALLTHROUGH
We use -Wimplicit-fallthrough to guard potentially unintended fall-through between cases of a switch.
Definition fallthrough.h:15
#define xmalloc
Old name of ruby_xmalloc.
Definition xmalloc.h:53
int len
Length of the buffer.
Definition io.h:8
#define PRISM_INLINE
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
Definition inline.h:12
VALUE type(ANYARGS)
ANYARGS-ed function type.
T * cast(pm_node_t *node) noexcept
Cast the given node, which must not be null and must be of the given struct, to that struct.
Definition node.hpp:50
static const uint8_t PM_OPTIONS_COMMAND_LINE_N
A bit representing whether or not the command line -n option was set.
Definition options.h:100
#define PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
String literals should not be frozen.
Definition options.h:35
#define PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
String literals should be made frozen.
Definition options.h:46
#define PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
String literals may be frozen or mutable depending on the implementation default.
Definition options.h:41
static const uint8_t PM_OPTIONS_COMMAND_LINE_P
A bit representing whether or not the command line -p option was set.
Definition options.h:106
PRISM_NODISCARD PRISM_EXPORTED_FUNCTION pm_parser_t * pm_parser_new(pm_arena_t *arena, const uint8_t *source, size_t size, const pm_options_t *options) PRISM_NONNULL(1)
Allocate and initialize a parser with the given start and end pointers.
Definition prism.c:23292
PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser) PRISM_NONNULL(1)
Free both the memory held by the given parser and the parser itself.
Definition prism.c:23313
PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser) PRISM_NONNULL(1)
Initiate the parser with the given parser.
Definition prism.c:23484
std::string_view constant(const pm_parser_t *parser PRISM_LIFETIMEBOUND, pm_constant_id_t constant_id) noexcept
The name of the constant with the given id, which must not be zero.
Definition parser.hpp:64
pm_visit_result_t
The value returned by a visitor callback, which determines where the walk goes next.
Definition node.h:49
@ PM_VISIT_SKIP
Continue the walk without visiting this node's children.
Definition node.h:51
@ PM_VISIT_DESCEND
Continue the walk by visiting this node's children.
Definition node.h:54
#define PM_NODE_LIST_FOREACH(list, index, node)
Loop through each node in the node list, writing each node to the given pm_node_t pointer.
Definition node.h:22
The version of the Prism library.
#define PRISM_VERSION
The version of the Prism library as a constant string.
Definition version.h:33
#define PRISM_VERSION_PATCH
The patch version of the Prism library as an int.
Definition version.h:28
#define PRISM_VERSION_MINOR
The minor version of the Prism library as an int.
Definition version.h:23
#define PRISM_VERSION_MAJOR
The major version of the Prism library as an int.
Definition version.h:18
PRISM_EXPORTED_FUNCTION const char * pm_version(void)
The prism version and the serialization format.
Definition prism.c:74
The functions related to serializing the AST to a binary format.
static PRISM_INLINE uintptr_t pm_stack_address(void)
Returns the current position of the C stack, for measuring how much of it has been consumed since an ...
Definition stack.h:34
Functions for parsing streams.
This is a special out parameter to the parse_arguments_list function that includes opening and closin...
Definition prism.c:1734
pm_node_t * block
The optional block attached to the call.
Definition prism.c:1751
bool has_forwarding
The flag indicating whether this arguments list has forwarding argument.
Definition prism.c:1754
pm_location_t opening_loc
The optional location of the opening parenthesis or bracket.
Definition prism.c:1736
pm_node_flags_t list_flags
The flags of the arguments node that depend on the arguments in list.
Definition prism.c:1745
pm_node_list_t list
The open list of arguments while they are being parsed.
Definition prism.c:1739
pm_location_t list_loc
The location covering the arguments in list.
Definition prism.c:1742
pm_location_t closing_loc
The optional location of the closing parenthesis or bracket.
Definition prism.c:1748
The elements of an ArrayNode node that is still being parsed.
Definition prism.c:2425
pm_node_list_t nodes
The open list of elements.
Definition prism.c:2427
PM_NODE_ALIGNAS struct pm_node * constant
ArrayPatternNode::constant.
Definition ast.h:1487
pm_location_t opening_loc
ArrayPatternNode::opening_loc.
Definition ast.h:1527
pm_location_t closing_loc
ArrayPatternNode::closing_loc.
Definition ast.h:1537
PM_NODE_ALIGNAS struct pm_node * value
AssocNode::value.
Definition ast.h:1583
PM_NODE_ALIGNAS struct pm_node * key
AssocNode::key.
Definition ast.h:1570
PM_NODE_ALIGNAS struct pm_rescue_node * rescue_clause
BeginNode::rescue_clause.
Definition ast.h:1707
pm_node_t base
The embedded base node.
Definition ast.h:1677
This struct represents a set of binding powers used for a given token.
Definition prism.c:12722
bool binary
Whether or not this token can be used as a binary operator.
Definition prism.c:12730
pm_binding_power_t left
The left binding power.
Definition prism.c:12724
bool nonassoc
Whether or not this token can be used as non-associative binary operator.
Definition prism.c:12736
pm_binding_power_t right
The right binding power.
Definition prism.c:12727
pm_location_t opening_loc
CallNode::opening_loc.
Definition ast.h:2232
pm_location_t closing_loc
CallNode::closing_loc.
Definition ast.h:2252
pm_constant_id_t name
CallNode::name.
Definition ast.h:2212
PM_NODE_ALIGNAS struct pm_arguments_node * arguments
CallNode::arguments.
Definition ast.h:2242
pm_location_t equal_loc
CallNode::equal_loc.
Definition ast.h:2265
pm_location_t call_operator_loc
CallNode::call_operator_loc.
Definition ast.h:2202
pm_location_t message_loc
CallNode::message_loc.
Definition ast.h:2222
PM_NODE_ALIGNAS struct pm_node * block
CallNode::block.
Definition ast.h:2275
PM_NODE_ALIGNAS struct pm_node * receiver
CallNode::receiver.
Definition ast.h:2189
A list of constant IDs.
pm_location_t opening_loc
FindPatternNode::opening_loc.
Definition ast.h:3915
PM_NODE_ALIGNAS struct pm_node * constant
FindPatternNode::constant.
Definition ast.h:3863
pm_location_t closing_loc
FindPatternNode::closing_loc.
Definition ast.h:3928
The elements of a HashNode or KeywordHashNode node that is still being parsed.
Definition prism.c:4414
PM_NODE_ALIGNAS struct pm_node * constant
HashPatternNode::constant.
Definition ast.h:4469
pm_location_t opening_loc
HashPatternNode::opening_loc.
Definition ast.h:4508
pm_location_t closing_loc
HashPatternNode::closing_loc.
Definition ast.h:4521
A Ruby keyword, along with the lex state that it transitions to and the token types that it produces.
Definition prism.c:8351
uint32_t * offsets
The list of offsets.
size_t size
The number of offsets in the list.
uint32_t depth
LocalVariableReadNode::depth.
Definition ast.h:5760
pm_constant_id_t name
LocalVariableReadNode::name.
Definition ast.h:5747
This struct represents a slice in the source code, defined by an offset and a length.
Definition ast.h:579
uint32_t start
The offset of the location from the start of the source.
Definition ast.h:581
uint32_t length
The length of the location.
Definition ast.h:584
pm_location_t lparen_loc
MultiTargetNode::lparen_loc.
Definition ast.h:6187
pm_location_t rparen_loc
MultiTargetNode::rparen_loc.
Definition ast.h:6197
The targets of a MultiTargetNode node that is still being parsed, in source order.
Definition prism.c:5487
uint32_t size
The number of nodes in the list.
Definition ast.h:598
struct pm_node ** nodes
The nodes in the list.
Definition ast.h:601
pm_node_type_t type
This represents the type of the node.
Definition ast.h:1096
pm_location_t location
This is the location of the node in the source.
Definition ast.h:1116
The parameters of a ParametersNode node that is still being parsed.
Definition prism.c:5753
The token buffer for the content of a regular expression.
Definition prism.c:9898
pm_regexp_offsets_t offsets
The places recorded so far.
Definition prism.c:9903
pm_token_buffer_t base
The buffer of the content.
Definition prism.c:9900
PM_NODE_ALIGNAS struct pm_rescue_node * subsequent
RescueNode::subsequent.
Definition ast.h:7172
PM_NODE_ALIGNAS struct pm_node * expression
SplatNode::expression.
Definition ast.h:7437
struct pm_node_list body
StatementsNode::body.
Definition ast.h:7462
pm_node_t base
The embedded base node.
Definition ast.h:7457
The body of a StatementsNode node that is still being parsed.
Definition prism.c:6315
pm_string_t unescaped
StringNode::unescaped.
Definition ast.h:7511
pm_location_t content_loc
StringNode::content_loc.
Definition ast.h:7501
A view of a sequence of bytes.
Definition stringy.h:23
size_t length
The length of the string in bytes of memory.
Definition stringy.h:28
pm_location_t lparen_loc
SuperNode::lparen_loc.
Definition ast.h:7543
PM_NODE_ALIGNAS struct pm_node * block
SuperNode::block.
Definition ast.h:7560
pm_location_t content_loc
SymbolNode::content_loc.
Definition ast.h:7595
pm_string_t unescaped
SymbolNode::unescaped.
Definition ast.h:7605
When we're lexing certain types (strings, symbols, lists, etc.) we have string content associated wit...
Definition prism.c:9780
pm_buffer_t buffer
The buffer that we're using to keep track of the string content.
Definition prism.c:9785
const uint8_t * cursor
The cursor into the source string that points to how far we have currently copied into the buffer.
Definition prism.c:9791
This struct represents a token in the Ruby source.
Definition ast.h:551
const uint8_t * end
A pointer to the end location of the token in the source.
Definition ast.h:559
const uint8_t * start
A pointer to the start location of the token in the source.
Definition ast.h:556
pm_token_type_t type
The type of the token.
Definition ast.h:553
#define PRISM_UNUSED
GCC will warn if you specify a function or parameter that is unused at runtime.
Definition unused.h:13