Ruby 4.1.0dev (2026-11-19 revision e1d5a61acb9109f023398f26c68e447f69e310d3)
parser.h
1#ifndef PRISM_INTERNAL_PARSER_H
2#define PRISM_INTERNAL_PARSER_H
3
7
8#include "prism/internal/arena.h"
9#include "prism/internal/comments.h"
10#include "prism/internal/constant_pool.h"
11#include "prism/internal/encoding.h"
12#include "prism/internal/list.h"
13#include "prism/internal/node.h"
14#include "prism/internal/options.h"
15#include "prism/internal/static_literals.h"
16#include "prism/internal/strpbrk.h"
17
18#include "prism/ast.h"
20#include "prism/parser.h"
21
22#include <stdbool.h>
23#include <stddef.h>
24#include <stdint.h>
25
26/*
27 * A place in the unescaped content of a regular expression where it stops
28 * lining up with the source, because the lexer dropped or rewrote an escape
29 * just before it. From here up to the next such place, each byte of the content
30 * is the byte at the same distance into the source.
31 */
32typedef struct {
33 /* The offset of the place in the unescaped content. */
34 uint32_t content;
35
36 /* The offset in the source of the byte at that place. */
37 uint32_t source;
39
40/*
41 * The places where the unescaped content of a regular expression stops lining
42 * up with the source, in order. Before the first one, each byte of the content
43 * is the byte at the same distance from the start of the content in the source.
44 * The values are allocated in the scratch arena, since they are only read while
45 * the regular expression is validated.
46 */
47typedef struct {
48 /* The places, in order of their offsets in the content. */
49 pm_regexp_offset_t *values;
50
51 /* The number of places. */
52 size_t size;
53
54 /* The number of places that fit in the values before they must grow. */
55 size_t capacity;
57
58/*
59 * This enum provides various bits that represent different kinds of states that
60 * the lexer can track. This is used to determine which kind of token to return
61 * based on the context of the parser.
62 */
63typedef enum {
64 PM_LEX_STATE_BIT_BEG,
65 PM_LEX_STATE_BIT_END,
66 PM_LEX_STATE_BIT_ENDARG,
67 PM_LEX_STATE_BIT_ENDFN,
68 PM_LEX_STATE_BIT_ARG,
69 PM_LEX_STATE_BIT_CMDARG,
70 PM_LEX_STATE_BIT_MID,
71 PM_LEX_STATE_BIT_FNAME,
72 PM_LEX_STATE_BIT_DOT,
73 PM_LEX_STATE_BIT_CLASS,
74 PM_LEX_STATE_BIT_LABEL,
75 PM_LEX_STATE_BIT_LABELED,
76 PM_LEX_STATE_BIT_FITEM
77} pm_lex_state_bit_t;
78
79/*
80 * This enum combines the various bits from the above enum into individual
81 * values that represent the various states of the lexer.
82 */
83typedef enum {
84 PM_LEX_STATE_NONE = 0,
85 PM_LEX_STATE_BEG = (1 << PM_LEX_STATE_BIT_BEG),
86 PM_LEX_STATE_END = (1 << PM_LEX_STATE_BIT_END),
87 PM_LEX_STATE_ENDARG = (1 << PM_LEX_STATE_BIT_ENDARG),
88 PM_LEX_STATE_ENDFN = (1 << PM_LEX_STATE_BIT_ENDFN),
89 PM_LEX_STATE_ARG = (1 << PM_LEX_STATE_BIT_ARG),
90 PM_LEX_STATE_CMDARG = (1 << PM_LEX_STATE_BIT_CMDARG),
91 PM_LEX_STATE_MID = (1 << PM_LEX_STATE_BIT_MID),
92 PM_LEX_STATE_FNAME = (1 << PM_LEX_STATE_BIT_FNAME),
93 PM_LEX_STATE_DOT = (1 << PM_LEX_STATE_BIT_DOT),
94 PM_LEX_STATE_CLASS = (1 << PM_LEX_STATE_BIT_CLASS),
95 PM_LEX_STATE_LABEL = (1 << PM_LEX_STATE_BIT_LABEL),
96 PM_LEX_STATE_LABELED = (1 << PM_LEX_STATE_BIT_LABELED),
97 PM_LEX_STATE_FITEM = (1 << PM_LEX_STATE_BIT_FITEM),
98 PM_LEX_STATE_BEG_ANY = PM_LEX_STATE_BEG | PM_LEX_STATE_MID | PM_LEX_STATE_CLASS,
99 PM_LEX_STATE_ARG_ANY = PM_LEX_STATE_ARG | PM_LEX_STATE_CMDARG,
100 PM_LEX_STATE_END_ANY = PM_LEX_STATE_END | PM_LEX_STATE_ENDARG | PM_LEX_STATE_ENDFN
101} pm_lex_state_t;
102
103/*
104 * The type of quote that a heredoc uses.
105 */
106typedef enum {
107 PM_HEREDOC_QUOTE_NONE,
108 PM_HEREDOC_QUOTE_SINGLE = '\'',
109 PM_HEREDOC_QUOTE_DOUBLE = '"',
110 PM_HEREDOC_QUOTE_BACKTICK = '`',
111} pm_heredoc_quote_t;
112
113/*
114 * The type of indentation that a heredoc uses.
115 */
116typedef enum {
117 PM_HEREDOC_INDENT_NONE,
118 PM_HEREDOC_INDENT_DASH,
119 PM_HEREDOC_INDENT_TILDE,
120} pm_heredoc_indent_t;
121
122/*
123 * All of the information necessary to store to lexing a heredoc.
124 */
125typedef struct {
126 /* A pointer to the start of the heredoc identifier. */
127 const uint8_t *ident_start;
128
129 /* The length of the heredoc identifier. */
130 size_t ident_length;
131
132 /* The type of quote that the heredoc uses. */
133 pm_heredoc_quote_t quote;
134
135 /* The type of indentation that the heredoc uses. */
136 pm_heredoc_indent_t indent;
138
139/*
140 * When lexing Ruby source, the lexer has a small amount of state to tell which
141 * kind of token it is currently lexing. For example, when we find the start of
142 * a string, the first token that we return is a TOKEN_STRING_BEGIN token. After
143 * that the lexer is now in the PM_LEX_STRING mode, and will return tokens that
144 * are found as part of a string.
145 */
146typedef struct pm_lex_mode {
147 /* The type of this lex mode. */
148 enum {
149 /* This state is used when any given token is being lexed. */
150 PM_LEX_DEFAULT,
151
152 /*
153 * This state is used when we're lexing as normal but inside an embedded
154 * expression of a string.
155 */
156 PM_LEX_EMBEXPR,
157
158 /*
159 * This state is used when we're lexing a variable that is embedded
160 * directly inside of a string with the # shorthand.
161 */
162 PM_LEX_EMBVAR,
163
164 /* This state is used when you are inside the content of a heredoc. */
165 PM_LEX_HEREDOC,
166
167 /*
168 * This state is used when we are lexing a list of tokens, as in a %w
169 * word list literal or a %i symbol list literal.
170 */
171 PM_LEX_LIST,
172
173 /*
174 * This state is used when a regular expression has been begun and we
175 * are looking for the terminator.
176 */
177 PM_LEX_REGEXP,
178
179 /*
180 * This state is used when we are lexing a string or a string-like
181 * token, as in string content with either quote or an xstring.
182 */
183 PM_LEX_STRING
184 } mode;
185
186 /* The data associated with this type of lex mode. */
187 union {
188 struct {
189 /* This keeps track of the nesting level of the list. */
190 size_t nesting;
191
192 /* Whether or not interpolation is allowed in this list. */
193 bool interpolation;
194
195 /*
196 * Whether any token has been emitted from this list. A word
197 * separator delimits the opener from the first word, so one
198 * without source characters is emitted when the list does not
199 * start with whitespace.
200 */
201 bool started;
202
203 /*
204 * Whether the previously emitted token was a word separator. A
205 * word separator delimits the last word from the terminator, so
206 * one without source characters is emitted when the list does
207 * not end with whitespace.
208 */
209 bool separated;
210
211 /*
212 * When lexing a list, it takes into account balancing the
213 * terminator if the terminator is one of (), [], {}, or <>.
214 */
215 uint8_t incrementor;
216
217 /* This is the terminator of the list literal. */
218 uint8_t terminator;
219
220 /*
221 * This is the character set that should be used to delimit the
222 * tokens within the list.
223 */
224 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
225 } list;
226
227 struct {
228 /*
229 * This keeps track of the nesting level of the regular expression.
230 */
231 size_t nesting;
232
233 /*
234 * When lexing a regular expression, it takes into account balancing
235 * the terminator if the terminator is one of (), [], {}, or <>.
236 */
237 uint8_t incrementor;
238
239 /* This is the terminator of the regular expression. */
240 uint8_t terminator;
241
242 /*
243 * This is the character set that should be used to delimit the
244 * tokens within the regular expression.
245 */
246 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
247 } regexp;
248
249 struct {
250 /* This keeps track of the nesting level of the string. */
251 size_t nesting;
252
253 /* Whether or not interpolation is allowed in this string. */
254 bool interpolation;
255
256 /*
257 * Whether or not at the end of the string we should allow a :,
258 * which would indicate this was a dynamic symbol instead of a
259 * string.
260 */
261 bool label_allowed;
262
263 /* Whether the final content token joined text after the last
264 * breakpoint, where this string used to split at EOF. */
265 bool joined_at_eof;
266
267 /*
268 * When lexing a string, it takes into account balancing the
269 * terminator if the terminator is one of (), [], {}, or <>.
270 */
271 uint8_t incrementor;
272
273 /*
274 * This is the terminator of the string. It is typically either a
275 * single or double quote.
276 */
277 uint8_t terminator;
278
279 /*
280 * This is the character set that should be used to delimit the
281 * tokens within the string.
282 */
283 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
284 } string;
285
286 struct {
287 /*
288 * All of the data necessary to lex a heredoc.
289 */
291
292 /*
293 * This is the pointer to the character where lexing should resume
294 * once the heredoc has been completely processed.
295 */
296 const uint8_t *next_start;
297
298 /*
299 * This is used to track the amount of common whitespace on each
300 * line so that we know how much to dedent each line in the case of
301 * a tilde heredoc.
302 */
303 size_t *common_whitespace;
304
305 /* True if the previous token ended with a line continuation. */
306 bool line_continuation;
307 } heredoc;
308 } as;
309
310 /* The previous lex state so that it knows how to pop. */
311 struct pm_lex_mode *prev;
312} pm_lex_mode_t;
313
314/*
315 * While parsing, we keep track of a stack of contexts. This is helpful for
316 * error recovery so that we can pop back to a previous context when we hit a
317 * token that is understood by a parent context but not by the current context.
318 */
319typedef enum {
320 /* a null context, used for returning a value from a function */
321 PM_CONTEXT_NONE = 0,
322
323 /* a begin statement */
324 PM_CONTEXT_BEGIN,
325
326 /* an ensure statement with an explicit begin */
327 PM_CONTEXT_BEGIN_ENSURE,
328
329 /* a rescue else statement with an explicit begin */
330 PM_CONTEXT_BEGIN_ELSE,
331
332 /* a rescue statement with an explicit begin */
333 PM_CONTEXT_BEGIN_RESCUE,
334
335 /* expressions in block arguments using braces */
336 PM_CONTEXT_BLOCK_BRACES,
337
338 /* expressions in block arguments using do..end */
339 PM_CONTEXT_BLOCK_KEYWORDS,
340
341 /* an ensure statement within a do..end block */
342 PM_CONTEXT_BLOCK_ENSURE,
343
344 /* a rescue else statement within a do..end block */
345 PM_CONTEXT_BLOCK_ELSE,
346
347 /* expressions in block parameters `foo do |...| end ` */
348 PM_CONTEXT_BLOCK_PARAMETERS,
349
350 /* a rescue statement within a do..end block */
351 PM_CONTEXT_BLOCK_RESCUE,
352
353 /* a case when statements */
354 PM_CONTEXT_CASE_WHEN,
355
356 /* a case in statements */
357 PM_CONTEXT_CASE_IN,
358
359 /* a class declaration */
360 PM_CONTEXT_CLASS,
361
362 /* an ensure statement within a class statement */
363 PM_CONTEXT_CLASS_ENSURE,
364
365 /* a rescue else statement within a class statement */
366 PM_CONTEXT_CLASS_ELSE,
367
368 /* a rescue statement within a class statement */
369 PM_CONTEXT_CLASS_RESCUE,
370
371 /* a method definition */
372 PM_CONTEXT_DEF,
373
374 /* an ensure statement within a method definition */
375 PM_CONTEXT_DEF_ENSURE,
376
377 /* a rescue else statement within a method definition */
378 PM_CONTEXT_DEF_ELSE,
379
380 /* a rescue statement within a method definition */
381 PM_CONTEXT_DEF_RESCUE,
382
383 /* a method definition's parameters */
384 PM_CONTEXT_DEF_PARAMS,
385
386 /* a defined? expression */
387 PM_CONTEXT_DEFINED,
388
389 /* a method definition's default parameter */
390 PM_CONTEXT_DEFAULT_PARAMS,
391
392 /* an else clause */
393 PM_CONTEXT_ELSE,
394
395 /* an elsif clause */
396 PM_CONTEXT_ELSIF,
397
398 /* an interpolated expression */
399 PM_CONTEXT_EMBEXPR,
400
401 /* a for loop */
402 PM_CONTEXT_FOR,
403
404 /* a for loop's index */
405 PM_CONTEXT_FOR_INDEX,
406
407 /* an if statement */
408 PM_CONTEXT_IF,
409
410 /* a lambda expression with braces */
411 PM_CONTEXT_LAMBDA_BRACES,
412
413 /* a lambda expression with do..end */
414 PM_CONTEXT_LAMBDA_DO_END,
415
416 /* an ensure statement within a lambda expression */
417 PM_CONTEXT_LAMBDA_ENSURE,
418
419 /* a rescue else statement within a lambda expression */
420 PM_CONTEXT_LAMBDA_ELSE,
421
422 /* a rescue statement within a lambda expression */
423 PM_CONTEXT_LAMBDA_RESCUE,
424
425 /* the predicate clause of a loop statement */
426 PM_CONTEXT_LOOP_PREDICATE,
427
428 /* the top level context */
429 PM_CONTEXT_MAIN,
430
431 /* a module declaration */
432 PM_CONTEXT_MODULE,
433
434 /* an ensure statement within a module statement */
435 PM_CONTEXT_MODULE_ENSURE,
436
437 /* a rescue else statement within a module statement */
438 PM_CONTEXT_MODULE_ELSE,
439
440 /* a rescue statement within a module statement */
441 PM_CONTEXT_MODULE_RESCUE,
442
443 /* a multiple target expression */
444 PM_CONTEXT_MULTI_TARGET,
445
446 /* a parenthesized expression */
447 PM_CONTEXT_PARENS,
448
449 /* an END block */
450 PM_CONTEXT_POSTEXE,
451
452 /* a predicate inside an if/elsif/unless statement */
453 PM_CONTEXT_PREDICATE,
454
455 /* a BEGIN block */
456 PM_CONTEXT_PREEXE,
457
458 /* a modifier rescue clause */
459 PM_CONTEXT_RESCUE_MODIFIER,
460
461 /* a singleton class definition */
462 PM_CONTEXT_SCLASS,
463
464 /* an ensure statement with a singleton class */
465 PM_CONTEXT_SCLASS_ENSURE,
466
467 /* a rescue else statement with a singleton class */
468 PM_CONTEXT_SCLASS_ELSE,
469
470 /* a rescue statement with a singleton class */
471 PM_CONTEXT_SCLASS_RESCUE,
472
473 /* a ternary expression */
474 PM_CONTEXT_TERNARY,
475
476 /* an unless statement */
477 PM_CONTEXT_UNLESS,
478
479 /* an until statement */
480 PM_CONTEXT_UNTIL,
481
482 /* a while statement */
483 PM_CONTEXT_WHILE,
484
485 /* the number of contexts, which is not itself a context */
486 PM_CONTEXT_MAXIMUM,
487} pm_context_t;
488
489/* This is a node in a linked list of contexts. */
490typedef struct pm_context_node {
491 /* The context that this node represents. */
492 pm_context_t context;
493
494 /* A pointer to the previous context in the linked list. */
495 struct pm_context_node *prev;
496
497 /* One bit set per context in this list, including one for this node. */
498 uint64_t mask;
499} pm_context_node_t;
500
501/* The type of shareable constant value that can be set. */
502typedef uint8_t pm_shareable_constant_value_t;
503static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_NONE = 0x0;
504static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_LITERAL = PM_SHAREABLE_CONSTANT_NODE_FLAGS_LITERAL;
505static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_EVERYTHING;
506static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY;
507
508/*
509 * This tracks an individual local variable in a certain lexical context, as
510 * well as the number of times is it read. The locals of every live scope are
511 * kept on a single stack on the parser, since a local is only ever declared in
512 * the innermost scope and is discarded when that scope is popped.
513 */
514typedef struct {
515 /* The name of the local variable. */
516 pm_constant_id_t name;
517
518 /* The location of the local variable in the source. */
519 pm_location_t location;
520
521 /* The number of times the local variable is read. */
522 uint32_t reads;
523
524 /* The level of the scope that declares the local variable. */
525 uint32_t level;
526
527 /*
528 * The index in the locals stack of the local with the same name that this
529 * local hides while its scope is live, or 0 if it hides nothing. It is
530 * restored as the binding of the name when the scope is popped.
531 */
532 uint32_t shadowed;
533} pm_local_t;
534
535/* The flags about scope parameters that can be set. */
536typedef uint8_t pm_scope_parameters_t;
537static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NONE = 0x0;
538static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x1;
539static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x2;
540static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x4;
541static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x8;
542static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED = 0x10;
543static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_INNER = 0x20;
544static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_FOUND = 0x40;
545
546/*
547 * This struct represents a node in a linked list of scopes. Some scopes can see
548 * into their parent scopes, while others cannot.
549 */
550typedef struct pm_scope {
551 /* A pointer to the previous scope in the linked list. */
552 struct pm_scope *previous;
553
554 /*
555 * The index in the locals stack of the first local of this scope. The
556 * locals of this scope run from here to the top of the stack.
557 */
558 uint32_t locals_start;
559
560 /*
561 * The position of this scope in the scope stack, where the outermost scope
562 * is at level 1.
563 */
564 uint32_t level;
565
566 /*
567 * The locals_start of the nearest closed scope at or above this one. Every
568 * local below this index in the locals stack belongs to a scope that this
569 * scope cannot see into.
570 */
571 uint32_t floor;
572
573 /*
574 * This is a list of the implicit parameters contained within the block.
575 * These will be processed after the block is parsed to determine the kind
576 * of parameters node that should be used and to check if any errors need to
577 * be added.
578 */
579 pm_scratch_node_list_t implicit_parameters;
580
581 /*
582 * This is a bitfield that indicates the parameters that are being used in
583 * this scope. It is a combination of the PM_SCOPE_PARAMETERS_* constants.
584 * There are three different kinds of parameters that can be used in a
585 * scope:
586 *
587 * - Ordinary parameters (e.g., def foo(bar); end)
588 * - Numbered parameters (e.g., def foo; _1; end)
589 * - The it parameter (e.g., def foo; it; end)
590 *
591 * If ordinary parameters are being used, then certain parameters can be
592 * forwarded to another method/structure. Those are indicated by four
593 * additional bits in the params field. For example, some combinations of:
594 *
595 * - def foo(*); end
596 * - def foo(**); end
597 * - def foo(&); end
598 * - def foo(...); end
599 */
600 pm_scope_parameters_t parameters;
601
602 /*
603 * The current state of constant shareability for this scope. This is
604 * changed by magic shareable_constant_value comments.
605 */
606 pm_shareable_constant_value_t shareable_constant;
607
608 /*
609 * A boolean indicating whether or not this scope can see into its parent.
610 * If closed is true, then the scope cannot see into its parent.
611 */
612 bool closed;
613} pm_scope_t;
614
615/*
616 * A stack of boolean values with no fixed depth limit. The newest frames live
617 * in an inline word, so pushing and popping are usually a shift and a compare.
618 * When that word fills, it moves to the spill array and the inline word starts
619 * over empty. The spill array is only allocated once the nesting is more than
620 * 64 frames deep.
621 */
622typedef struct {
623 /*
624 * The newest frames, with the top of the stack in bit 0. It holds between
625 * 1 and 64 frames while the stack is not empty, and is 0 when it is.
626 */
627 uint64_t top;
628
629 /*
630 * The number of frames on the whole stack. The inline word holds the
631 * frames above the last multiple of 64, so the spill array holds
632 * (depth - 1) / 64 words.
633 */
634 uint32_t depth;
635
636 /* The number of words the spill array has room for. */
637 uint32_t capacity;
638
639 /*
640 * Full words of 64 older frames each, oldest first, allocated from the
641 * scratch arena.
642 */
643 uint64_t *spill;
645
646/*
647 * This struct represents the overall parser. It contains a reference to the
648 * source file, as well as pointers that indicate where in the source it's
649 * currently parsing. It also contains the most recent and current token that
650 * it's considering.
651 */
653 /* The arena used for all AST-lifetime allocations. Caller-owned. */
654 pm_arena_t *arena;
655
656 /*
657 * The arena used for parser metadata (comments, diagnostics, etc.), which
658 * lives as long as the parser.
659 */
660 pm_arena_t metadata_arena;
661
662 /*
663 * The arena used for state that is only needed while parsing, such as
664 * scopes, locals, and contexts. It is released when pm_parse returns, so
665 * nothing allocated in it can be reachable from the tree or from the
666 * parser's metadata.
667 */
668 pm_arena_t scratch_arena;
669
670 /*
671 * The next node identifier that will be assigned. This is a unique
672 * identifier used to track nodes such that the syntax tree can be dropped
673 * but the node can be found through another parse.
674 */
675 uint32_t node_id;
676
677 /*
678 * A single-entry cache for pm_parser_constant_id_raw. Avoids redundant
679 * constant pool lookups when the same token is resolved multiple times
680 * (e.g., once during lexing for local variable detection, and again
681 * during parsing for node creation).
682 */
683 struct {
684 const uint8_t *start;
685 const uint8_t *end;
687 } constant_cache;
688
689 /* The current state of the lexer. */
690 pm_lex_state_t lex_state;
691
692 /* Tracks the current nesting of (), [], and {}. */
693 int enclosure_nesting;
694
695 /*
696 * Used to temporarily track the nesting of enclosures to determine if a {
697 * is the beginning of a lambda following the parameters of a lambda.
698 */
699 int lambda_enclosure_nesting;
700
701 /*
702 * Used to track the nesting of braces to ensure we get the correct value
703 * when we are interpolating blocks with braces.
704 */
705 int brace_nesting;
706
707 /*
708 * The stack used to determine if a do keyword belongs to the predicate of a
709 * while, until, or for loop.
710 */
711 pm_state_stack_t do_loop_stack;
712
713 /*
714 * The stack used to determine if a do keyword belongs to the beginning of a
715 * block.
716 */
717 pm_state_stack_t accepts_block_stack;
718
719 /* A stack of lex modes, linked through their prev pointers. */
720 struct {
721 /* The current mode of the lexer. */
722 pm_lex_mode_t *current;
723
724 /*
725 * The mode at the bottom of the stack, which is never popped. Every
726 * other mode is allocated from the scratch arena.
727 */
728 pm_lex_mode_t base;
729
730 /*
731 * Lex modes that have been popped, kept for reuse by the next push.
732 * They are allocated from the scratch arena, so they are never freed
733 * individually.
734 */
735 pm_lex_mode_t *free;
736 } lex_modes;
737
738 /*
739 * The chunks that open node lists are built in. They are carved from the
740 * scratch arena, so they are never freed individually.
741 */
742 pm_node_chunks_t node_chunks;
743
744 /* The pointer to the start of the source. */
745 const uint8_t *start;
746
747 /* The pointer to the end of the source. */
748 const uint8_t *end;
749
750 /* The previous token we were considering. */
751 pm_token_t previous;
752
753 /* The current token we're considering. */
754 pm_token_t current;
755
756 /*
757 * This is a special field set on the parser when we need the parser to jump
758 * to a specific location when lexing the next token, as opposed to just
759 * using the end of the previous token. Normally this is NULL.
760 */
761 const uint8_t *next_start;
762
763 /*
764 * This field indicates the end of a heredoc whose identifier was found on
765 * the current line. If another heredoc is found on the same line, then this
766 * will be moved forward to the end of that heredoc. If no heredocs are
767 * found on a line then this is NULL.
768 */
769 const uint8_t *heredoc_end;
770
771 /* The list of comments that have been found while parsing. */
772 pm_list_t comment_list;
773
774 /*
775 * The attachments of comments to the nodes that own them, populated after
776 * parsing when the attach_comments option is set.
777 */
778 pm_comment_attachments_t comment_attachments;
779
780 /* The list of magic comments that have been found while parsing. */
781 pm_list_t magic_comment_list;
782
783 /*
784 * An optional location that represents the location of the __END__ marker
785 * and the rest of the content of the file. This content is loaded into the
786 * DATA constant when the file being parsed is the main file being executed.
787 */
788 pm_location_t data_loc;
789
790 /* The list of warnings that have been found while parsing. */
791 pm_list_t warning_list;
792
793 /* The list of errors that have been found while parsing. */
794 pm_list_t error_list;
795
796 /* The current local scope. */
797 pm_scope_t *current_scope;
798
799 /*
800 * The locals of every live scope, in the order they were declared. Index 0
801 * is never used, so that 0 can mean the absence of a local.
802 */
803 pm_local_t *locals;
804
805 /* The number of slots in use in locals, including the unused slot 0. */
806 uint32_t locals_size;
807
808 /* The number of slots allocated in locals. */
809 uint32_t locals_capacity;
810
811 /*
812 * The index in the locals stack of the innermost live local with each
813 * constant id as its name, or 0 if there is none. A local that is declared
814 * in a scope replaces the binding of its name, and the binding it replaced
815 * is restored when the scope is popped. Constant ids at or beyond the
816 * capacity have never been declared as locals, so they are unbound.
817 */
818 uint32_t *bindings;
819
820 /* The number of slots allocated in bindings. */
821 uint32_t bindings_capacity;
822
823 /*
824 * Scopes that have been popped, kept for reuse by the next push. Scopes
825 * are allocated from the scratch arena, so they are never freed
826 * individually.
827 */
828 pm_scope_t *free_scopes;
829
830 /* The current parsing context. */
831 pm_context_node_t *current_context;
832
833 /*
834 * Context nodes that have been popped, kept for reuse by the next push.
835 * Context nodes are allocated from the scratch arena, so they are never
836 * freed individually.
837 */
838 pm_context_node_t *free_contexts;
839
840 /*
841 * The encoding functions for the current file is attached to the parser as
842 * it's parsing so that it can change with a magic comment.
843 */
844 const pm_encoding_t *encoding;
845
846 /*
847 * When the encoding that is being used to parse the source is changed by
848 * prism, we provide the ability here to call out to a user-defined
849 * function.
850 */
851 pm_encoding_changed_callback_t encoding_changed_callback;
852
853 /*
854 * This pointer indicates where a comment must start if it is to be
855 * considered an encoding comment.
856 */
857 const uint8_t *encoding_comment_start;
858
859 /*
860 * When you are lexing through a file, the lexer needs all of the information
861 * that the parser additionally provides (for example, the local table). So if
862 * you want to properly lex Ruby, you need to actually lex it in the context of
863 * the parser. In order to provide this functionality, we optionally allow a
864 * struct to be attached to the parser that calls back out to a user-provided
865 * callback when each token is lexed.
866 */
867 struct {
868 /*
869 * This is the callback that is called when a token is lexed. It is
870 * passed the opaque data pointer, the parser, and the token that was
871 * lexed.
872 */
873 pm_lex_callback_t callback;
874
875 /*
876 * This opaque pointer is used to provide whatever information the user
877 * deemed necessary to the callback. In our case we use it to pass the
878 * array that the tokens get appended into.
879 */
880 void *data;
881 } lex_callback;
882
883 /*
884 * This is the path of the file being parsed. We use the filepath when
885 * constructing SourceFileNodes.
886 */
887 pm_string_t filepath;
888
889 /*
890 * This constant pool keeps all of the constants defined throughout the file
891 * so that we can reference them later.
892 */
893 pm_constant_pool_t constant_pool;
894
895 /* This is the list of line offsets in the source file. */
896 pm_line_offset_list_t line_offsets;
897
898 /*
899 * State communicated from the lexer to the parser for integer tokens.
900 */
901 struct {
902 /*
903 * A flag indicating the base of the integer (binary, octal, decimal,
904 * hexadecimal). Set during lexing and read during node creation.
905 */
906 pm_node_flags_t base;
907
908 /*
909 * When lexing a decimal integer of at most 19 digits, which always
910 * fits in a uint64_t, or a plain 0, we compute the value during
911 * lexing to avoid re-scanning the digits during parsing. If start is
912 * set, this holds the result for the token that begins there and
913 * pm_integer_parse can be skipped.
914 */
915 uint64_t value;
916
917 /*
918 * The start of the token whose value is held in value, or NULL before
919 * any has been. The parser creates the node for a token only after
920 * lexing the next one, which may be another integer that replaces the
921 * value, so node creation checks that it belongs to the token at hand.
922 */
923 const uint8_t *start;
924 } integer;
925
926 /*
927 * This string is used to pass information from the lexer to the parser. It
928 * is particularly necessary because of escape sequences.
929 */
930 pm_string_t current_string;
931
932 /*
933 * Where current_string stops lining up with the source, when it is the
934 * content of a regular expression.
935 */
936 pm_regexp_offsets_t current_regexp_offsets;
937
938 /* Named captures from the regexp immediately before a pending =~. */
939 const struct pm_regexp_match *regexp_match;
940
941 /*
942 * The line number at the start of the parse. This will be used to offset
943 * the line numbers of all of the locations.
944 */
945 int32_t start_line;
946
947 /*
948 * When a string-like expression is being lexed, any byte or escape sequence
949 * that resolves to a value whose top bit is set (i.e., >= 0x80) will
950 * explicitly set the encoding to the same encoding as the source.
951 * Alternatively, if a unicode escape sequence is used (e.g., \\u{80}) that
952 * resolves to a value whose top bit is set, then the encoding will be
953 * explicitly set to UTF-8.
954 *
955 * The _next_ time this happens, if the encoding that is about to become the
956 * explicitly set encoding does not match the previously set explicit
957 * encoding, a mixed encoding error will be emitted.
958 *
959 * When the expression is finished being lexed, the explicit encoding
960 * controls the encoding of the expression. For the most part this means
961 * that the expression will either be encoded in the source encoding or
962 * UTF-8. This holds for all encodings except US-ASCII. If the source is
963 * US-ASCII and an explicit encoding was set that was _not_ UTF-8, then the
964 * expression will be encoded as ASCII-8BIT.
965 *
966 * Note that if the expression is a list, different elements within the same
967 * list can have different encodings, so this will get reset between each
968 * element. Furthermore all of this only applies to lists that support
969 * interpolation, because otherwise escapes that could change the encoding
970 * are ignored.
971 *
972 * At first glance, it may make more sense for this to live on the lexer
973 * mode, but we need it here to communicate back to the parser for character
974 * literals that do not push a new lexer mode.
975 */
976 const pm_encoding_t *explicit_encoding;
977
978 /*
979 * When parsing block exits (e.g., break, next, redo), we need to validate
980 * that they are in correct contexts. For the most part we can do this by
981 * looking at our parent contexts. However, modifier while and until
982 * expressions can change that context to make block exits valid. In these
983 * cases, we need to keep track of the block exits and then validate them
984 * after the expression has been parsed.
985 *
986 * We use a pointer here because we don't want to keep a whole list attached
987 * since this will only be used in the context of begin/end expressions.
988 */
989 pm_scratch_node_list_t *current_block_exits;
990
991 /*
992 * The address of the C stack when pm_parse was entered. The stack budget
993 * is measured as the distance from this address.
994 */
995 uintptr_t stack_base;
996
997 /*
998 * The number of bytes of C stack below stack_base that the recursive
999 * descent may use before it reports that the stack is exhausted. SIZE_MAX
1000 * when no budget was given, so the check never fires.
1001 */
1002 size_t stack_budget;
1003
1004 /* The version of prism that we should use to parse. */
1005 pm_options_version_t version;
1006
1007 /* The command line flags given from the options. */
1008 uint8_t command_line;
1009
1010 /*
1011 * Whether or not we have found a frozen_string_literal magic comment with
1012 * a true or false value.
1013 * May be:
1014 * - PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
1015 * - PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
1016 * - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
1017 */
1018 int8_t frozen_string_literal;
1019
1020 /*
1021 * Whether or not we are parsing an eval string. This impacts whether or not
1022 * we should evaluate if block exits/yields are valid.
1023 */
1024 bool parsing_eval;
1025
1026 /*
1027 * Whether or not we are parsing a "partial" script, which is a script that
1028 * will be evaluated in the context of another script, so we should not
1029 * check jumps (next/break/etc.) for validity.
1030 */
1031 bool partial_script;
1032
1033 /*
1034 * Whether or not comments should be attached to their owning nodes once
1035 * parsing has finished.
1036 */
1037 bool attach_comments;
1038
1039 /* Whether or not we're at the beginning of a command. */
1040 bool command_start;
1041
1042 /*
1043 * Whether or not we're currently parsing the body of an endless method
1044 * definition. In this context, PM_TOKEN_KEYWORD_DO_BLOCK should not be
1045 * consumed by commands (it should bubble up to the outer context).
1046 */
1047 bool in_endless_def_body;
1048
1049 /* Whether or not we're currently recovering from a syntax error. */
1050 bool recovering;
1051
1052 /*
1053 * Whether or not the source being parsed could become valid if more input
1054 * were appended. This is set to false when the parser encounters a token
1055 * that is definitively wrong (e.g., a stray `end` or `]`) as opposed to
1056 * merely incomplete.
1057 */
1058 bool continuable;
1059
1060 /*
1061 * This is very specialized behavior for when you want to parse in a context
1062 * that does not respect encoding comments. Its main use case is translating
1063 * into the whitequark/parser AST which re-encodes source files in UTF-8
1064 * before they are parsed and ignores encoding comments.
1065 */
1066 bool encoding_locked;
1067
1068 /*
1069 * Whether or not the encoding has been changed by a magic comment. We use
1070 * this to provide a fast path for the lexer instead of going through the
1071 * function pointer.
1072 */
1073 bool encoding_changed;
1074
1075 /*
1076 * This flag indicates that we are currently parsing a pattern matching
1077 * expression and impacts that calculation of newlines.
1078 */
1079 bool pattern_matching_newlines;
1080
1081 /* This flag indicates that we are currently parsing a keyword argument. */
1082 bool in_keyword_arg;
1083
1084 /*
1085 * Whether or not the parser has seen a token that has semantic meaning
1086 * (i.e., a token that is not a comment or whitespace).
1087 */
1088 bool semantic_token_seen;
1089
1090 /*
1091 * By default, Ruby always warns about mismatched indentation. This can be
1092 * toggled with a magic comment.
1093 */
1094 bool warn_mismatched_indentation;
1095
1096 /*
1097 * Whether the stack exhausted error has been reported. Once the budget is
1098 * used up, every recursion that unwinds past it would report the error
1099 * again, so it is only reported the first time.
1100 */
1101 bool stack_exhausted_reported;
1102
1103#if defined(PRISM_HAS_NEON) || defined(PRISM_HAS_SSE2) || defined(PRISM_HAS_SWAR)
1104 /*
1105 * Cached lookup tables for pm_strpbrk's SIMD fast path. Avoids rebuilding
1106 * the nibble-based tables on every call when the charset hasn't changed
1107 * (which is the common case during string/regex/list lexing).
1108 */
1109 struct {
1110 /* The cached charset (null-terminated, max 11 chars + NUL). */
1111 uint8_t charset[12];
1112
1113 /* Nibble-based low lookup table for SIMD matching. */
1114 uint8_t low_lut[16];
1115
1116 /* Nibble-based high lookup table for SIMD matching. */
1117 uint8_t high_lut[16];
1118
1119 /* Scalar fallback table (4 x 64-bit bitmasks covering all ASCII). */
1120 uint64_t table[4];
1121 } strpbrk_cache;
1122#endif
1123};
1124
1125/*
1126 * Initialize a parser with the given start and end pointers.
1127 */
1128void pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options);
1129
1130/*
1131 * Free the memory held by the given parser.
1132 *
1133 * This does not free the `pm_options_t` object that was used to initialize the
1134 * parser.
1135 */
1136void pm_parser_cleanup(pm_parser_t *parser);
1137
1138/*
1139 * Returns true if the bytes between start and end spell a keyword, using the
1140 * same lookup that the lexer does.
1141 */
1142bool pm_keyword_p(const uint8_t *start, const uint8_t *end);
1143
1144/*
1145 * Returns true if parsing has used more C stack since pm_parse was entered than
1146 * the stack budget allows. Every recursion that runs inside pm_parse checks
1147 * this alongside its own depth limit, so that the budget bounds all of them.
1148 * The distance is taken without assuming which direction the stack grows. With
1149 * no budget the limit is SIZE_MAX, so this never fires.
1150 */
1151static PRISM_INLINE bool
1152pm_parser_stack_exhausted(const pm_parser_t *parser) {
1153 uintptr_t current = pm_stack_address();
1154 uintptr_t used = current < parser->stack_base ? parser->stack_base - current : current - parser->stack_base;
1155 return used > parser->stack_budget;
1156}
1157
1158/*
1159 * Reports that the stack budget has been used up, at the given location. This
1160 * is a separate error from the nesting depth limit, with its own error level,
1161 * because the source may be valid and parse with more stack. It is reported
1162 * once per parse.
1163 */
1164void pm_parser_err_stack_exhausted(pm_parser_t *parser, uint32_t start, uint32_t length);
1165
1166#endif
uint32_t pm_constant_id_t
A constant id is a unique identifier for a constant in the constant pool.
#define PRISM_INLINE
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
Definition inline.h:12
A list of byte offsets of newlines in a string.
The parser used to parse Ruby source.
void(* pm_lex_callback_t)(pm_parser_t *parser, pm_token_t *token, void *data)
This is the callback that is called when a token is lexed.
Definition parser.h:59
void(* pm_encoding_changed_callback_t)(pm_parser_t *parser)
When the encoding that is being used to parse the source is changed by prism, we provide the ability ...
Definition parser.h:53
static PRISM_INLINE uintptr_t pm_stack_address(void)
Returns the current position of the C stack, for measuring how much of it has been consumed since an ...
Definition stack.h:34
C99 shim for <stdbool.h>.
A list of offsets of the start of lines in a string.
This struct represents a slice in the source code, defined by an offset and a length.
Definition ast.h:579
A view of a sequence of bytes.
Definition stringy.h:23
This struct represents a token in the Ruby source.
Definition ast.h:551