Ruby 4.1.0dev (2026-11-18 revision bcf3b03b44ae598e3adb59d359cf32202091a318)
parser.h
1#ifndef PRISM_INTERNAL_PARSER_H
2#define PRISM_INTERNAL_PARSER_H
3
7
8#include "prism/internal/arena.h"
9#include "prism/internal/comments.h"
10#include "prism/internal/constant_pool.h"
11#include "prism/internal/encoding.h"
12#include "prism/internal/list.h"
13#include "prism/internal/node.h"
14#include "prism/internal/options.h"
15#include "prism/internal/static_literals.h"
16#include "prism/internal/strpbrk.h"
17
18#include "prism/ast.h"
20#include "prism/parser.h"
21
22#include <stdbool.h>
23#include <stddef.h>
24#include <stdint.h>
25
26/*
27 * This enum provides various bits that represent different kinds of states that
28 * the lexer can track. This is used to determine which kind of token to return
29 * based on the context of the parser.
30 */
31typedef enum {
32 PM_LEX_STATE_BIT_BEG,
33 PM_LEX_STATE_BIT_END,
34 PM_LEX_STATE_BIT_ENDARG,
35 PM_LEX_STATE_BIT_ENDFN,
36 PM_LEX_STATE_BIT_ARG,
37 PM_LEX_STATE_BIT_CMDARG,
38 PM_LEX_STATE_BIT_MID,
39 PM_LEX_STATE_BIT_FNAME,
40 PM_LEX_STATE_BIT_DOT,
41 PM_LEX_STATE_BIT_CLASS,
42 PM_LEX_STATE_BIT_LABEL,
43 PM_LEX_STATE_BIT_LABELED,
44 PM_LEX_STATE_BIT_FITEM
45} pm_lex_state_bit_t;
46
47/*
48 * This enum combines the various bits from the above enum into individual
49 * values that represent the various states of the lexer.
50 */
51typedef enum {
52 PM_LEX_STATE_NONE = 0,
53 PM_LEX_STATE_BEG = (1 << PM_LEX_STATE_BIT_BEG),
54 PM_LEX_STATE_END = (1 << PM_LEX_STATE_BIT_END),
55 PM_LEX_STATE_ENDARG = (1 << PM_LEX_STATE_BIT_ENDARG),
56 PM_LEX_STATE_ENDFN = (1 << PM_LEX_STATE_BIT_ENDFN),
57 PM_LEX_STATE_ARG = (1 << PM_LEX_STATE_BIT_ARG),
58 PM_LEX_STATE_CMDARG = (1 << PM_LEX_STATE_BIT_CMDARG),
59 PM_LEX_STATE_MID = (1 << PM_LEX_STATE_BIT_MID),
60 PM_LEX_STATE_FNAME = (1 << PM_LEX_STATE_BIT_FNAME),
61 PM_LEX_STATE_DOT = (1 << PM_LEX_STATE_BIT_DOT),
62 PM_LEX_STATE_CLASS = (1 << PM_LEX_STATE_BIT_CLASS),
63 PM_LEX_STATE_LABEL = (1 << PM_LEX_STATE_BIT_LABEL),
64 PM_LEX_STATE_LABELED = (1 << PM_LEX_STATE_BIT_LABELED),
65 PM_LEX_STATE_FITEM = (1 << PM_LEX_STATE_BIT_FITEM),
66 PM_LEX_STATE_BEG_ANY = PM_LEX_STATE_BEG | PM_LEX_STATE_MID | PM_LEX_STATE_CLASS,
67 PM_LEX_STATE_ARG_ANY = PM_LEX_STATE_ARG | PM_LEX_STATE_CMDARG,
68 PM_LEX_STATE_END_ANY = PM_LEX_STATE_END | PM_LEX_STATE_ENDARG | PM_LEX_STATE_ENDFN
69} pm_lex_state_t;
70
71/*
72 * The type of quote that a heredoc uses.
73 */
74typedef enum {
75 PM_HEREDOC_QUOTE_NONE,
76 PM_HEREDOC_QUOTE_SINGLE = '\'',
77 PM_HEREDOC_QUOTE_DOUBLE = '"',
78 PM_HEREDOC_QUOTE_BACKTICK = '`',
79} pm_heredoc_quote_t;
80
81/*
82 * The type of indentation that a heredoc uses.
83 */
84typedef enum {
85 PM_HEREDOC_INDENT_NONE,
86 PM_HEREDOC_INDENT_DASH,
87 PM_HEREDOC_INDENT_TILDE,
88} pm_heredoc_indent_t;
89
90/*
91 * All of the information necessary to store to lexing a heredoc.
92 */
93typedef struct {
94 /* A pointer to the start of the heredoc identifier. */
95 const uint8_t *ident_start;
96
97 /* The length of the heredoc identifier. */
98 size_t ident_length;
99
100 /* The type of quote that the heredoc uses. */
101 pm_heredoc_quote_t quote;
102
103 /* The type of indentation that the heredoc uses. */
104 pm_heredoc_indent_t indent;
106
107/*
108 * When lexing Ruby source, the lexer has a small amount of state to tell which
109 * kind of token it is currently lexing. For example, when we find the start of
110 * a string, the first token that we return is a TOKEN_STRING_BEGIN token. After
111 * that the lexer is now in the PM_LEX_STRING mode, and will return tokens that
112 * are found as part of a string.
113 */
114typedef struct pm_lex_mode {
115 /* The type of this lex mode. */
116 enum {
117 /* This state is used when any given token is being lexed. */
118 PM_LEX_DEFAULT,
119
120 /*
121 * This state is used when we're lexing as normal but inside an embedded
122 * expression of a string.
123 */
124 PM_LEX_EMBEXPR,
125
126 /*
127 * This state is used when we're lexing a variable that is embedded
128 * directly inside of a string with the # shorthand.
129 */
130 PM_LEX_EMBVAR,
131
132 /* This state is used when you are inside the content of a heredoc. */
133 PM_LEX_HEREDOC,
134
135 /*
136 * This state is used when we are lexing a list of tokens, as in a %w
137 * word list literal or a %i symbol list literal.
138 */
139 PM_LEX_LIST,
140
141 /*
142 * This state is used when a regular expression has been begun and we
143 * are looking for the terminator.
144 */
145 PM_LEX_REGEXP,
146
147 /*
148 * This state is used when we are lexing a string or a string-like
149 * token, as in string content with either quote or an xstring.
150 */
151 PM_LEX_STRING
152 } mode;
153
154 /* The data associated with this type of lex mode. */
155 union {
156 struct {
157 /* This keeps track of the nesting level of the list. */
158 size_t nesting;
159
160 /* Whether or not interpolation is allowed in this list. */
161 bool interpolation;
162
163 /*
164 * Whether any token has been emitted from this list. A word
165 * separator delimits the opener from the first word, so one
166 * without source characters is emitted when the list does not
167 * start with whitespace.
168 */
169 bool started;
170
171 /*
172 * Whether the previously emitted token was a word separator. A
173 * word separator delimits the last word from the terminator, so
174 * one without source characters is emitted when the list does
175 * not end with whitespace.
176 */
177 bool separated;
178
179 /*
180 * When lexing a list, it takes into account balancing the
181 * terminator if the terminator is one of (), [], {}, or <>.
182 */
183 uint8_t incrementor;
184
185 /* This is the terminator of the list literal. */
186 uint8_t terminator;
187
188 /*
189 * This is the character set that should be used to delimit the
190 * tokens within the list.
191 */
192 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
193 } list;
194
195 struct {
196 /*
197 * This keeps track of the nesting level of the regular expression.
198 */
199 size_t nesting;
200
201 /*
202 * When lexing a regular expression, it takes into account balancing
203 * the terminator if the terminator is one of (), [], {}, or <>.
204 */
205 uint8_t incrementor;
206
207 /* This is the terminator of the regular expression. */
208 uint8_t terminator;
209
210 /*
211 * This is the character set that should be used to delimit the
212 * tokens within the regular expression.
213 */
214 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
215 } regexp;
216
217 struct {
218 /* This keeps track of the nesting level of the string. */
219 size_t nesting;
220
221 /* Whether or not interpolation is allowed in this string. */
222 bool interpolation;
223
224 /*
225 * Whether or not at the end of the string we should allow a :,
226 * which would indicate this was a dynamic symbol instead of a
227 * string.
228 */
229 bool label_allowed;
230
231 /*
232 * When lexing a string, it takes into account balancing the
233 * terminator if the terminator is one of (), [], {}, or <>.
234 */
235 uint8_t incrementor;
236
237 /*
238 * This is the terminator of the string. It is typically either a
239 * single or double quote.
240 */
241 uint8_t terminator;
242
243 /*
244 * This is the character set that should be used to delimit the
245 * tokens within the string.
246 */
247 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
248 } string;
249
250 struct {
251 /*
252 * All of the data necessary to lex a heredoc.
253 */
255
256 /*
257 * This is the pointer to the character where lexing should resume
258 * once the heredoc has been completely processed.
259 */
260 const uint8_t *next_start;
261
262 /*
263 * This is used to track the amount of common whitespace on each
264 * line so that we know how much to dedent each line in the case of
265 * a tilde heredoc.
266 */
267 size_t *common_whitespace;
268
269 /* True if the previous token ended with a line continuation. */
270 bool line_continuation;
271 } heredoc;
272 } as;
273
274 /* The previous lex state so that it knows how to pop. */
275 struct pm_lex_mode *prev;
276} pm_lex_mode_t;
277
278/*
279 * While parsing, we keep track of a stack of contexts. This is helpful for
280 * error recovery so that we can pop back to a previous context when we hit a
281 * token that is understood by a parent context but not by the current context.
282 */
283typedef enum {
284 /* a null context, used for returning a value from a function */
285 PM_CONTEXT_NONE = 0,
286
287 /* a begin statement */
288 PM_CONTEXT_BEGIN,
289
290 /* an ensure statement with an explicit begin */
291 PM_CONTEXT_BEGIN_ENSURE,
292
293 /* a rescue else statement with an explicit begin */
294 PM_CONTEXT_BEGIN_ELSE,
295
296 /* a rescue statement with an explicit begin */
297 PM_CONTEXT_BEGIN_RESCUE,
298
299 /* expressions in block arguments using braces */
300 PM_CONTEXT_BLOCK_BRACES,
301
302 /* expressions in block arguments using do..end */
303 PM_CONTEXT_BLOCK_KEYWORDS,
304
305 /* an ensure statement within a do..end block */
306 PM_CONTEXT_BLOCK_ENSURE,
307
308 /* a rescue else statement within a do..end block */
309 PM_CONTEXT_BLOCK_ELSE,
310
311 /* expressions in block parameters `foo do |...| end ` */
312 PM_CONTEXT_BLOCK_PARAMETERS,
313
314 /* a rescue statement within a do..end block */
315 PM_CONTEXT_BLOCK_RESCUE,
316
317 /* a case when statements */
318 PM_CONTEXT_CASE_WHEN,
319
320 /* a case in statements */
321 PM_CONTEXT_CASE_IN,
322
323 /* a class declaration */
324 PM_CONTEXT_CLASS,
325
326 /* an ensure statement within a class statement */
327 PM_CONTEXT_CLASS_ENSURE,
328
329 /* a rescue else statement within a class statement */
330 PM_CONTEXT_CLASS_ELSE,
331
332 /* a rescue statement within a class statement */
333 PM_CONTEXT_CLASS_RESCUE,
334
335 /* a method definition */
336 PM_CONTEXT_DEF,
337
338 /* an ensure statement within a method definition */
339 PM_CONTEXT_DEF_ENSURE,
340
341 /* a rescue else statement within a method definition */
342 PM_CONTEXT_DEF_ELSE,
343
344 /* a rescue statement within a method definition */
345 PM_CONTEXT_DEF_RESCUE,
346
347 /* a method definition's parameters */
348 PM_CONTEXT_DEF_PARAMS,
349
350 /* a defined? expression */
351 PM_CONTEXT_DEFINED,
352
353 /* a method definition's default parameter */
354 PM_CONTEXT_DEFAULT_PARAMS,
355
356 /* an else clause */
357 PM_CONTEXT_ELSE,
358
359 /* an elsif clause */
360 PM_CONTEXT_ELSIF,
361
362 /* an interpolated expression */
363 PM_CONTEXT_EMBEXPR,
364
365 /* a for loop */
366 PM_CONTEXT_FOR,
367
368 /* a for loop's index */
369 PM_CONTEXT_FOR_INDEX,
370
371 /* an if statement */
372 PM_CONTEXT_IF,
373
374 /* a lambda expression with braces */
375 PM_CONTEXT_LAMBDA_BRACES,
376
377 /* a lambda expression with do..end */
378 PM_CONTEXT_LAMBDA_DO_END,
379
380 /* an ensure statement within a lambda expression */
381 PM_CONTEXT_LAMBDA_ENSURE,
382
383 /* a rescue else statement within a lambda expression */
384 PM_CONTEXT_LAMBDA_ELSE,
385
386 /* a rescue statement within a lambda expression */
387 PM_CONTEXT_LAMBDA_RESCUE,
388
389 /* the predicate clause of a loop statement */
390 PM_CONTEXT_LOOP_PREDICATE,
391
392 /* the top level context */
393 PM_CONTEXT_MAIN,
394
395 /* a module declaration */
396 PM_CONTEXT_MODULE,
397
398 /* an ensure statement within a module statement */
399 PM_CONTEXT_MODULE_ENSURE,
400
401 /* a rescue else statement within a module statement */
402 PM_CONTEXT_MODULE_ELSE,
403
404 /* a rescue statement within a module statement */
405 PM_CONTEXT_MODULE_RESCUE,
406
407 /* a multiple target expression */
408 PM_CONTEXT_MULTI_TARGET,
409
410 /* a parenthesized expression */
411 PM_CONTEXT_PARENS,
412
413 /* an END block */
414 PM_CONTEXT_POSTEXE,
415
416 /* a predicate inside an if/elsif/unless statement */
417 PM_CONTEXT_PREDICATE,
418
419 /* a BEGIN block */
420 PM_CONTEXT_PREEXE,
421
422 /* a modifier rescue clause */
423 PM_CONTEXT_RESCUE_MODIFIER,
424
425 /* a singleton class definition */
426 PM_CONTEXT_SCLASS,
427
428 /* an ensure statement with a singleton class */
429 PM_CONTEXT_SCLASS_ENSURE,
430
431 /* a rescue else statement with a singleton class */
432 PM_CONTEXT_SCLASS_ELSE,
433
434 /* a rescue statement with a singleton class */
435 PM_CONTEXT_SCLASS_RESCUE,
436
437 /* a ternary expression */
438 PM_CONTEXT_TERNARY,
439
440 /* an unless statement */
441 PM_CONTEXT_UNLESS,
442
443 /* an until statement */
444 PM_CONTEXT_UNTIL,
445
446 /* a while statement */
447 PM_CONTEXT_WHILE,
448
449 /* the number of contexts, which is not itself a context */
450 PM_CONTEXT_MAXIMUM,
451} pm_context_t;
452
453/* This is a node in a linked list of contexts. */
454typedef struct pm_context_node {
455 /* The context that this node represents. */
456 pm_context_t context;
457
458 /* A pointer to the previous context in the linked list. */
459 struct pm_context_node *prev;
460
461 /* One bit set per context in this list, including one for this node. */
462 uint64_t mask;
463} pm_context_node_t;
464
465/* The type of shareable constant value that can be set. */
466typedef uint8_t pm_shareable_constant_value_t;
467static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_NONE = 0x0;
468static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_LITERAL = PM_SHAREABLE_CONSTANT_NODE_FLAGS_LITERAL;
469static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_EVERYTHING;
470static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY;
471
472/*
473 * This tracks an individual local variable in a certain lexical context, as
474 * well as the number of times is it read. The locals of every live scope are
475 * kept on a single stack on the parser, since a local is only ever declared in
476 * the innermost scope and is discarded when that scope is popped.
477 */
478typedef struct {
479 /* The name of the local variable. */
480 pm_constant_id_t name;
481
482 /* The location of the local variable in the source. */
483 pm_location_t location;
484
485 /* The number of times the local variable is read. */
486 uint32_t reads;
487
488 /* The level of the scope that declares the local variable. */
489 uint32_t level;
490
491 /*
492 * The index in the locals stack of the local with the same name that this
493 * local hides while its scope is live, or 0 if it hides nothing. It is
494 * restored as the binding of the name when the scope is popped.
495 */
496 uint32_t shadowed;
497} pm_local_t;
498
499/* The flags about scope parameters that can be set. */
500typedef uint8_t pm_scope_parameters_t;
501static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NONE = 0x0;
502static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x1;
503static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x2;
504static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x4;
505static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x8;
506static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED = 0x10;
507static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_INNER = 0x20;
508static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_FOUND = 0x40;
509
510/*
511 * This struct represents a node in a linked list of scopes. Some scopes can see
512 * into their parent scopes, while others cannot.
513 */
514typedef struct pm_scope {
515 /* A pointer to the previous scope in the linked list. */
516 struct pm_scope *previous;
517
518 /*
519 * The index in the locals stack of the first local of this scope. The
520 * locals of this scope run from here to the top of the stack.
521 */
522 uint32_t locals_start;
523
524 /*
525 * The position of this scope in the scope stack, where the outermost scope
526 * is at level 1.
527 */
528 uint32_t level;
529
530 /*
531 * The locals_start of the nearest closed scope at or above this one. Every
532 * local below this index in the locals stack belongs to a scope that this
533 * scope cannot see into.
534 */
535 uint32_t floor;
536
537 /*
538 * This is a list of the implicit parameters contained within the block.
539 * These will be processed after the block is parsed to determine the kind
540 * of parameters node that should be used and to check if any errors need to
541 * be added.
542 */
543 pm_scratch_node_list_t implicit_parameters;
544
545 /*
546 * This is a bitfield that indicates the parameters that are being used in
547 * this scope. It is a combination of the PM_SCOPE_PARAMETERS_* constants.
548 * There are three different kinds of parameters that can be used in a
549 * scope:
550 *
551 * - Ordinary parameters (e.g., def foo(bar); end)
552 * - Numbered parameters (e.g., def foo; _1; end)
553 * - The it parameter (e.g., def foo; it; end)
554 *
555 * If ordinary parameters are being used, then certain parameters can be
556 * forwarded to another method/structure. Those are indicated by four
557 * additional bits in the params field. For example, some combinations of:
558 *
559 * - def foo(*); end
560 * - def foo(**); end
561 * - def foo(&); end
562 * - def foo(...); end
563 */
564 pm_scope_parameters_t parameters;
565
566 /*
567 * The current state of constant shareability for this scope. This is
568 * changed by magic shareable_constant_value comments.
569 */
570 pm_shareable_constant_value_t shareable_constant;
571
572 /*
573 * A boolean indicating whether or not this scope can see into its parent.
574 * If closed is true, then the scope cannot see into its parent.
575 */
576 bool closed;
577} pm_scope_t;
578
579/*
580 * A stack of boolean values with no fixed depth limit. The newest frames live
581 * in an inline word, so pushing and popping are usually a shift and a compare.
582 * When that word fills, it moves to the spill array and the inline word starts
583 * over empty. The spill array is only allocated once the nesting is more than
584 * 64 frames deep.
585 */
586typedef struct {
587 /*
588 * The newest frames, with the top of the stack in bit 0. It holds between
589 * 1 and 64 frames while the stack is not empty, and is 0 when it is.
590 */
591 uint64_t top;
592
593 /*
594 * The number of frames on the whole stack. The inline word holds the
595 * frames above the last multiple of 64, so the spill array holds
596 * (depth - 1) / 64 words.
597 */
598 uint32_t depth;
599
600 /* The number of words the spill array has room for. */
601 uint32_t capacity;
602
603 /*
604 * Full words of 64 older frames each, oldest first, allocated from the
605 * scratch arena.
606 */
607 uint64_t *spill;
609
610/*
611 * This struct represents the overall parser. It contains a reference to the
612 * source file, as well as pointers that indicate where in the source it's
613 * currently parsing. It also contains the most recent and current token that
614 * it's considering.
615 */
617 /* The arena used for all AST-lifetime allocations. Caller-owned. */
618 pm_arena_t *arena;
619
620 /*
621 * The arena used for parser metadata (comments, diagnostics, etc.), which
622 * lives as long as the parser.
623 */
624 pm_arena_t metadata_arena;
625
626 /*
627 * The arena used for state that is only needed while parsing, such as
628 * scopes, locals, and contexts. It is released when pm_parse returns, so
629 * nothing allocated in it can be reachable from the tree or from the
630 * parser's metadata.
631 */
632 pm_arena_t scratch_arena;
633
634 /*
635 * The next node identifier that will be assigned. This is a unique
636 * identifier used to track nodes such that the syntax tree can be dropped
637 * but the node can be found through another parse.
638 */
639 uint32_t node_id;
640
641 /*
642 * A single-entry cache for pm_parser_constant_id_raw. Avoids redundant
643 * constant pool lookups when the same token is resolved multiple times
644 * (e.g., once during lexing for local variable detection, and again
645 * during parsing for node creation).
646 */
647 struct {
648 const uint8_t *start;
649 const uint8_t *end;
651 } constant_cache;
652
653 /* The current state of the lexer. */
654 pm_lex_state_t lex_state;
655
656 /* Tracks the current nesting of (), [], and {}. */
657 int enclosure_nesting;
658
659 /*
660 * Used to temporarily track the nesting of enclosures to determine if a {
661 * is the beginning of a lambda following the parameters of a lambda.
662 */
663 int lambda_enclosure_nesting;
664
665 /*
666 * Used to track the nesting of braces to ensure we get the correct value
667 * when we are interpolating blocks with braces.
668 */
669 int brace_nesting;
670
671 /*
672 * The stack used to determine if a do keyword belongs to the predicate of a
673 * while, until, or for loop.
674 */
675 pm_state_stack_t do_loop_stack;
676
677 /*
678 * The stack used to determine if a do keyword belongs to the beginning of a
679 * block.
680 */
681 pm_state_stack_t accepts_block_stack;
682
683 /* A stack of lex modes, linked through their prev pointers. */
684 struct {
685 /* The current mode of the lexer. */
686 pm_lex_mode_t *current;
687
688 /*
689 * The mode at the bottom of the stack, which is never popped. Every
690 * other mode is allocated from the scratch arena.
691 */
692 pm_lex_mode_t base;
693
694 /*
695 * Lex modes that have been popped, kept for reuse by the next push.
696 * They are allocated from the scratch arena, so they are never freed
697 * individually.
698 */
699 pm_lex_mode_t *free;
700 } lex_modes;
701
702 /*
703 * The chunks that open node lists are built in. They are carved from the
704 * scratch arena, so they are never freed individually.
705 */
706 pm_node_chunks_t node_chunks;
707
708 /* The pointer to the start of the source. */
709 const uint8_t *start;
710
711 /* The pointer to the end of the source. */
712 const uint8_t *end;
713
714 /* The previous token we were considering. */
715 pm_token_t previous;
716
717 /* The current token we're considering. */
718 pm_token_t current;
719
720 /*
721 * This is a special field set on the parser when we need the parser to jump
722 * to a specific location when lexing the next token, as opposed to just
723 * using the end of the previous token. Normally this is NULL.
724 */
725 const uint8_t *next_start;
726
727 /*
728 * This field indicates the end of a heredoc whose identifier was found on
729 * the current line. If another heredoc is found on the same line, then this
730 * will be moved forward to the end of that heredoc. If no heredocs are
731 * found on a line then this is NULL.
732 */
733 const uint8_t *heredoc_end;
734
735 /* The list of comments that have been found while parsing. */
736 pm_list_t comment_list;
737
738 /*
739 * The attachments of comments to the nodes that own them, populated after
740 * parsing when the attach_comments option is set.
741 */
742 pm_comment_attachments_t comment_attachments;
743
744 /* The list of magic comments that have been found while parsing. */
745 pm_list_t magic_comment_list;
746
747 /*
748 * An optional location that represents the location of the __END__ marker
749 * and the rest of the content of the file. This content is loaded into the
750 * DATA constant when the file being parsed is the main file being executed.
751 */
752 pm_location_t data_loc;
753
754 /* The list of warnings that have been found while parsing. */
755 pm_list_t warning_list;
756
757 /* The list of errors that have been found while parsing. */
758 pm_list_t error_list;
759
760 /* The current local scope. */
761 pm_scope_t *current_scope;
762
763 /*
764 * The locals of every live scope, in the order they were declared. Index 0
765 * is never used, so that 0 can mean the absence of a local.
766 */
767 pm_local_t *locals;
768
769 /* The number of slots in use in locals, including the unused slot 0. */
770 uint32_t locals_size;
771
772 /* The number of slots allocated in locals. */
773 uint32_t locals_capacity;
774
775 /*
776 * The index in the locals stack of the innermost live local with each
777 * constant id as its name, or 0 if there is none. A local that is declared
778 * in a scope replaces the binding of its name, and the binding it replaced
779 * is restored when the scope is popped. Constant ids at or beyond the
780 * capacity have never been declared as locals, so they are unbound.
781 */
782 uint32_t *bindings;
783
784 /* The number of slots allocated in bindings. */
785 uint32_t bindings_capacity;
786
787 /*
788 * Scopes that have been popped, kept for reuse by the next push. Scopes
789 * are allocated from the scratch arena, so they are never freed
790 * individually.
791 */
792 pm_scope_t *free_scopes;
793
794 /* The current parsing context. */
795 pm_context_node_t *current_context;
796
797 /*
798 * Context nodes that have been popped, kept for reuse by the next push.
799 * Context nodes are allocated from the scratch arena, so they are never
800 * freed individually.
801 */
802 pm_context_node_t *free_contexts;
803
804 /*
805 * The encoding functions for the current file is attached to the parser as
806 * it's parsing so that it can change with a magic comment.
807 */
808 const pm_encoding_t *encoding;
809
810 /*
811 * When the encoding that is being used to parse the source is changed by
812 * prism, we provide the ability here to call out to a user-defined
813 * function.
814 */
815 pm_encoding_changed_callback_t encoding_changed_callback;
816
817 /*
818 * This pointer indicates where a comment must start if it is to be
819 * considered an encoding comment.
820 */
821 const uint8_t *encoding_comment_start;
822
823 /*
824 * When you are lexing through a file, the lexer needs all of the information
825 * that the parser additionally provides (for example, the local table). So if
826 * you want to properly lex Ruby, you need to actually lex it in the context of
827 * the parser. In order to provide this functionality, we optionally allow a
828 * struct to be attached to the parser that calls back out to a user-provided
829 * callback when each token is lexed.
830 */
831 struct {
832 /*
833 * This is the callback that is called when a token is lexed. It is
834 * passed the opaque data pointer, the parser, and the token that was
835 * lexed.
836 */
837 pm_lex_callback_t callback;
838
839 /*
840 * This opaque pointer is used to provide whatever information the user
841 * deemed necessary to the callback. In our case we use it to pass the
842 * array that the tokens get appended into.
843 */
844 void *data;
845 } lex_callback;
846
847 /*
848 * This is the path of the file being parsed. We use the filepath when
849 * constructing SourceFileNodes.
850 */
851 pm_string_t filepath;
852
853 /*
854 * This constant pool keeps all of the constants defined throughout the file
855 * so that we can reference them later.
856 */
857 pm_constant_pool_t constant_pool;
858
859 /* This is the list of line offsets in the source file. */
860 pm_line_offset_list_t line_offsets;
861
862 /*
863 * State communicated from the lexer to the parser for integer tokens.
864 */
865 struct {
866 /*
867 * A flag indicating the base of the integer (binary, octal, decimal,
868 * hexadecimal). Set during lexing and read during node creation.
869 */
870 pm_node_flags_t base;
871
872 /*
873 * When lexing a decimal integer of at most 19 digits, which always
874 * fits in a uint64_t, or a plain 0, we compute the value during
875 * lexing to avoid re-scanning the digits during parsing. If start is
876 * set, this holds the result for the token that begins there and
877 * pm_integer_parse can be skipped.
878 */
879 uint64_t value;
880
881 /*
882 * The start of the token whose value is held in value, or NULL before
883 * any has been. The parser creates the node for a token only after
884 * lexing the next one, which may be another integer that replaces the
885 * value, so node creation checks that it belongs to the token at hand.
886 */
887 const uint8_t *start;
888 } integer;
889
890 /*
891 * This string is used to pass information from the lexer to the parser. It
892 * is particularly necessary because of escape sequences.
893 */
894 pm_string_t current_string;
895
896 /*
897 * The line number at the start of the parse. This will be used to offset
898 * the line numbers of all of the locations.
899 */
900 int32_t start_line;
901
902 /*
903 * When a string-like expression is being lexed, any byte or escape sequence
904 * that resolves to a value whose top bit is set (i.e., >= 0x80) will
905 * explicitly set the encoding to the same encoding as the source.
906 * Alternatively, if a unicode escape sequence is used (e.g., \\u{80}) that
907 * resolves to a value whose top bit is set, then the encoding will be
908 * explicitly set to UTF-8.
909 *
910 * The _next_ time this happens, if the encoding that is about to become the
911 * explicitly set encoding does not match the previously set explicit
912 * encoding, a mixed encoding error will be emitted.
913 *
914 * When the expression is finished being lexed, the explicit encoding
915 * controls the encoding of the expression. For the most part this means
916 * that the expression will either be encoded in the source encoding or
917 * UTF-8. This holds for all encodings except US-ASCII. If the source is
918 * US-ASCII and an explicit encoding was set that was _not_ UTF-8, then the
919 * expression will be encoded as ASCII-8BIT.
920 *
921 * Note that if the expression is a list, different elements within the same
922 * list can have different encodings, so this will get reset between each
923 * element. Furthermore all of this only applies to lists that support
924 * interpolation, because otherwise escapes that could change the encoding
925 * are ignored.
926 *
927 * At first glance, it may make more sense for this to live on the lexer
928 * mode, but we need it here to communicate back to the parser for character
929 * literals that do not push a new lexer mode.
930 */
931 const pm_encoding_t *explicit_encoding;
932
933 /*
934 * When parsing block exits (e.g., break, next, redo), we need to validate
935 * that they are in correct contexts. For the most part we can do this by
936 * looking at our parent contexts. However, modifier while and until
937 * expressions can change that context to make block exits valid. In these
938 * cases, we need to keep track of the block exits and then validate them
939 * after the expression has been parsed.
940 *
941 * We use a pointer here because we don't want to keep a whole list attached
942 * since this will only be used in the context of begin/end expressions.
943 */
944 pm_scratch_node_list_t *current_block_exits;
945
946 /*
947 * The address of the C stack when pm_parse was entered. The stack budget
948 * is measured as the distance from this address.
949 */
950 uintptr_t stack_base;
951
952 /*
953 * The number of bytes of C stack below stack_base that the recursive
954 * descent may use before it reports that the stack is exhausted. SIZE_MAX
955 * when no budget was given, so the check never fires.
956 */
957 size_t stack_budget;
958
959 /* The version of prism that we should use to parse. */
960 pm_options_version_t version;
961
962 /* The command line flags given from the options. */
963 uint8_t command_line;
964
965 /*
966 * Whether or not we have found a frozen_string_literal magic comment with
967 * a true or false value.
968 * May be:
969 * - PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
970 * - PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
971 * - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
972 */
973 int8_t frozen_string_literal;
974
975 /*
976 * Whether or not we are parsing an eval string. This impacts whether or not
977 * we should evaluate if block exits/yields are valid.
978 */
979 bool parsing_eval;
980
981 /*
982 * Whether or not we are parsing a "partial" script, which is a script that
983 * will be evaluated in the context of another script, so we should not
984 * check jumps (next/break/etc.) for validity.
985 */
986 bool partial_script;
987
988 /*
989 * Whether or not comments should be attached to their owning nodes once
990 * parsing has finished.
991 */
992 bool attach_comments;
993
994 /* Whether or not we're at the beginning of a command. */
995 bool command_start;
996
997 /*
998 * Whether or not we're currently parsing the body of an endless method
999 * definition. In this context, PM_TOKEN_KEYWORD_DO_BLOCK should not be
1000 * consumed by commands (it should bubble up to the outer context).
1001 */
1002 bool in_endless_def_body;
1003
1004 /* Whether or not we're currently recovering from a syntax error. */
1005 bool recovering;
1006
1007 /*
1008 * Whether or not the source being parsed could become valid if more input
1009 * were appended. This is set to false when the parser encounters a token
1010 * that is definitively wrong (e.g., a stray `end` or `]`) as opposed to
1011 * merely incomplete.
1012 */
1013 bool continuable;
1014
1015 /*
1016 * This is very specialized behavior for when you want to parse in a context
1017 * that does not respect encoding comments. Its main use case is translating
1018 * into the whitequark/parser AST which re-encodes source files in UTF-8
1019 * before they are parsed and ignores encoding comments.
1020 */
1021 bool encoding_locked;
1022
1023 /*
1024 * Whether or not the encoding has been changed by a magic comment. We use
1025 * this to provide a fast path for the lexer instead of going through the
1026 * function pointer.
1027 */
1028 bool encoding_changed;
1029
1030 /*
1031 * This flag indicates that we are currently parsing a pattern matching
1032 * expression and impacts that calculation of newlines.
1033 */
1034 bool pattern_matching_newlines;
1035
1036 /* This flag indicates that we are currently parsing a keyword argument. */
1037 bool in_keyword_arg;
1038
1039 /*
1040 * Whether or not the parser has seen a token that has semantic meaning
1041 * (i.e., a token that is not a comment or whitespace).
1042 */
1043 bool semantic_token_seen;
1044
1045 /*
1046 * By default, Ruby always warns about mismatched indentation. This can be
1047 * toggled with a magic comment.
1048 */
1049 bool warn_mismatched_indentation;
1050
1051 /*
1052 * Whether the stack exhausted error has been reported. Once the budget is
1053 * used up, every recursion that unwinds past it would report the error
1054 * again, so it is only reported the first time.
1055 */
1056 bool stack_exhausted_reported;
1057
1058#if defined(PRISM_HAS_NEON) || defined(PRISM_HAS_SSE2) || defined(PRISM_HAS_SWAR)
1059 /*
1060 * Cached lookup tables for pm_strpbrk's SIMD fast path. Avoids rebuilding
1061 * the nibble-based tables on every call when the charset hasn't changed
1062 * (which is the common case during string/regex/list lexing).
1063 */
1064 struct {
1065 /* The cached charset (null-terminated, max 11 chars + NUL). */
1066 uint8_t charset[12];
1067
1068 /* Nibble-based low lookup table for SIMD matching. */
1069 uint8_t low_lut[16];
1070
1071 /* Nibble-based high lookup table for SIMD matching. */
1072 uint8_t high_lut[16];
1073
1074 /* Scalar fallback table (4 x 64-bit bitmasks covering all ASCII). */
1075 uint64_t table[4];
1076 } strpbrk_cache;
1077#endif
1078};
1079
1080/*
1081 * Initialize a parser with the given start and end pointers.
1082 */
1083void pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options);
1084
1085/*
1086 * Free the memory held by the given parser.
1087 *
1088 * This does not free the `pm_options_t` object that was used to initialize the
1089 * parser.
1090 */
1091void pm_parser_cleanup(pm_parser_t *parser);
1092
1093/*
1094 * Returns true if the bytes between start and end spell a keyword, using the
1095 * same lookup that the lexer does.
1096 */
1097bool pm_keyword_p(const uint8_t *start, const uint8_t *end);
1098
1099/*
1100 * Returns true if parsing has used more C stack since pm_parse was entered than
1101 * the stack budget allows. Every recursion that runs inside pm_parse checks
1102 * this alongside its own depth limit, so that the budget bounds all of them.
1103 * The distance is taken without assuming which direction the stack grows. With
1104 * no budget the limit is SIZE_MAX, so this never fires.
1105 */
1106static PRISM_INLINE bool
1107pm_parser_stack_exhausted(const pm_parser_t *parser) {
1108 uintptr_t current = pm_stack_address();
1109 uintptr_t used = current < parser->stack_base ? parser->stack_base - current : current - parser->stack_base;
1110 return used > parser->stack_budget;
1111}
1112
1113/*
1114 * Reports that the stack budget has been used up, at the given location. This
1115 * is a separate error from the nesting depth limit, with its own error level,
1116 * because the source may be valid and parse with more stack. It is reported
1117 * once per parse.
1118 */
1119void pm_parser_err_stack_exhausted(pm_parser_t *parser, uint32_t start, uint32_t length);
1120
1121#endif
uint32_t pm_constant_id_t
A constant id is a unique identifier for a constant in the constant pool.
#define PRISM_INLINE
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
Definition inline.h:12
A list of byte offsets of newlines in a string.
The parser used to parse Ruby source.
void(* pm_lex_callback_t)(pm_parser_t *parser, pm_token_t *token, void *data)
This is the callback that is called when a token is lexed.
Definition parser.h:59
void(* pm_encoding_changed_callback_t)(pm_parser_t *parser)
When the encoding that is being used to parse the source is changed by prism, we provide the ability ...
Definition parser.h:53
static PRISM_INLINE uintptr_t pm_stack_address(void)
Returns the current position of the C stack, for measuring how much of it has been consumed since an ...
Definition stack.h:30
C99 shim for <stdbool.h>.
A list of offsets of the start of lines in a string.
This struct represents a slice in the source code, defined by an offset and a length.
Definition ast.h:579
A view of a sequence of bytes.
Definition stringy.h:23
This struct represents a token in the Ruby source.
Definition ast.h:551