Ruby 4.1.0dev (2026-11-20 revision 6b79116e7664881b709b47e982bdef67a05f7866)
parser.h
1#ifndef PRISM_INTERNAL_PARSER_H
2#define PRISM_INTERNAL_PARSER_H
3
7
8#include "prism/internal/arena.h"
9#include "prism/internal/buffer.h"
10#include "prism/internal/comments.h"
11#include "prism/internal/constant_pool.h"
12#include "prism/internal/encoding.h"
13#include "prism/internal/list.h"
14#include "prism/internal/node.h"
15#include "prism/internal/options.h"
16#include "prism/internal/static_literals.h"
17#include "prism/internal/strpbrk.h"
18
19#include "prism/ast.h"
21#include "prism/parser.h"
22
23#include <stdbool.h>
24#include <stddef.h>
25#include <stdint.h>
26
27/*
28 * A place in the unescaped content of a regular expression where it stops
29 * lining up with the source, because the lexer dropped or rewrote an escape
30 * just before it. From here up to the next such place, each byte of the content
31 * is the byte at the same distance into the source.
32 */
33typedef struct {
34 /* The offset of the place in the unescaped content. */
35 uint32_t content;
36
37 /* The offset in the source of the byte at that place. */
38 uint32_t source;
40
41/*
42 * The places where the unescaped content of a regular expression stops lining
43 * up with the source, in order. Before the first one, each byte of the content
44 * is the byte at the same distance from the start of the content in the source.
45 * The values are allocated in the scratch arena, since they are only read while
46 * the regular expression is validated.
47 */
48typedef struct {
49 /* The places, in order of their offsets in the content. */
50 pm_regexp_offset_t *values;
51
52 /* The number of places. */
53 size_t size;
54
55 /* The number of places that fit in the values before they must grow. */
56 size_t capacity;
58
59/*
60 * This enum provides various bits that represent different kinds of states that
61 * the lexer can track. This is used to determine which kind of token to return
62 * based on the context of the parser.
63 */
64typedef enum {
65 PM_LEX_STATE_BIT_BEG,
66 PM_LEX_STATE_BIT_END,
67 PM_LEX_STATE_BIT_ENDARG,
68 PM_LEX_STATE_BIT_ENDFN,
69 PM_LEX_STATE_BIT_ARG,
70 PM_LEX_STATE_BIT_CMDARG,
71 PM_LEX_STATE_BIT_MID,
72 PM_LEX_STATE_BIT_FNAME,
73 PM_LEX_STATE_BIT_DOT,
74 PM_LEX_STATE_BIT_CLASS,
75 PM_LEX_STATE_BIT_LABEL,
76 PM_LEX_STATE_BIT_LABELED,
77 PM_LEX_STATE_BIT_FITEM
78} pm_lex_state_bit_t;
79
80/*
81 * This enum combines the various bits from the above enum into individual
82 * values that represent the various states of the lexer.
83 */
84typedef enum {
85 PM_LEX_STATE_NONE = 0,
86 PM_LEX_STATE_BEG = (1 << PM_LEX_STATE_BIT_BEG),
87 PM_LEX_STATE_END = (1 << PM_LEX_STATE_BIT_END),
88 PM_LEX_STATE_ENDARG = (1 << PM_LEX_STATE_BIT_ENDARG),
89 PM_LEX_STATE_ENDFN = (1 << PM_LEX_STATE_BIT_ENDFN),
90 PM_LEX_STATE_ARG = (1 << PM_LEX_STATE_BIT_ARG),
91 PM_LEX_STATE_CMDARG = (1 << PM_LEX_STATE_BIT_CMDARG),
92 PM_LEX_STATE_MID = (1 << PM_LEX_STATE_BIT_MID),
93 PM_LEX_STATE_FNAME = (1 << PM_LEX_STATE_BIT_FNAME),
94 PM_LEX_STATE_DOT = (1 << PM_LEX_STATE_BIT_DOT),
95 PM_LEX_STATE_CLASS = (1 << PM_LEX_STATE_BIT_CLASS),
96 PM_LEX_STATE_LABEL = (1 << PM_LEX_STATE_BIT_LABEL),
97 PM_LEX_STATE_LABELED = (1 << PM_LEX_STATE_BIT_LABELED),
98 PM_LEX_STATE_FITEM = (1 << PM_LEX_STATE_BIT_FITEM),
99 PM_LEX_STATE_BEG_ANY = PM_LEX_STATE_BEG | PM_LEX_STATE_MID | PM_LEX_STATE_CLASS,
100 PM_LEX_STATE_ARG_ANY = PM_LEX_STATE_ARG | PM_LEX_STATE_CMDARG,
101 PM_LEX_STATE_END_ANY = PM_LEX_STATE_END | PM_LEX_STATE_ENDARG | PM_LEX_STATE_ENDFN
102} pm_lex_state_t;
103
104/*
105 * The type of quote that a heredoc uses.
106 */
107typedef enum {
108 PM_HEREDOC_QUOTE_NONE,
109 PM_HEREDOC_QUOTE_SINGLE = '\'',
110 PM_HEREDOC_QUOTE_DOUBLE = '"',
111 PM_HEREDOC_QUOTE_BACKTICK = '`',
112} pm_heredoc_quote_t;
113
114/*
115 * The type of indentation that a heredoc uses.
116 */
117typedef enum {
118 PM_HEREDOC_INDENT_NONE,
119 PM_HEREDOC_INDENT_DASH,
120 PM_HEREDOC_INDENT_TILDE,
121} pm_heredoc_indent_t;
122
123/*
124 * All of the information necessary to store to lexing a heredoc.
125 */
126typedef struct {
127 /* A pointer to the start of the heredoc identifier. */
128 const uint8_t *ident_start;
129
130 /* The length of the heredoc identifier. */
131 size_t ident_length;
132
133 /* The type of quote that the heredoc uses. */
134 pm_heredoc_quote_t quote;
135
136 /* The type of indentation that the heredoc uses. */
137 pm_heredoc_indent_t indent;
139
140/*
141 * When lexing Ruby source, the lexer has a small amount of state to tell which
142 * kind of token it is currently lexing. For example, when we find the start of
143 * a string, the first token that we return is a TOKEN_STRING_BEGIN token. After
144 * that the lexer is now in the PM_LEX_STRING mode, and will return tokens that
145 * are found as part of a string.
146 */
147typedef struct pm_lex_mode {
148 /* The type of this lex mode. */
149 enum {
150 /* This state is used when any given token is being lexed. */
151 PM_LEX_DEFAULT,
152
153 /*
154 * This state is used when we're lexing as normal but inside an embedded
155 * expression of a string.
156 */
157 PM_LEX_EMBEXPR,
158
159 /*
160 * This state is used when we're lexing a variable that is embedded
161 * directly inside of a string with the # shorthand.
162 */
163 PM_LEX_EMBVAR,
164
165 /* This state is used when you are inside the content of a heredoc. */
166 PM_LEX_HEREDOC,
167
168 /*
169 * This state is used when we are lexing a list of tokens, as in a %w
170 * word list literal or a %i symbol list literal.
171 */
172 PM_LEX_LIST,
173
174 /*
175 * This state is used when a regular expression has been begun and we
176 * are looking for the terminator.
177 */
178 PM_LEX_REGEXP,
179
180 /*
181 * This state is used when we are lexing a string or a string-like
182 * token, as in string content with either quote or an xstring.
183 */
184 PM_LEX_STRING
185 } mode;
186
187 /* The data associated with this type of lex mode. */
188 union {
189 struct {
190 /* This keeps track of the nesting level of the list. */
191 size_t nesting;
192
193 /* Whether or not interpolation is allowed in this list. */
194 bool interpolation;
195
196 /*
197 * Whether any token has been emitted from this list. A word
198 * separator delimits the opener from the first word, so one
199 * without source characters is emitted when the list does not
200 * start with whitespace.
201 */
202 bool started;
203
204 /*
205 * Whether the previously emitted token was a word separator. A
206 * word separator delimits the last word from the terminator, so
207 * one without source characters is emitted when the list does
208 * not end with whitespace.
209 */
210 bool separated;
211
212 /*
213 * When lexing a list, it takes into account balancing the
214 * terminator if the terminator is one of (), [], {}, or <>.
215 */
216 uint8_t incrementor;
217
218 /* This is the terminator of the list literal. */
219 uint8_t terminator;
220
221 /*
222 * This is the character set that should be used to delimit the
223 * tokens within the list.
224 */
225 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
226 } list;
227
228 struct {
229 /*
230 * This keeps track of the nesting level of the regular expression.
231 */
232 size_t nesting;
233
234 /*
235 * When lexing a regular expression, it takes into account balancing
236 * the terminator if the terminator is one of (), [], {}, or <>.
237 */
238 uint8_t incrementor;
239
240 /* This is the terminator of the regular expression. */
241 uint8_t terminator;
242
243 /*
244 * This is the character set that should be used to delimit the
245 * tokens within the regular expression.
246 */
247 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
248 } regexp;
249
250 struct {
251 /* This keeps track of the nesting level of the string. */
252 size_t nesting;
253
254 /* Whether or not interpolation is allowed in this string. */
255 bool interpolation;
256
257 /*
258 * Whether or not at the end of the string we should allow a :,
259 * which would indicate this was a dynamic symbol instead of a
260 * string.
261 */
262 bool label_allowed;
263
264 /* Whether the final content token joined text after the last
265 * breakpoint, where this string used to split at EOF. */
266 bool joined_at_eof;
267
268 /*
269 * When lexing a string, it takes into account balancing the
270 * terminator if the terminator is one of (), [], {}, or <>.
271 */
272 uint8_t incrementor;
273
274 /*
275 * This is the terminator of the string. It is typically either a
276 * single or double quote.
277 */
278 uint8_t terminator;
279
280 /*
281 * This is the character set that should be used to delimit the
282 * tokens within the string.
283 */
284 uint8_t breakpoints[PM_STRPBRK_CACHE_SIZE];
285 } string;
286
287 struct {
288 /*
289 * All of the data necessary to lex a heredoc.
290 */
292
293 /*
294 * This is the pointer to the character where lexing should resume
295 * once the heredoc has been completely processed.
296 */
297 const uint8_t *next_start;
298
299 /*
300 * This is used to track the amount of common whitespace on each
301 * line so that we know how much to dedent each line in the case of
302 * a tilde heredoc.
303 */
304 size_t *common_whitespace;
305
306 /* True if the previous token ended with a line continuation. */
307 bool line_continuation;
308 } heredoc;
309 } as;
310
311 /* The previous lex state so that it knows how to pop. */
312 struct pm_lex_mode *prev;
313} pm_lex_mode_t;
314
315/*
316 * While parsing, we keep track of a stack of contexts. This is helpful for
317 * error recovery so that we can pop back to a previous context when we hit a
318 * token that is understood by a parent context but not by the current context.
319 */
320typedef enum {
321 /* a null context, used for returning a value from a function */
322 PM_CONTEXT_NONE = 0,
323
324 /* a begin statement */
325 PM_CONTEXT_BEGIN,
326
327 /* an ensure statement with an explicit begin */
328 PM_CONTEXT_BEGIN_ENSURE,
329
330 /* a rescue else statement with an explicit begin */
331 PM_CONTEXT_BEGIN_ELSE,
332
333 /* a rescue statement with an explicit begin */
334 PM_CONTEXT_BEGIN_RESCUE,
335
336 /* expressions in block arguments using braces */
337 PM_CONTEXT_BLOCK_BRACES,
338
339 /* expressions in block arguments using do..end */
340 PM_CONTEXT_BLOCK_KEYWORDS,
341
342 /* an ensure statement within a do..end block */
343 PM_CONTEXT_BLOCK_ENSURE,
344
345 /* a rescue else statement within a do..end block */
346 PM_CONTEXT_BLOCK_ELSE,
347
348 /* expressions in block parameters `foo do |...| end ` */
349 PM_CONTEXT_BLOCK_PARAMETERS,
350
351 /* a rescue statement within a do..end block */
352 PM_CONTEXT_BLOCK_RESCUE,
353
354 /* a case when statements */
355 PM_CONTEXT_CASE_WHEN,
356
357 /* a case in statements */
358 PM_CONTEXT_CASE_IN,
359
360 /* a class declaration */
361 PM_CONTEXT_CLASS,
362
363 /* an ensure statement within a class statement */
364 PM_CONTEXT_CLASS_ENSURE,
365
366 /* a rescue else statement within a class statement */
367 PM_CONTEXT_CLASS_ELSE,
368
369 /* a rescue statement within a class statement */
370 PM_CONTEXT_CLASS_RESCUE,
371
372 /* a method definition */
373 PM_CONTEXT_DEF,
374
375 /* an ensure statement within a method definition */
376 PM_CONTEXT_DEF_ENSURE,
377
378 /* a rescue else statement within a method definition */
379 PM_CONTEXT_DEF_ELSE,
380
381 /* a rescue statement within a method definition */
382 PM_CONTEXT_DEF_RESCUE,
383
384 /* a method definition's parameters */
385 PM_CONTEXT_DEF_PARAMS,
386
387 /* a defined? expression */
388 PM_CONTEXT_DEFINED,
389
390 /* a method definition's default parameter */
391 PM_CONTEXT_DEFAULT_PARAMS,
392
393 /* an else clause */
394 PM_CONTEXT_ELSE,
395
396 /* an elsif clause */
397 PM_CONTEXT_ELSIF,
398
399 /* an interpolated expression */
400 PM_CONTEXT_EMBEXPR,
401
402 /* a for loop */
403 PM_CONTEXT_FOR,
404
405 /* a for loop's index */
406 PM_CONTEXT_FOR_INDEX,
407
408 /* an if statement */
409 PM_CONTEXT_IF,
410
411 /* a lambda expression with braces */
412 PM_CONTEXT_LAMBDA_BRACES,
413
414 /* a lambda expression with do..end */
415 PM_CONTEXT_LAMBDA_DO_END,
416
417 /* an ensure statement within a lambda expression */
418 PM_CONTEXT_LAMBDA_ENSURE,
419
420 /* a rescue else statement within a lambda expression */
421 PM_CONTEXT_LAMBDA_ELSE,
422
423 /* a rescue statement within a lambda expression */
424 PM_CONTEXT_LAMBDA_RESCUE,
425
426 /* the predicate clause of a loop statement */
427 PM_CONTEXT_LOOP_PREDICATE,
428
429 /* the top level context */
430 PM_CONTEXT_MAIN,
431
432 /* a module declaration */
433 PM_CONTEXT_MODULE,
434
435 /* an ensure statement within a module statement */
436 PM_CONTEXT_MODULE_ENSURE,
437
438 /* a rescue else statement within a module statement */
439 PM_CONTEXT_MODULE_ELSE,
440
441 /* a rescue statement within a module statement */
442 PM_CONTEXT_MODULE_RESCUE,
443
444 /* a multiple target expression */
445 PM_CONTEXT_MULTI_TARGET,
446
447 /* a parenthesized expression */
448 PM_CONTEXT_PARENS,
449
450 /* an END block */
451 PM_CONTEXT_POSTEXE,
452
453 /* a predicate inside an if/elsif/unless statement */
454 PM_CONTEXT_PREDICATE,
455
456 /* a BEGIN block */
457 PM_CONTEXT_PREEXE,
458
459 /* a modifier rescue clause */
460 PM_CONTEXT_RESCUE_MODIFIER,
461
462 /* a singleton class definition */
463 PM_CONTEXT_SCLASS,
464
465 /* an ensure statement with a singleton class */
466 PM_CONTEXT_SCLASS_ENSURE,
467
468 /* a rescue else statement with a singleton class */
469 PM_CONTEXT_SCLASS_ELSE,
470
471 /* a rescue statement with a singleton class */
472 PM_CONTEXT_SCLASS_RESCUE,
473
474 /* a ternary expression */
475 PM_CONTEXT_TERNARY,
476
477 /* an unless statement */
478 PM_CONTEXT_UNLESS,
479
480 /* an until statement */
481 PM_CONTEXT_UNTIL,
482
483 /* a while statement */
484 PM_CONTEXT_WHILE,
485
486 /* the number of contexts, which is not itself a context */
487 PM_CONTEXT_MAXIMUM,
488} pm_context_t;
489
490/* This is a node in a linked list of contexts. */
491typedef struct pm_context_node {
492 /* The context that this node represents. */
493 pm_context_t context;
494
495 /* A pointer to the previous context in the linked list. */
496 struct pm_context_node *prev;
497
498 /* One bit set per context in this list, including one for this node. */
499 uint64_t mask;
500} pm_context_node_t;
501
502/* The type of shareable constant value that can be set. */
503typedef uint8_t pm_shareable_constant_value_t;
504static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_NONE = 0x0;
505static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_LITERAL = PM_SHAREABLE_CONSTANT_NODE_FLAGS_LITERAL;
506static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_EVERYTHING;
507static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY;
508
509/*
510 * This tracks an individual local variable in a certain lexical context, as
511 * well as the number of times is it read. The locals of every live scope are
512 * kept on a single stack on the parser, since a local is only ever declared in
513 * the innermost scope and is discarded when that scope is popped.
514 */
515typedef struct {
516 /* The name of the local variable. */
517 pm_constant_id_t name;
518
519 /* The location of the local variable in the source. */
520 pm_location_t location;
521
522 /* The number of times the local variable is read. */
523 uint32_t reads;
524
525 /* The level of the scope that declares the local variable. */
526 uint32_t level;
527
528 /*
529 * The index in the locals stack of the local with the same name that this
530 * local hides while its scope is live, or 0 if it hides nothing. It is
531 * restored as the binding of the name when the scope is popped.
532 */
533 uint32_t shadowed;
534} pm_local_t;
535
536/* The flags about scope parameters that can be set. */
537typedef uint8_t pm_scope_parameters_t;
538static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NONE = 0x0;
539static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x1;
540static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x2;
541static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x4;
542static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x8;
543static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED = 0x10;
544static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_INNER = 0x20;
545static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_FOUND = 0x40;
546
547/*
548 * This struct represents a node in a linked list of scopes. Some scopes can see
549 * into their parent scopes, while others cannot.
550 */
551typedef struct pm_scope {
552 /* A pointer to the previous scope in the linked list. */
553 struct pm_scope *previous;
554
555 /*
556 * The index in the locals stack of the first local of this scope. The
557 * locals of this scope run from here to the top of the stack.
558 */
559 uint32_t locals_start;
560
561 /*
562 * The position of this scope in the scope stack, where the outermost scope
563 * is at level 1.
564 */
565 uint32_t level;
566
567 /*
568 * The locals_start of the nearest closed scope at or above this one. Every
569 * local below this index in the locals stack belongs to a scope that this
570 * scope cannot see into.
571 */
572 uint32_t floor;
573
574 /*
575 * This is a list of the implicit parameters contained within the block.
576 * These will be processed after the block is parsed to determine the kind
577 * of parameters node that should be used and to check if any errors need to
578 * be added.
579 */
580 pm_scratch_node_list_t implicit_parameters;
581
582 /*
583 * This is a bitfield that indicates the parameters that are being used in
584 * this scope. It is a combination of the PM_SCOPE_PARAMETERS_* constants.
585 * There are three different kinds of parameters that can be used in a
586 * scope:
587 *
588 * - Ordinary parameters (e.g., def foo(bar); end)
589 * - Numbered parameters (e.g., def foo; _1; end)
590 * - The it parameter (e.g., def foo; it; end)
591 *
592 * If ordinary parameters are being used, then certain parameters can be
593 * forwarded to another method/structure. Those are indicated by four
594 * additional bits in the params field. For example, some combinations of:
595 *
596 * - def foo(*); end
597 * - def foo(**); end
598 * - def foo(&); end
599 * - def foo(...); end
600 */
601 pm_scope_parameters_t parameters;
602
603 /*
604 * The current state of constant shareability for this scope. This is
605 * changed by magic shareable_constant_value comments.
606 */
607 pm_shareable_constant_value_t shareable_constant;
608
609 /*
610 * A boolean indicating whether or not this scope can see into its parent.
611 * If closed is true, then the scope cannot see into its parent.
612 */
613 bool closed;
614} pm_scope_t;
615
616/*
617 * A stack of boolean values with no fixed depth limit. The newest frames live
618 * in an inline word, so pushing and popping are usually a shift and a compare.
619 * When that word fills, it moves to the spill array and the inline word starts
620 * over empty. The spill array is only allocated once the nesting is more than
621 * 64 frames deep.
622 */
623typedef struct {
624 /*
625 * The newest frames, with the top of the stack in bit 0. It holds between
626 * 1 and 64 frames while the stack is not empty, and is 0 when it is.
627 */
628 uint64_t top;
629
630 /*
631 * The number of frames on the whole stack. The inline word holds the
632 * frames above the last multiple of 64, so the spill array holds
633 * (depth - 1) / 64 words.
634 */
635 uint32_t depth;
636
637 /* The number of words the spill array has room for. */
638 uint32_t capacity;
639
640 /*
641 * Full words of 64 older frames each, oldest first, allocated from the
642 * scratch arena.
643 */
644 uint64_t *spill;
646
647/*
648 * This struct represents the overall parser. It contains a reference to the
649 * source file, as well as pointers that indicate where in the source it's
650 * currently parsing. It also contains the most recent and current token that
651 * it's considering.
652 */
654 /* The arena used for all AST-lifetime allocations. Caller-owned. */
655 pm_arena_t *arena;
656
657 /*
658 * The arena used for parser metadata (comments, diagnostics, etc.), which
659 * lives as long as the parser.
660 */
661 pm_arena_t metadata_arena;
662
663 /*
664 * The arena used for state that is only needed while parsing, such as
665 * scopes, locals, and contexts. It is released when pm_parse returns, so
666 * nothing allocated in it can be reachable from the tree or from the
667 * parser's metadata.
668 */
669 pm_arena_t scratch_arena;
670
671 /*
672 * The next node identifier that will be assigned. This is a unique
673 * identifier used to track nodes such that the syntax tree can be dropped
674 * but the node can be found through another parse.
675 */
676 uint32_t node_id;
677
678 /*
679 * A single-entry cache for pm_parser_constant_id_raw. Avoids redundant
680 * constant pool lookups when the same token is resolved multiple times
681 * (e.g., once during lexing for local variable detection, and again
682 * during parsing for node creation).
683 */
684 struct {
685 const uint8_t *start;
686 const uint8_t *end;
688 } constant_cache;
689
690 /* The current state of the lexer. */
691 pm_lex_state_t lex_state;
692
693 /* Tracks the current nesting of (), [], and {}. */
694 int enclosure_nesting;
695
696 /*
697 * Used to temporarily track the nesting of enclosures to determine if a {
698 * is the beginning of a lambda following the parameters of a lambda.
699 */
700 int lambda_enclosure_nesting;
701
702 /*
703 * Used to track the nesting of braces to ensure we get the correct value
704 * when we are interpolating blocks with braces.
705 */
706 int brace_nesting;
707
708 /*
709 * The stack used to determine if a do keyword belongs to the predicate of a
710 * while, until, or for loop.
711 */
712 pm_state_stack_t do_loop_stack;
713
714 /*
715 * The stack used to determine if a do keyword belongs to the beginning of a
716 * block.
717 */
718 pm_state_stack_t accepts_block_stack;
719
720 /* A stack of lex modes, linked through their prev pointers. */
721 struct {
722 /* The current mode of the lexer. */
723 pm_lex_mode_t *current;
724
725 /*
726 * The mode at the bottom of the stack, which is never popped. Every
727 * other mode is allocated from the scratch arena.
728 */
729 pm_lex_mode_t base;
730
731 /*
732 * Lex modes that have been popped, kept for reuse by the next push.
733 * They are allocated from the scratch arena, so they are never freed
734 * individually.
735 */
736 pm_lex_mode_t *free;
737 } lex_modes;
738
739 /*
740 * The chunks that open node lists are built in. They are carved from the
741 * scratch arena, so they are never freed individually.
742 */
743 pm_node_chunks_t node_chunks;
744
745 /* The pointer to the start of the source. */
746 const uint8_t *start;
747
748 /* The pointer to the end of the source. */
749 const uint8_t *end;
750
751 /* The previous token we were considering. */
752 pm_token_t previous;
753
754 /* The current token we're considering. */
755 pm_token_t current;
756
757 /*
758 * This is a special field set on the parser when we need the parser to jump
759 * to a specific location when lexing the next token, as opposed to just
760 * using the end of the previous token. Normally this is NULL.
761 */
762 const uint8_t *next_start;
763
764 /*
765 * This field indicates the end of a heredoc whose identifier was found on
766 * the current line. If another heredoc is found on the same line, then this
767 * will be moved forward to the end of that heredoc. If no heredocs are
768 * found on a line then this is NULL.
769 */
770 const uint8_t *heredoc_end;
771
772 /* The list of comments that have been found while parsing. */
773 pm_list_t comment_list;
774
775 /*
776 * The attachments of comments to the nodes that own them, populated after
777 * parsing when the attach_comments option is set.
778 */
779 pm_comment_attachments_t comment_attachments;
780
781 /* The list of magic comments that have been found while parsing. */
782 pm_list_t magic_comment_list;
783
784 /*
785 * An optional location that represents the location of the __END__ marker
786 * and the rest of the content of the file. This content is loaded into the
787 * DATA constant when the file being parsed is the main file being executed.
788 */
789 pm_location_t data_loc;
790
791 /* The list of warnings that have been found while parsing. */
792 pm_list_t warning_list;
793
794 /* The list of errors that have been found while parsing. */
795 pm_list_t error_list;
796
797 /* The current local scope. */
798 pm_scope_t *current_scope;
799
800 /*
801 * The locals of every live scope, in the order they were declared. Index 0
802 * is never used, so that 0 can mean the absence of a local.
803 */
804 pm_local_t *locals;
805
806 /* The number of slots in use in locals, including the unused slot 0. */
807 uint32_t locals_size;
808
809 /* The number of slots allocated in locals. */
810 uint32_t locals_capacity;
811
812 /*
813 * The index in the locals stack of the innermost live local with each
814 * constant id as its name, or 0 if there is none. A local that is declared
815 * in a scope replaces the binding of its name, and the binding it replaced
816 * is restored when the scope is popped. Constant ids at or beyond the
817 * capacity have never been declared as locals, so they are unbound.
818 */
819 uint32_t *bindings;
820
821 /* The number of slots allocated in bindings. */
822 uint32_t bindings_capacity;
823
824 /*
825 * Scopes that have been popped, kept for reuse by the next push. Scopes
826 * are allocated from the scratch arena, so they are never freed
827 * individually.
828 */
829 pm_scope_t *free_scopes;
830
831 /* The current parsing context. */
832 pm_context_node_t *current_context;
833
834 /*
835 * Context nodes that have been popped, kept for reuse by the next push.
836 * Context nodes are allocated from the scratch arena, so they are never
837 * freed individually.
838 */
839 pm_context_node_t *free_contexts;
840
841 /*
842 * The encoding functions for the current file is attached to the parser as
843 * it's parsing so that it can change with a magic comment.
844 */
845 const pm_encoding_t *encoding;
846
847 /*
848 * When the encoding that is being used to parse the source is changed by
849 * prism, we provide the ability here to call out to a user-defined
850 * function.
851 */
852 pm_encoding_changed_callback_t encoding_changed_callback;
853
854 /*
855 * This pointer indicates where a comment must start if it is to be
856 * considered an encoding comment.
857 */
858 const uint8_t *encoding_comment_start;
859
860 /*
861 * When you are lexing through a file, the lexer needs all of the information
862 * that the parser additionally provides (for example, the local table). So if
863 * you want to properly lex Ruby, you need to actually lex it in the context of
864 * the parser. In order to provide this functionality, we optionally allow a
865 * struct to be attached to the parser that calls back out to a user-provided
866 * callback when each token is lexed.
867 */
868 struct {
869 /*
870 * This is the callback that is called when a token is lexed. It is
871 * passed the opaque data pointer, the parser, and the token that was
872 * lexed.
873 */
874 pm_lex_callback_t callback;
875
876 /*
877 * This opaque pointer is used to provide whatever information the user
878 * deemed necessary to the callback. In our case we use it to pass the
879 * array that the tokens get appended into.
880 */
881 void *data;
882 } lex_callback;
883
884 /*
885 * This is the path of the file being parsed. We use the filepath when
886 * constructing SourceFileNodes.
887 */
888 pm_string_t filepath;
889
890 /*
891 * This constant pool keeps all of the constants defined throughout the file
892 * so that we can reference them later.
893 */
894 pm_constant_pool_t constant_pool;
895
896 /* This is the list of line offsets in the source file. */
897 pm_line_offset_list_t line_offsets;
898
899 /*
900 * State communicated from the lexer to the parser for integer tokens.
901 */
902 struct {
903 /*
904 * A flag indicating the base of the integer (binary, octal, decimal,
905 * hexadecimal). Set during lexing and read during node creation.
906 */
907 pm_node_flags_t base;
908
909 /*
910 * When lexing a decimal integer of at most 19 digits, which always
911 * fits in a uint64_t, or a plain 0, we compute the value during
912 * lexing to avoid re-scanning the digits during parsing. If start is
913 * set, this holds the result for the token that begins there and
914 * pm_integer_parse can be skipped.
915 */
916 uint64_t value;
917
918 /*
919 * The start of the token whose value is held in value, or NULL before
920 * any has been. The parser creates the node for a token only after
921 * lexing the next one, which may be another integer that replaces the
922 * value, so node creation checks that it belongs to the token at hand.
923 */
924 const uint8_t *start;
925 } integer;
926
927 /*
928 * This string is used to pass information from the lexer to the parser. It
929 * is particularly necessary because of escape sequences.
930 */
931 pm_string_t current_string;
932
933 /* Reusable workspace for rewriting escaped token content. */
934 pm_buffer_t lex_buffer;
935
936 /*
937 * Where current_string stops lining up with the source, when it is the
938 * content of a regular expression.
939 */
940 pm_regexp_offsets_t current_regexp_offsets;
941
942 /* Named captures from the regexp immediately before a pending =~. */
943 const struct pm_regexp_match *regexp_match;
944
945 /*
946 * The line number at the start of the parse. This will be used to offset
947 * the line numbers of all of the locations.
948 */
949 int32_t start_line;
950
951 /*
952 * When a string-like expression is being lexed, any byte or escape sequence
953 * that resolves to a value whose top bit is set (i.e., >= 0x80) will
954 * explicitly set the encoding to the same encoding as the source.
955 * Alternatively, if a unicode escape sequence is used (e.g., \\u{80}) that
956 * resolves to a value whose top bit is set, then the encoding will be
957 * explicitly set to UTF-8.
958 *
959 * The _next_ time this happens, if the encoding that is about to become the
960 * explicitly set encoding does not match the previously set explicit
961 * encoding, a mixed encoding error will be emitted.
962 *
963 * When the expression is finished being lexed, the explicit encoding
964 * controls the encoding of the expression. For the most part this means
965 * that the expression will either be encoded in the source encoding or
966 * UTF-8. This holds for all encodings except US-ASCII. If the source is
967 * US-ASCII and an explicit encoding was set that was _not_ UTF-8, then the
968 * expression will be encoded as ASCII-8BIT.
969 *
970 * Note that if the expression is a list, different elements within the same
971 * list can have different encodings, so this will get reset between each
972 * element. Furthermore all of this only applies to lists that support
973 * interpolation, because otherwise escapes that could change the encoding
974 * are ignored.
975 *
976 * At first glance, it may make more sense for this to live on the lexer
977 * mode, but we need it here to communicate back to the parser for character
978 * literals that do not push a new lexer mode.
979 */
980 const pm_encoding_t *explicit_encoding;
981
982 /*
983 * When parsing block exits (e.g., break, next, redo), we need to validate
984 * that they are in correct contexts. For the most part we can do this by
985 * looking at our parent contexts. However, modifier while and until
986 * expressions can change that context to make block exits valid. In these
987 * cases, we need to keep track of the block exits and then validate them
988 * after the expression has been parsed.
989 *
990 * We use a pointer here because we don't want to keep a whole list attached
991 * since this will only be used in the context of begin/end expressions.
992 */
993 pm_scratch_node_list_t *current_block_exits;
994
995 /*
996 * The address of the C stack when pm_parse was entered. The stack budget
997 * is measured as the distance from this address.
998 */
999 uintptr_t stack_base;
1000
1001 /*
1002 * The number of bytes of C stack below stack_base that the recursive
1003 * descent may use before it reports that the stack is exhausted. SIZE_MAX
1004 * when no budget was given, so the check never fires.
1005 */
1006 size_t stack_budget;
1007
1008 /* The version of prism that we should use to parse. */
1009 pm_options_version_t version;
1010
1011 /* The command line flags given from the options. */
1012 uint8_t command_line;
1013
1014 /*
1015 * Whether or not we have found a frozen_string_literal magic comment with
1016 * a true or false value.
1017 * May be:
1018 * - PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
1019 * - PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
1020 * - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
1021 */
1022 int8_t frozen_string_literal;
1023
1024 /*
1025 * Whether or not we are parsing an eval string. This impacts whether or not
1026 * we should evaluate if block exits/yields are valid.
1027 */
1028 bool parsing_eval;
1029
1030 /*
1031 * Whether or not we are parsing a "partial" script, which is a script that
1032 * will be evaluated in the context of another script, so we should not
1033 * check jumps (next/break/etc.) for validity.
1034 */
1035 bool partial_script;
1036
1037 /*
1038 * Whether or not comments should be attached to their owning nodes once
1039 * parsing has finished.
1040 */
1041 bool attach_comments;
1042
1043 /* Whether or not we're at the beginning of a command. */
1044 bool command_start;
1045
1046 /*
1047 * Whether or not we're currently parsing the body of an endless method
1048 * definition. In this context, PM_TOKEN_KEYWORD_DO_BLOCK should not be
1049 * consumed by commands (it should bubble up to the outer context).
1050 */
1051 bool in_endless_def_body;
1052
1053 /* Whether or not we're currently recovering from a syntax error. */
1054 bool recovering;
1055
1056 /*
1057 * Whether or not the source being parsed could become valid if more input
1058 * were appended. This is set to false when the parser encounters a token
1059 * that is definitively wrong (e.g., a stray `end` or `]`) as opposed to
1060 * merely incomplete.
1061 */
1062 bool continuable;
1063
1064 /*
1065 * This is very specialized behavior for when you want to parse in a context
1066 * that does not respect encoding comments. Its main use case is translating
1067 * into the whitequark/parser AST which re-encodes source files in UTF-8
1068 * before they are parsed and ignores encoding comments.
1069 */
1070 bool encoding_locked;
1071
1072 /*
1073 * Whether or not the encoding has been changed by a magic comment. We use
1074 * this to provide a fast path for the lexer instead of going through the
1075 * function pointer.
1076 */
1077 bool encoding_changed;
1078
1079 /*
1080 * This flag indicates that we are currently parsing a pattern matching
1081 * expression and impacts that calculation of newlines.
1082 */
1083 bool pattern_matching_newlines;
1084
1085 /* This flag indicates that we are currently parsing a keyword argument. */
1086 bool in_keyword_arg;
1087
1088 /*
1089 * Whether or not the parser has seen a token that has semantic meaning
1090 * (i.e., a token that is not a comment or whitespace).
1091 */
1092 bool semantic_token_seen;
1093
1094 /*
1095 * By default, Ruby always warns about mismatched indentation. This can be
1096 * toggled with a magic comment.
1097 */
1098 bool warn_mismatched_indentation;
1099
1100 /*
1101 * Whether the stack exhausted error has been reported. Once the budget is
1102 * used up, every recursion that unwinds past it would report the error
1103 * again, so it is only reported the first time.
1104 */
1105 bool stack_exhausted_reported;
1106
1107#if defined(PRISM_HAS_NEON) || defined(PRISM_HAS_SSE2) || defined(PRISM_HAS_SWAR)
1108 /*
1109 * Cached lookup tables for pm_strpbrk's SIMD fast path. Avoids rebuilding
1110 * the nibble-based tables on every call when the charset hasn't changed
1111 * (which is the common case during string/regex/list lexing).
1112 */
1113 struct {
1114 /* The cached charset (null-terminated, max 11 chars + NUL). */
1115 uint8_t charset[12];
1116
1117 /* Nibble-based low lookup table for SIMD matching. */
1118 uint8_t low_lut[16];
1119
1120 /* Nibble-based high lookup table for SIMD matching. */
1121 uint8_t high_lut[16];
1122
1123 /* Scalar fallback table (4 x 64-bit bitmasks covering all ASCII). */
1124 uint64_t table[4];
1125 } strpbrk_cache;
1126#endif
1127};
1128
1129/*
1130 * Initialize a parser with the given start and end pointers.
1131 */
1132void pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options);
1133
1134/*
1135 * Free the memory held by the given parser.
1136 *
1137 * This does not free the `pm_options_t` object that was used to initialize the
1138 * parser.
1139 */
1140void pm_parser_cleanup(pm_parser_t *parser);
1141
1142/*
1143 * Returns true if the bytes between start and end spell a keyword, using the
1144 * same lookup that the lexer does.
1145 */
1146bool pm_keyword_p(const uint8_t *start, const uint8_t *end);
1147
1148/*
1149 * Returns true if parsing has used more C stack since pm_parse was entered than
1150 * the stack budget allows. Every recursion that runs inside pm_parse checks
1151 * this alongside its own depth limit, so that the budget bounds all of them.
1152 * The distance is taken without assuming which direction the stack grows. With
1153 * no budget the limit is SIZE_MAX, so this never fires.
1154 */
1155static PRISM_INLINE bool
1156pm_parser_stack_exhausted(const pm_parser_t *parser) {
1157 uintptr_t current = pm_stack_address();
1158 uintptr_t used = current < parser->stack_base ? parser->stack_base - current : current - parser->stack_base;
1159 return used > parser->stack_budget;
1160}
1161
1162/*
1163 * Reports that the stack budget has been used up, at the given location. This
1164 * is a separate error from the nesting depth limit, with its own error level,
1165 * because the source may be valid and parse with more stack. It is reported
1166 * once per parse.
1167 */
1168void pm_parser_err_stack_exhausted(pm_parser_t *parser, uint32_t start, uint32_t length);
1169
1170#endif
uint32_t pm_constant_id_t
A constant id is a unique identifier for a constant in the constant pool.
#define PRISM_INLINE
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
Definition inline.h:12
A list of byte offsets of newlines in a string.
The parser used to parse Ruby source.
void(* pm_lex_callback_t)(pm_parser_t *parser, pm_token_t *token, void *data)
This is the callback that is called when a token is lexed.
Definition parser.h:59
void(* pm_encoding_changed_callback_t)(pm_parser_t *parser)
When the encoding that is being used to parse the source is changed by prism, we provide the ability ...
Definition parser.h:53
static PRISM_INLINE uintptr_t pm_stack_address(void)
Returns the current position of the C stack, for measuring how much of it has been consumed since an ...
Definition stack.h:34
C99 shim for <stdbool.h>.
A list of offsets of the start of lines in a string.
This struct represents a slice in the source code, defined by an offset and a length.
Definition ast.h:579
A view of a sequence of bytes.
Definition stringy.h:23
This struct represents a token in the Ruby source.
Definition ast.h:551