class RDoc::RubyLex
Ruby lexer adapted from irb.
The internals are not documented because they are scary.
Public Class Methods
new(content, options)
click to toggle source
Creates a new lexer for content
. options
is an
RDoc::Options, only +tab_width is used.
# File lib/rdoc/ruby_lex.rb, line 81 def initialize(content, options) lex_init if /\t/ =~ content then tab_width = options.tab_width content = content.split(/\n/).map do |line| 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width) } && $~ line end.join("\n") end content << "\n" unless content[-1, 1] == "\n" set_input StringIO.new content @base_char_no = 0 @char_no = 0 @exp_line_no = @line_no = 1 @here_readed = [] @readed = [] @rests = [] @seek = 0 @here_header = false @indent = 0 @indent_stack = [] @lex_state = :EXPR_BEG @space_seen = false @continue = false @line = "" @skip_space = false @readed_auto_clean_up = false @exception_on_syntax_error = true @prompt = nil @prev_seek = nil @ltype = nil end
tokenize(ruby, options)
click to toggle source
Returns an Array of ruby
tokens. See ::new for a description of
options
.
# File lib/rdoc/ruby_lex.rb, line 64 def self.tokenize ruby, options tokens = [] scanner = RDoc::RubyLex.new ruby, options scanner.exception_on_syntax_error = true while token = scanner.token do tokens << token end tokens end