class RDoc::Parser::Ruby

# File lib/rdoc/parser/ruby.rb, line 164
def initialize(top_level, file_name, content, options, stats)
  super

  if /\t/ =~ content then
    tab_width = @options.tab_width
    content = content.split(/\n/).map do |line|
      1 while line.gsub!(/\t+/) {
        ' ' * (tab_width*$&.length - $`.length % tab_width)
      }  && $~
      line
    end.join("\n")
  end

  @size = 0
  @token_listeners = nil
  content = RDoc::Encoding.remove_magic_comment content
  @scanner = RDoc::Parser::RipperStateLex.parse(content)
  @content = content
  @scanner_point = 0
  @prev_seek = nil
  @markup = @options.markup
  @track_visibility = :nodoc != @options.visibility
  @encoding = @options.encoding

  reset
end

# File lib/rdoc/parser/ruby.rb, line 241
def collect_first_comment
  skip_tkspace
  comment = ''.dup
  comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
  first_line = true
  first_comment_tk_kind = nil
  line_no = nil

  tk = get_tk

  while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
    comment_body = retrieve_comment_body(tk)
    if first_line and comment_body =~ /\A#!/ then
      skip_tkspace
      tk = get_tk
    elsif first_line and comment_body =~ /\A#\s*-\*-/ then
      first_line = false
      skip_tkspace
      tk = get_tk
    else
      break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind]
      first_comment_tk_kind = tk[:kind]

      line_no = tk[:line_no] if first_line
      first_line = false
      comment << comment_body
      tk = get_tk

      if :on_nl === tk then
        skip_tkspace_without_nl
        tk = get_tk
      end
    end
  end

  unget_tk tk

  new_comment comment, line_no
end

# File lib/rdoc/parser/ruby.rb, line 318
def error(msg)
  msg = make_message msg

  abort msg
end

# File lib/rdoc/parser/ruby.rb, line 327
def get_bool
  skip_tkspace
  tk = get_tk
  if :on_kw == tk[:kind] && 'true' == tk[:text]
    true
  elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text])
    false
  else
    unget_tk tk
    true
  end
end

# File lib/rdoc/parser/ruby.rb, line 345
def get_class_or_module container, ignore_constants = false
  skip_tkspace
  name_t = get_tk
  given_name = ''.dup

  # class ::A -> A is in the top level
  if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug
    name_t = get_tk
    container = @top_level
    given_name << '::'
  end

  skip_tkspace_without_nl
  given_name << name_t[:text]

  is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
  new_modules = []
  while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do
    prev_container = container
    container = container.find_module_named name_t[:text]
    container ||=
      if ignore_constants then
        c = RDoc::NormalModule.new name_t[:text]
        c.store = @store
        new_modules << [prev_container, c]
        c
      else
        c = prev_container.add_module RDoc::NormalModule, name_t[:text]
        c.ignore unless prev_container.document_children
        @top_level.add_to_classes_or_modules c
        c
      end

    record_location container

    get_tk
    skip_tkspace
    if :on_lparen == peek_tk[:kind] # ProcObjectInConstant::()
      parse_method_or_yield_parameters
      break
    end
    name_t = get_tk
    unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
      raise RDoc::Error, "Invalid class or module definition: #{given_name}"
    end
    if prev_container == container and !ignore_constants
      given_name = name_t[:text]
    else
      given_name << '::' + name_t[:text]
    end
  end

  skip_tkspace_without_nl

  return [container, name_t, given_name, new_modules]
end

# File lib/rdoc/parser/ruby.rb, line 405
def get_class_specification
  tk = peek_tk
  if tk.nil?
    return ''
  elsif :on_kw == tk[:kind] && 'self' == tk[:text]
    return 'self'
  elsif :on_gvar == tk[:kind]
    return ''
  end

  res = get_constant

  skip_tkspace_without_nl

  get_tkread # empty out read buffer

  tk = get_tk
  return res unless tk

  case tk[:kind]
  when :on_nl, :on_comment, :on_embdoc, :on_semicolon then
    unget_tk(tk)
    return res
  end

  res += parse_call_parameters(tk)
  res
end

# File lib/rdoc/parser/ruby.rb, line 438
def get_constant
  res = ""
  skip_tkspace_without_nl
  tk = get_tk

  while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
    res += tk[:text]
    tk = get_tk
  end

  unget_tk(tk)
  res
end

# File lib/rdoc/parser/ruby.rb, line 455
def get_included_module_with_optional_parens
  skip_tkspace_without_nl
  get_tkread
  tk = get_tk
  end_token = get_end_token tk
  return '' unless end_token

  nest = 0
  continue = false
  only_constant = true

  while tk != nil do
    is_element_of_constant = false
    case tk[:kind]
    when :on_semicolon then
      break if nest == 0
    when :on_lbracket then
      nest += 1
    when :on_rbracket then
      nest -= 1
    when :on_lbrace then
      nest += 1
    when :on_rbrace then
      nest -= 1
      if nest <= 0
        # we might have a.each { |i| yield i }
        unget_tk(tk) if nest < 0
        break
      end
    when :on_lparen then
      nest += 1
    when end_token[:kind] then
      if end_token[:kind] == :on_rparen
        nest -= 1
        break if nest <= 0
      else
        break if nest <= 0
      end
    when :on_rparen then
      nest -= 1
    when :on_comment, :on_embdoc then
      @read.pop
      if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
        (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then
        break if !continue and nest <= 0
      end
    when :on_comma then
      continue = true
    when :on_ident then
      continue = false if continue
    when :on_kw then
      case tk[:text]
      when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
        nest += 1
      when 'if', 'unless', 'while', 'until', 'rescue'
        # postfix if/unless/while/until/rescue must be EXPR_LABEL
        nest += 1 unless (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0
      when 'end'
        nest -= 1
        break if nest == 0
      end
    when :on_const then
      is_element_of_constant = true
    when :on_op then
      is_element_of_constant = true if '::' == tk[:text]
    end
    only_constant = false unless is_element_of_constant
    tk = get_tk
  end

  if only_constant
    get_tkread_clean(/\s+/, ' ')
  else
    ''
  end
end

# File lib/rdoc/parser/ruby.rb, line 603
def get_symbol_or_name
  tk = get_tk
  case tk[:kind]
  when :on_symbol then
    text = tk[:text].sub(/^:/, '')

    next_tk = peek_tk
    if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then
      get_tk
      text << '='
    end

    text
  when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then
    tk[:text]
  when :on_tstring, :on_dstring then
    tk[:text][1..-2]
  else
    raise RDoc::Error, "Name or symbol expected (got #{tk})"
  end
end

# File lib/rdoc/parser/ruby.rb, line 643
def look_for_directives_in container, comment
  @preprocess.handle comment, container do |directive, param|
    case directive
    when 'method', 'singleton-method',
         'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
      false # handled elsewhere
    when 'section' then
      break unless container.kind_of?(RDoc::Context)
      container.set_current_section param, comment.dup
      comment.text = ''
      break
    end
  end

  comment.remove_private
end

# File lib/rdoc/parser/ruby.rb, line 663
def make_message message
  prefix = "#{@file_name}:".dup

  tk = peek_tk
  prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk

  "#{prefix} #{message}"
end

# File lib/rdoc/parser/ruby.rb, line 675
def new_comment comment, line_no = nil
  c = RDoc::Comment.new comment, @top_level, :ruby
  c.line = line_no
  c.format = @markup
  c
end

# File lib/rdoc/parser/ruby.rb, line 744
def parse_alias(context, single, tk, comment)
  line_no = tk[:line_no]

  skip_tkspace

  if :on_lparen === peek_tk[:kind] then
    get_tk
    skip_tkspace
  end

  new_name = get_symbol_or_name

  skip_tkspace
  if :on_comma === peek_tk[:kind] then
    get_tk
    skip_tkspace
  end

  begin
    old_name = get_symbol_or_name
  rescue RDoc::Error
    return
  end

  al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
                       single == SINGLE)
  record_location al
  al.line   = line_no

  read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
  context.add_alias al
  @stats.add_alias al

  al
end

# File lib/rdoc/parser/ruby.rb, line 686
def parse_attr(context, single, tk, comment)
  line_no = tk[:line_no]

  args = parse_symbol_arg 1
  if args.size > 0 then
    name = args[0]
    rw = "R"
    skip_tkspace_without_nl
    tk = get_tk

    if :on_comma == tk[:kind] then
      rw = "RW" if get_bool
    else
      unget_tk tk
    end

    att = create_attr context, single, name, rw, comment
    att.line   = line_no

    read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
  else
    warn "'attr' ignored - looks like a variable"
  end
end

# File lib/rdoc/parser/ruby.rb, line 715
def parse_attr_accessor(context, single, tk, comment)
  line_no = tk[:line_no]

  args = parse_symbol_arg
  rw = "?"

  tmp = RDoc::CodeObject.new
  read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
  # TODO In most other places we let the context keep track of document_self
  # and add found items appropriately but here we do not.  I'm not sure why.
  return if @track_visibility and not tmp.document_self

  case tk[:text]
  when "attr_reader"   then rw = "R"
  when "attr_writer"   then rw = "W"
  when "attr_accessor" then rw = "RW"
  else
    rw = '?'
  end

  for name in args
    att = create_attr context, single, name, rw, comment
    att.line   = line_no
  end
end

# File lib/rdoc/parser/ruby.rb, line 783
def parse_call_parameters(tk)
  end_token = case tk[:kind]
              when :on_lparen
                :on_rparen
              when :on_rparen
                return ""
              else
                :on_nl
              end
  nest = 0

  loop do
    break if tk.nil?
    case tk[:kind]
    when :on_semicolon
      break
    when :on_lparen
      nest += 1
    when end_token
      if end_token == :on_rparen
        nest -= 1
        break if RDoc::Parser::RipperStateLex.end?(tk) and nest <= 0
      else
        break if RDoc::Parser::RipperStateLex.end?(tk)
      end
    when :on_comment, :on_embdoc
      unget_tk(tk)
      break
    when :on_op
      if tk[:text] =~ /^(.{1,2})?=$/
        unget_tk(tk)
        break
      end
    end
    tk = get_tk
  end

  get_tkread_clean "\n", " "
end

# File lib/rdoc/parser/ruby.rb, line 826
def parse_class container, single, tk, comment
  line_no = tk[:line_no]

  declaration_context = container
  container, name_t, given_name, = get_class_or_module container

  if name_t[:kind] == :on_const
    cls = parse_class_regular container, declaration_context, single,
      name_t, given_name, comment
  elsif name_t[:kind] == :on_op && name_t[:text] == '<<'
    case name = get_class_specification
    when 'self', container.name
      read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
      parse_statements container, SINGLE
      return # don't update line
    else
      cls = parse_class_singleton container, name, comment
    end
  else
    warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}"
    return
  end

  cls.line   = line_no

  # after end modifiers
  read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS

  cls
end

# File lib/rdoc/parser/ruby.rb, line 1065
def parse_comment container, tk, comment
  return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
  column  = tk[:char_no]
  line_no = comment.line.nil? ? tk[:line_no] : comment.line

  comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
  singleton = !!$~

  co =
    if (comment.text = comment.text.sub(/^# +:?method: *(\S*).*?\n/i, '')) && !!$~ then
      line_no += $`.count("\n")
      parse_comment_ghost container, comment.text, $1, column, line_no, comment
    elsif (comment.text = comment.text.sub(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '')) && !!$~ then
      parse_comment_attr container, $1, $3, comment
    end

  if co then
    co.singleton = singleton
    co.line      = line_no
  end

  true
end

# File lib/rdoc/parser/ruby.rb, line 1144
def parse_comment_tomdoc container, tk, comment
  return unless signature = RDoc::TomDoc.signature(comment)
  column  = tk[:char_no]
  line_no = tk[:line_no]

  name, = signature.split %r%[ \(]%, 2

  meth = RDoc::GhostMethod.new get_tkread, name
  record_location meth
  meth.line      = line_no

  meth.start_collecting_tokens
  indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
  position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
  position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
  newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
  meth.add_tokens [position_comment, newline, indent]

  meth.call_seq = signature

  comment.normalize

  return unless meth.name

  container.add_method meth

  meth.comment = comment

  @stats.add_method meth
end

# File lib/rdoc/parser/ruby.rb, line 939
def parse_constant container, tk, comment, ignore_constants = false
  line_no = tk[:line_no]

  name = tk[:text]
  skip_tkspace_without_nl

  return unless name =~ /^\w+$/

  new_modules = []
  if :on_op == peek_tk[:kind] && '::' == peek_tk[:text] then
    unget_tk tk

    container, name_t, _, new_modules = get_class_or_module container, true

    name = name_t[:text]
  end

  is_array_or_hash = false
  if peek_tk && :on_lbracket == peek_tk[:kind]
    get_tk
    nest = 1
    while bracket_tk = get_tk
      case bracket_tk[:kind]
      when :on_lbracket
        nest += 1
      when :on_rbracket
        nest -= 1
        break if nest == 0
      end
    end
    skip_tkspace_without_nl
    is_array_or_hash = true
  end

  unless peek_tk && :on_op == peek_tk[:kind] && '=' == peek_tk[:text] then
    return false
  end
  get_tk

  unless ignore_constants
    new_modules.each do |prev_c, new_module|
      prev_c.add_module_by_normal_module new_module
      new_module.ignore unless prev_c.document_children
      @top_level.add_to_classes_or_modules new_module
    end
  end

  value = ''
  con = RDoc::Constant.new name, value, comment

  body = parse_constant_body container, con, is_array_or_hash

  return unless body

  con.value = body
  record_location con
  con.line   = line_no
  read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS

  return if is_array_or_hash

  @stats.add_constant con
  container.add_constant con

  true
end

# File lib/rdoc/parser/ruby.rb, line 2076
def parse_constant_visibility(container, single, tk)
  args = parse_symbol_arg
  case tk[:text]
  when 'private_constant'
    vis = :private
  when 'public_constant'
    vis = :public
  else
    raise RDoc::Error, 'Unreachable'
  end
  container.set_constant_visibility_for args, vis
end

# File lib/rdoc/parser/ruby.rb, line 1281
def parse_meta_attr(context, single, tk, comment)
  args = parse_symbol_arg
  rw = "?"

  # If nodoc is given, don't document any of them

  tmp = RDoc::CodeObject.new
  read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS

  regexp = /^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i
  if regexp =~ comment.text then
    comment.text = comment.text.sub(regexp, '')
    rw = case $1
         when 'attr_reader' then 'R'
         when 'attr_writer' then 'W'
         else 'RW'
         end
    name = $3 unless $3.empty?
  end

  if name then
    att = create_attr context, single, name, rw, comment
  else
    args.each do |attr_name|
      att = create_attr context, single, attr_name, rw, comment
    end
  end

  att
end

# File lib/rdoc/parser/ruby.rb, line 1315
def parse_meta_method(container, single, tk, comment)
  column  = tk[:char_no]
  line_no = tk[:line_no]

  start_collecting_tokens
  add_token tk
  add_token_listener self

  skip_tkspace_without_nl

  comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
  singleton = !!$~

  name = parse_meta_method_name comment, tk

  return unless name

  meth = RDoc::MetaMethod.new get_tkread, name
  record_location meth
  meth.line   = line_no
  meth.singleton = singleton

  remove_token_listener self

  meth.start_collecting_tokens
  indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
  position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
  position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
  newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
  meth.add_tokens [position_comment, newline, indent]
  meth.add_tokens @token_stream

  parse_meta_method_params container, single, meth, tk, comment

  meth.comment = comment

  @stats.add_method meth

  meth
end

# File lib/rdoc/parser/ruby.rb, line 1417
def parse_method(container, single, tk, comment)
  singleton = nil
  added_container = false
  name = nil
  column  = tk[:char_no]
  line_no = tk[:line_no]

  start_collecting_tokens
  add_token tk

  token_listener self do
    prev_container = container
    name, container, singleton = parse_method_name container
    added_container = container != prev_container
  end

  return unless name

  meth = RDoc::AnyMethod.new get_tkread, name
  look_for_directives_in meth, comment
  meth.singleton = single == SINGLE ? true : singleton

  record_location meth
  meth.line   = line_no

  meth.start_collecting_tokens
  indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
  token = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
  token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
  newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
  meth.add_tokens [token, newline, indent]
  meth.add_tokens @token_stream

  parse_method_params_and_body container, single, meth, added_container

  comment.normalize
  comment.extract_call_seq meth

  meth.comment = comment

  # after end modifiers
  read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS

  @stats.add_method meth
end

# File lib/rdoc/parser/ruby.rb, line 1496
def parse_method_dummy container
  dummy = RDoc::Context.new
  dummy.parent = container
  dummy.store  = container.store
  skip_method dummy
end

# File lib/rdoc/parser/ruby.rb, line 1598
def parse_method_or_yield_parameters(method = nil,
                                     modifiers = RDoc::METHOD_MODIFIERS)
  skip_tkspace_without_nl
  tk = get_tk
  end_token = get_end_token tk
  return '' unless end_token

  nest = 0
  continue = false

  while tk != nil do
    case tk[:kind]
    when :on_semicolon then
      break if nest == 0
    when :on_lbracket then
      nest += 1
    when :on_rbracket then
      nest -= 1
    when :on_lbrace then
      nest += 1
    when :on_rbrace then
      nest -= 1
      if nest <= 0
        # we might have a.each { |i| yield i }
        unget_tk(tk) if nest < 0
        break
      end
    when :on_lparen then
      nest += 1
    when end_token[:kind] then
      if end_token[:kind] == :on_rparen
        nest -= 1
        break if nest <= 0
      else
        break
      end
    when :on_rparen then
      nest -= 1
    when :on_comment, :on_embdoc then
      @read.pop
      if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
        (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then
        if method && method.block_params.nil? then
          unget_tk tk
          read_documentation_modifiers method, modifiers
        end
        break if !continue and nest <= 0
      end
    when :on_comma then
      continue = true
    when :on_ident then
      continue = false if continue
    end
    tk = get_tk
  end

  get_tkread_clean(/\s+/, ' ')
end

# File lib/rdoc/parser/ruby.rb, line 1665
def parse_method_parameters method
  res = parse_method_or_yield_parameters method

  res = "(#{res})" unless res =~ /\A\(/
  method.params = res unless method.params

  return if  method.block_params

  skip_tkspace_without_nl
  read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
end

# File lib/rdoc/parser/ruby.rb, line 1466
def parse_method_params_and_body container, single, meth, added_container
  token_listener meth do
    parse_method_parameters meth

    if meth.document_self or not @track_visibility then
      container.add_method meth
    elsif added_container then
      container.document_self = false
    end

    # Having now read the method parameters and documentation modifiers, we
    # now know whether we have to rename #initialize to ::new

    if meth.name == "initialize" && !meth.singleton then
      if meth.dont_rename_initialize then
        meth.visibility = :protected
      else
        meth.singleton = true
        meth.name = "new"
        meth.visibility = :public
      end
    end

    parse_statements container, single, meth
  end
end

# File lib/rdoc/parser/ruby.rb, line 1680
def parse_module container, single, tk, comment
  container, name_t, = get_class_or_module container

  name = name_t[:text]

  mod = container.add_module RDoc::NormalModule, name
  mod.ignore unless container.document_children
  record_location mod

  read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
  mod.add_comment comment, @top_level
  parse_statements mod

  # after end modifiers
  read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS

  @stats.add_module mod
end

# File lib/rdoc/parser/ruby.rb, line 1702
def parse_require(context, comment)
  skip_tkspace_comment
  tk = get_tk

  if :on_lparen == tk[:kind] then
    skip_tkspace_comment
    tk = get_tk
  end

  name = tk[:text][1..-2] if :on_tstring == tk[:kind]

  if name then
    @top_level.add_require RDoc::Require.new(name, comment)
  else
    unget_tk tk
  end
end

# File lib/rdoc/parser/ruby.rb, line 1723
def parse_rescue
  skip_tkspace_without_nl

  while tk = get_tk
    case tk[:kind]
    when :on_nl, :on_semicolon, :on_comment then
      break
    when :on_comma then
      skip_tkspace_without_nl

      get_tk if :on_nl == peek_tk[:kind]
    end

    skip_tkspace_without_nl
  end
end

# File lib/rdoc/parser/ruby.rb, line 1754
def parse_statements(container, single = NORMAL, current_method = nil,
                     comment = new_comment(''))
  raise 'no' unless RDoc::Comment === comment
  comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding

  nest = 1
  save_visibility = container.visibility

  non_comment_seen = true

  while tk = get_tk do
    keep_comment = false
    try_parse_comment = false

    non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])

    case tk[:kind]
    when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then
      if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
        skip_tkspace
        tk = get_tk
      else
        past_tokens = @read.size > 1 ? @read[0..-2] : []
        nl_position = 0
        past_tokens.reverse.each_with_index do |read_tk, i|
          if read_tk =~ /^\n$/ then
            nl_position = (past_tokens.size - 1) - i
            break
          elsif read_tk =~ /^#.*\n$/ then
            nl_position = ((past_tokens.size - 1) - i) + 1
            break
          end
        end
        comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ }
        unless comment_only_line then
          tk = get_tk
        end
      end

      if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
        if non_comment_seen then
          # Look for RDoc in a comment about to be thrown away
          non_comment_seen = parse_comment container, tk, comment unless
            comment.empty?

          comment = ''
          comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
        end

        line_no = nil
        while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do
          comment_body = retrieve_comment_body(tk)
          line_no = tk[:line_no] if comment.empty?
          comment += comment_body
          comment << "\n" unless comment_body =~ /\n\z/

          if comment_body.size > 1 && comment_body =~ /\n\z/ then
            skip_tkspace_without_nl # leading spaces
          end
          tk = get_tk
        end

        comment = new_comment comment, line_no

        unless comment.empty? then
          look_for_directives_in container, comment

          if container.done_documenting then
            throw :eof if RDoc::TopLevel === container
            container.ongoing_visibility = save_visibility
          end
        end

        keep_comment = true
      else
        non_comment_seen = true
      end

      unget_tk tk
      keep_comment = true
      container.current_line_visibility = nil

    when :on_kw then
      case tk[:text]
      when 'class' then
        parse_class container, single, tk, comment

      when 'module' then
        parse_module container, single, tk, comment

      when 'def' then
        parse_method container, single, tk, comment

      when 'alias' then
        parse_alias container, single, tk, comment unless current_method

      when 'yield' then
        if current_method.nil? then
          warn "Warning: yield outside of method" if container.document_self
        else
          parse_yield container, single, tk, current_method
        end

      when 'until', 'while' then
        if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
          nest += 1
          skip_optional_do_after_expression
        end

      # Until and While can have a 'do', which shouldn't increase the nesting.
      # We can't solve the general case, but we can handle most occurrences by
      # ignoring a do at the end of a line.

      # 'for' is trickier
      when 'for' then
        nest += 1
        skip_for_variable
        skip_optional_do_after_expression

      when 'case', 'do', 'if', 'unless', 'begin' then
        if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
          nest += 1
        end

      when 'super' then
        current_method.calls_super = true if current_method

      when 'rescue' then
        parse_rescue

      when 'end' then
        nest -= 1
        if nest == 0 then
          container.ongoing_visibility = save_visibility

          parse_comment container, tk, comment unless comment.empty?

          return
        end
      end

    when :on_const then
      unless parse_constant container, tk, comment, current_method then
        try_parse_comment = true
      end

    when :on_ident then
      if nest == 1 and current_method.nil? then
        keep_comment = parse_identifier container, single, tk, comment
      end

      case tk[:text]
      when "require" then
        parse_require container, comment
      when "include" then
        parse_extend_or_include RDoc::Include, container, comment
      when "extend" then
        parse_extend_or_include RDoc::Extend, container, comment
      when "included" then
        parse_included_with_activesupport_concern container, comment
      end

    else
      try_parse_comment = nest == 1
    end

    if try_parse_comment then
      non_comment_seen = parse_comment container, tk, comment unless
        comment.empty?

      keep_comment = false
    end

    unless keep_comment then
      comment = new_comment ''
      comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
      container.params = nil
      container.block_params = nil
    end

    consume_trailing_spaces
  end

  container.params = nil
  container.block_params = nil
end

# File lib/rdoc/parser/ruby.rb, line 1944
def parse_symbol_arg(no = nil)
  skip_tkspace_comment

  tk = get_tk
  if tk[:kind] == :on_lparen
    parse_symbol_arg_paren no
  else
    parse_symbol_arg_space no, tk
  end
end

# File lib/rdoc/parser/ruby.rb, line 2018
def parse_symbol_in_arg
  tk = get_tk
  if :on_symbol == tk[:kind] then
    tk[:text].sub(/^:/, '')
  elsif :on_tstring == tk[:kind] then
    tk[:text][1..-2]
  elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then
    nil # ignore
  else
    warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
    nil
  end
end

# File lib/rdoc/parser/ruby.rb, line 2035
def parse_top_level_statements container
  comment = collect_first_comment

  look_for_directives_in container, comment

  throw :eof if container.done_documenting

  @markup = comment.format

  # HACK move if to RDoc::Context#comment=
  container.comment = comment if container.document_self unless comment.empty?

  parse_statements container, NORMAL, nil, comment
end

# File lib/rdoc/parser/ruby.rb, line 2053
def parse_visibility(container, single, tk)
  vis_type, vis, singleton = get_visibility_information tk, single

  skip_tkspace_comment false

  ptk = peek_tk
  # Ryan Davis suggested the extension to ignore modifiers, because he
  # often writes
  #
  #   protected unless $TESTING
  #
  if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then
    container.ongoing_visibility = vis
  elsif :on_kw == ptk[:kind] && 'def' == ptk[:text]
    container.current_line_visibility = vis
  else
    update_visibility container, vis_type, vis, singleton
  end
end

# File lib/rdoc/parser/ruby.rb, line 2092
def parse_yield(context, single, tk, method)
  return if method.block_params

  get_tkread
  method.block_params = parse_method_or_yield_parameters
end

# File lib/rdoc/parser/ruby.rb, line 2113
def read_directive allowed
  tokens = []

  while tk = get_tk do
    tokens << tk

    if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then
      return
    elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then
      return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/

      directive = $1.downcase

      return [directive, $2] if allowed.include? directive

      return
    end
  end
ensure
  unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then
    tokens.reverse_each do |token|
      unget_tk token
    end
  end
end

# File lib/rdoc/parser/ruby.rb, line 2145
def read_documentation_modifiers context, allowed
  skip_tkspace_without_nl
  directive, value = read_directive allowed

  return unless directive

  @preprocess.handle_directive '', directive, value, context do |dir, param|
    if %w[notnew not_new not-new].include? dir then
      context.dont_rename_initialize = true

      true
    end
  end
end

# File lib/rdoc/parser/ruby.rb, line 1743
def retrieve_comment_body(tk)
  if :on_embdoc == tk[:kind]
    tk[:text].gsub(/\A=begin.*\n/, '').gsub(/=end\n?\z/, '')
  else
    tk[:text]
  end
end

# File lib/rdoc/parser/ruby.rb, line 2176
  def scan
    reset

    catch :eof do
      begin
        parse_top_level_statements @top_level

      rescue StandardError => e
        if @content.include?('<%') and @content.include?('%>') then
          # Maybe, this is ERB.
          $stderr.puts "\033[2KRDoc detects ERB file. Skips it for compatibility:"
          $stderr.puts @file_name
          return
        end

        if @scanner_point >= @scanner.size
          now_line_no = @scanner[@scanner.size - 1][:line_no]
        else
          now_line_no = peek_tk[:line_no]
        end
        first_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no }
        last_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 }
        last_tk_index = last_tk_index ? last_tk_index - 1 : @scanner.size - 1
        code = @scanner[first_tk_index..last_tk_index].map{ |t| t[:text] }.join

        $stderr.puts <<-EOF

#{self.class} failure around line #{now_line_no} of
#{@file_name}

        EOF

        unless code.empty? then
          $stderr.puts code
          $stderr.puts
        end

        raise e
      end
    end

    @top_level
  end

# File lib/rdoc/parser/ruby.rb, line 2264
def skip_for_variable
  skip_tkspace_without_nl
  get_tk
  skip_tkspace_without_nl
  tk = get_tk
  unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
end

# File lib/rdoc/parser/ruby.rb, line 2275
def skip_method container
  meth = RDoc::AnyMethod.new "", "anon"
  parse_method_parameters meth
  parse_statements container, false, meth
end

# File lib/rdoc/parser/ruby.rb, line 2223
def skip_optional_do_after_expression
  skip_tkspace_without_nl
  tk = get_tk

  b_nest = 0
  nest = 0

  loop do
    break unless tk
    case tk[:kind]
    when :on_semicolon, :on_nl, :on_ignored_nl then
      break if b_nest.zero?
    when :on_lparen then
      nest += 1
    when :on_rparen then
      nest -= 1
    when :on_kw then
      case tk[:text]
      when 'begin'
        b_nest += 1
      when 'end'
        b_nest -= 1
      when 'do'
        break if nest.zero?
      end
    when :on_comment, :on_embdoc then
      if b_nest.zero? and "\n" == tk[:text][-1] then
        break
      end
    end
    tk = get_tk
  end

  skip_tkspace_without_nl

  get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
end

# File lib/rdoc/parser/ruby.rb, line 2284
def skip_tkspace_comment(skip_nl = true)
  loop do
    skip_nl ? skip_tkspace : skip_tkspace_without_nl
    next_tk = peek_tk
    return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
    get_tk
  end
end

# File lib/rdoc/parser/ruby.rb, line 191
def tk_nl?(tk)
  :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
end

# File lib/rdoc/parser/ruby.rb, line 2341
def warn message
  @options.warn make_message message
end