Skip to content

Commit

Permalink
Drop reimplementation of Ripper lex state
Browse files Browse the repository at this point in the history 
This code was for ruby 2.4 compatibility, but rdoc dropped support for
ruby 2.4 about three years ago, in f480b97.  This code was almost half
of the lines of code in rdoc/parser/ripper_state_lex.
  • Loading branch information
@nevans
nevans committed Jun 6, 2024
1 parent 4614f05 commit 5772f60
Showing 1 changed file with 1 addition and 285 deletions.
286 changes: 1 addition & 285 deletions lib/rdoc/parser/ripper_state_lex.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,6 @@
class RDoc::Parser::RipperStateLex
# :stopdoc:

# TODO: Remove this constants after Ruby 2.4 EOL
RIPPER_HAS_LEX_STATE = Ripper::Filter.method_defined?(:state)

Token = Struct.new(:line_no, :char_no, :kind, :text, :state)

EXPR_NONE = 0
Expand All @@ -31,284 +28,6 @@ class RDoc::Parser::RipperStateLex
EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG)
EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)

class InnerStateLex < Ripper::Filter
attr_accessor :lex_state

def initialize(code)
@lex_state = EXPR_BEG
@in_fname = false
@continue = false
reset
super(code)
end

def reset
@command_start = false
@cmd_state = @command_start
end

def on_nl(tok, data)
case @lex_state
when EXPR_FNAME, EXPR_DOT
@continue = true
else
@continue = false
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
end
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_ignored_nl(tok, data)
case @lex_state
when EXPR_FNAME, EXPR_DOT
@continue = true
else
@continue = false
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
end
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_op(tok, data)
case tok
when '&', '|', '!', '!=', '!~'
case @lex_state
when EXPR_FNAME, EXPR_DOT
@lex_state = EXPR_ARG
else
@lex_state = EXPR_BEG
end
when '<<'
# TODO next token?
case @lex_state
when EXPR_FNAME, EXPR_DOT
@lex_state = EXPR_ARG
else
@lex_state = EXPR_BEG
end
when '?'
@lex_state = EXPR_BEG
when '&&', '||', '+=', '-=', '*=', '**=',
'&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
@lex_state = EXPR_BEG
when '::'
case @lex_state
when EXPR_ARG, EXPR_CMDARG
@lex_state = EXPR_DOT
when EXPR_FNAME, EXPR_DOT
@lex_state = EXPR_ARG
else
@lex_state = EXPR_BEG
end
else
case @lex_state
when EXPR_FNAME, EXPR_DOT
@lex_state = EXPR_ARG
else
@lex_state = EXPR_BEG
end
end
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_kw(tok, data)
case tok
when 'class'
@lex_state = EXPR_CLASS
@in_fname = true
when 'def'
@lex_state = EXPR_FNAME
@continue = true
@in_fname = true
when 'if', 'unless', 'while', 'until'
if ((EXPR_MID | EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
@lex_state = EXPR_BEG | EXPR_LABEL
else
@lex_state = EXPR_BEG
end
when 'begin', 'case', 'when'
@lex_state = EXPR_BEG
when 'return', 'break'
@lex_state = EXPR_MID
else
if @lex_state == EXPR_FNAME
@lex_state = EXPR_END
else
@lex_state = EXPR_END
end
end
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_tstring_beg(tok, data)
@lex_state = EXPR_BEG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_tstring_end(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_CHAR(tok, data)
@lex_state = EXPR_END
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_period(tok, data)
@lex_state = EXPR_DOT
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_int(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_float(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_rational(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_imaginary(tok, data)
@lex_state = EXPR_END | EXPR_ENDARG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_symbeg(tok, data)
@lex_state = EXPR_FNAME
@continue = true
@in_fname = true
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

private def on_variables(event, tok, data)
if @in_fname
@lex_state = EXPR_ENDFN
@in_fname = false
@continue = false
elsif @continue
case @lex_state
when EXPR_DOT
@lex_state = EXPR_ARG
else
@lex_state = EXPR_ENDFN
@continue = false
end
else
@lex_state = EXPR_CMDARG
end
data << Token.new(lineno, column, event, tok, @lex_state)
end

def on_ident(tok, data)
on_variables(__method__, tok, data)
end

def on_ivar(tok, data)
@lex_state = EXPR_END
on_variables(__method__, tok, data)
end

def on_cvar(tok, data)
@lex_state = EXPR_END
on_variables(__method__, tok, data)
end

def on_gvar(tok, data)
@lex_state = EXPR_END
on_variables(__method__, tok, data)
end

def on_backref(tok, data)
@lex_state = EXPR_END
on_variables(__method__, tok, data)
end

def on_lparen(tok, data)
@lex_state = EXPR_LABEL | EXPR_BEG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_rparen(tok, data)
@lex_state = EXPR_ENDFN
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_lbrace(tok, data)
@lex_state = EXPR_LABEL | EXPR_BEG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_rbrace(tok, data)
@lex_state = EXPR_ENDARG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_lbracket(tok, data)
@lex_state = EXPR_LABEL | EXPR_BEG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_rbracket(tok, data)
@lex_state = EXPR_ENDARG
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_const(tok, data)
case @lex_state
when EXPR_FNAME
@lex_state = EXPR_ENDFN
when EXPR_CLASS, EXPR_CMDARG, EXPR_MID
@lex_state = EXPR_ARG
else
@lex_state = EXPR_CMDARG
end
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_sp(tok, data)
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_comma(tok, data)
@lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_comment(tok, data)
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_ignored_sp(tok, data)
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
data << Token.new(lineno, column, __method__, tok, @lex_state)
end

def on_heredoc_beg(tok, data)
data << Token.new(lineno, column, __method__, tok, @lex_state)
@lex_state = EXPR_END
data
end

def on_heredoc_end(tok, data)
data << Token.new(lineno, column, __method__, tok, @lex_state)
@lex_state = EXPR_BEG
data
end

def on_default(event, tok, data)
reset
data << Token.new(lineno, column, event, tok, @lex_state)
end
end unless RIPPER_HAS_LEX_STATE

class InnerStateLex < Ripper::Filter
def initialize(code)
super(code)
Expand All @@ -317,7 +36,7 @@ def initialize(code)
def on_default(event, tok, data)
data << Token.new(lineno, column, event, tok, state)
end
end if RIPPER_HAS_LEX_STATE
end

def get_squashed_tk
if @buf.empty?
Expand All @@ -333,7 +52,6 @@ def get_squashed_tk
tk = get_string_tk(tk)
when :on_backtick then
if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
@inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
tk[:kind] = :on_ident
tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
else
Expand All @@ -345,7 +63,6 @@ def get_squashed_tk
tk = get_embdoc_tk(tk)
when :on_heredoc_beg then
@heredoc_queue << retrieve_heredoc_info(tk)
@inner_lex.lex_state = EXPR_END unless RIPPER_HAS_LEX_STATE
when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
if !@heredoc_queue.empty?
get_heredoc_tk(*@heredoc_queue.shift)
Expand Down Expand Up @@ -549,7 +266,6 @@ def get_squashed_tk
private def get_op_tk(tk)
redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
if redefinable_operators.include?(tk[:text]) and tk[:state] == EXPR_ARG then
@inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
tk[:kind] = :on_ident
elsif tk[:text] =~ /^[-+]$/ then
Expand Down

0 comments on commit 5772f60

Please sign in to comment.