Gem::RequestSet::Lockfile::Tokenizer#tokenize

# File tmp/rubies/ruby-3.4.1/lib/rubygems/request_set/lockfile/tokenizer.rb, line 61
def tokenize(input)
  require "strscan"
  s = StringScanner.new input

  until s.eos? do
    pos = s.pos

    pos = s.pos if leading_whitespace = s.scan(/ +/)

    if s.scan(/[<|=>]{7}/)
      message = "your #{@filename} contains merge conflict markers"
      column, line = token_pos pos

      raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename
    end

    @tokens <<
      if s.scan(/\r?\n/)

        token = Token.new(:newline, nil, *token_pos(pos))
        @line_pos = s.pos
        @line += 1
        token
      elsif s.scan(/[A-Z]+/)

        if leading_whitespace
          text = s.matched
          text += s.scan(/[^\s)]*/).to_s # in case of no match
          Token.new(:text, text, *token_pos(pos))
        else
          Token.new(:section, s.matched, *token_pos(pos))
        end
      elsif s.scan(/([a-z]+):\s/)

        s.pos -= 1 # rewind for possible newline
        Token.new(:entry, s[1], *token_pos(pos))
      elsif s.scan(/\(/)

        Token.new(:l_paren, nil, *token_pos(pos))
      elsif s.scan(/\)/)

        Token.new(:r_paren, nil, *token_pos(pos))
      elsif s.scan(/<=|>=|=|~>|<|>|!=/)

        Token.new(:requirement, s.matched, *token_pos(pos))
      elsif s.scan(/,/)

        Token.new(:comma, nil, *token_pos(pos))
      elsif s.scan(/!/)

        Token.new(:bang, nil, *token_pos(pos))
      elsif s.scan(/[^\s),!]*/)

        Token.new(:text, s.matched, *token_pos(pos))
      else
        raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
      end
  end

  @tokens
end