Prism::Translation::Parser#tokenize

# File tmp/rubies/ruby-3.4.1/lib/prism/translation/parser.rb, line 79
def tokenize(source_buffer, recover = false)
  @source_buffer = source_buffer
  source = source_buffer.source

  offset_cache = build_offset_cache(source)
  result =
    begin
      unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
    rescue ::Parser::SyntaxError
      raise if !recover
    end

  program, tokens = result.value
  ast = build_ast(program, offset_cache) if result.success?

  [
    ast,
    build_comments(result.comments, offset_cache),
    build_tokens(tokens, offset_cache)
  ]
ensure
  @source_buffer = nil
end