split out lox.rb into multiple files
FossilOrigin-Name: 9f8294ceb7aa3c4ad7ea96bafc90afceb8211dc8fabae66a4ace9560aa99eba4
diff --git a/ruby/lib/lox.rb b/ruby/lib/lox.rb
index 416c12b..45c5ad7 100644
--- a/ruby/lib/lox.rb
+++ b/ruby/lib/lox.rb
@@ -1,22 +1,11 @@
#!/usr/bin/env ruby -w
-require "strscan"
+require_relative "lox/error"
+require_relative "lox/expr"
+require_relative "lox/scanner"
+require_relative "lox/token"
module Lox
- class Error < StandardError
- def initialize(line:, where: "", message:)
- @line, @where, @message = line, where, message
- end
-
- def to_s
- "[line #@line] Error#@where: #@message"
- end
- end
-
- def self.error(line, msg)
- raise Error(line:, message:)
- end
-
class Runner
def initialize(scanner: Scanner.new)
@scanner = scanner
@@ -28,180 +17,4 @@ module Lox
end
end
end
-
- class Scanner
- TOKENS = %w[
- ( LEFT_PAREN
- ) RIGHT_PAREN
- { LEFT_BRACE
- } RIGHT_BRACE
- , COMMA
- . DOT
- - MINUS
- + PLUS
- ; SEMICOLON
- * STAR
- != BANG_EQUAL
- ! BANG
- == EQUAL_EQUAL
- = EQUAL
- <= LESS_EQUAL
- < LESS
- >= GREATER_EQUAL
- > GREATER
- / SLASH
- ].each_slice(2).to_h.transform_values(&:to_sym)
- TOKENS_RE = Regexp.union(TOKENS.keys)
-
- KEYWORDS = %w[
- and AND
- class CLASS
- else ELSE
- false FALSE
- for FOR
- fun FUN
- if IF
- nil NIL
- or OR
- print PRINT
- return RETURN
- super SUPER
- this THIS
- true TRUE
- var VAR
- while WHILE
- ].each_slice(2).to_h.transform_values(&:to_sym)
-
- class State < Struct.new(:ss, :tokens, :errors, :line)
- def eos? = ss.eos?
- def scan(re) = ss.scan(re)
- def pos = ss.pos
-
- def initialize(src)
- super(StringScanner.new(src), [], [], 1)
- end
-
- def add_token(type, text: nil, literal: nil)
- text ||= ss.matched
- self.tokens << Token.new(type, text, literal, line)
- end
- end
-
- def scan(src)
- state = State.new(src)
-
- until state.eos?
- case
- when state.scan(/\/\/(?~\n)/)
- # ignore line comment
- when state.scan(/\/\*/)
- scan_block_comment(state)
- when matched = state.scan(TOKENS_RE)
- state.add_token(TOKENS.fetch(matched))
- when state.scan(/[ \r\t]/)
- # ignore whitespace
- when state.scan(/\n/)
- state.line += 1
- when state.scan(/"/)
- scan_str(state)
- when number = state.scan(/\d+(\.\d+)?/)
- state.add_token(:NUMBER, literal: number.to_f)
- when identifier = state.scan(/[a-zA-Z_]\w*/)
- type = KEYWORDS.fetch(identifier, :IDENTIFIER)
- state.add_token(type)
- else
- state.errors << Error.new(line: state.line, message: "Unexpected character.")
- state.scan(/./) # keep scanning
- end
- end
-
- fail unless state.errors.empty?
-
- state.add_token(:EOF, text: "")
- state.tokens
- end
-
- private
-
- def scan_str(state)
- text = ?"
- loop do
- case
- when state.scan(/"/)
- text << ?"
- state.add_token(:STRING, text:, literal: text[1..-2])
- return
- when state.scan(/\n/)
- text << ?\n
- state.line += 1
- when state.eos?
- state.errors << Error.new(line: state.line, message: "Unterminated string.")
- return
- when c = state.scan(/(?~"|\n)/)
- text << c
- else
- fail "unreachable!"
- end
- end
- end
-
- def scan_block_comment(state)
- loop do
- case
- when state.scan(/\/\*/)
- scan_block_comment(state)
- when state.scan(/\*\//)
- return
- when state.scan(/\n/)
- state.line += 1
- when state.eos?
- state.errors << Error.new(line: state.line, message: "Unterminated block comment.")
- return
- when state.scan(/./)
- # no-op
- else
- fail "unreachable!"
- end
- end
- end
- end
-
- Token = Struct.new(:type, :lexeme, :literal, :line) do
- def to_s
- "#{type} #{lexeme} #{literal}"
- end
- end
-
- module Expr
- Binary = Struct.new(:left, :op, :right) do
- def accept(visitor) = visitor.visit_binary(self)
- end
-
- Grouping = Struct.new(:expr) do
- def accept(visitor) = visitor.visit_grouping(self)
- end
-
- Literal = Struct.new(:value) do
- def accept(visitor) = visitor.visit_literal(self)
- end
-
- Unary = Struct.new(:op, :right) do
- def accept(visitor) = visitor.visit_unary(self)
- end
- end
-
- class AstPrinter
- def print(expr) = expr.accept(self)
-
- def visit_binary(expr) = parenthesize(expr.op.lexeme, expr.left, expr.right)
- def visit_grouping(expr) = parenthesize("group", expr.expr)
- def visit_literal(expr) = expr.value&.to_s || "nil"
- def visit_unary(expr) = parenthesize(expr.op.lexeme, expr.right)
-
- private
-
- def parenthesize(name, *exprs)
- "(#{name} #{exprs.map {|expr| expr.accept(self) }.join(" ")})"
- end
- end
end
diff --git a/ruby/lib/lox/ast_printer.rb b/ruby/lib/lox/ast_printer.rb
new file mode 100644
index 0000000..3a23375
--- /dev/null
+++ b/ruby/lib/lox/ast_printer.rb
@@ -0,0 +1,16 @@
+module Lox
+ class AstPrinter
+ def print(expr) = expr.accept(self)
+
+ def visit_binary(expr) = parenthesize(expr.op.lexeme, expr.left, expr.right)
+ def visit_grouping(expr) = parenthesize("group", expr.expr)
+ def visit_literal(expr) = expr.value&.to_s || "nil"
+ def visit_unary(expr) = parenthesize(expr.op.lexeme, expr.right)
+
+ private
+
+ def parenthesize(name, *exprs)
+ "(#{name} #{exprs.map {|expr| expr.accept(self) }.join(" ")})"
+ end
+ end
+end
diff --git a/ruby/lib/lox/error.rb b/ruby/lib/lox/error.rb
new file mode 100644
index 0000000..4120812
--- /dev/null
+++ b/ruby/lib/lox/error.rb
@@ -0,0 +1,11 @@
+module Lox
+ class Error < StandardError
+ def initialize(line:, where: "", message:)
+ @line, @where, @message = line, where, message
+ end
+
+ def to_s
+ "[line #@line] Error#@where: #@message"
+ end
+ end
+end
diff --git a/ruby/lib/lox/expr.rb b/ruby/lib/lox/expr.rb
new file mode 100644
index 0000000..38860a2
--- /dev/null
+++ b/ruby/lib/lox/expr.rb
@@ -0,0 +1,19 @@
+module Lox
+ module Expr
+ Binary = Struct.new(:left, :op, :right) do
+ def accept(visitor) = visitor.visit_binary(self)
+ end
+
+ Grouping = Struct.new(:expr) do
+ def accept(visitor) = visitor.visit_grouping(self)
+ end
+
+ Literal = Struct.new(:value) do
+ def accept(visitor) = visitor.visit_literal(self)
+ end
+
+ Unary = Struct.new(:op, :right) do
+ def accept(visitor) = visitor.visit_unary(self)
+ end
+ end
+end
diff --git a/ruby/lib/lox/scanner.rb b/ruby/lib/lox/scanner.rb
new file mode 100644
index 0000000..ca3d477
--- /dev/null
+++ b/ruby/lib/lox/scanner.rb
@@ -0,0 +1,142 @@
+require "strscan"
+
+require_relative "token"
+
+module Lox
+ class Scanner
+ TOKENS = %w[
+ ( LEFT_PAREN
+ ) RIGHT_PAREN
+ { LEFT_BRACE
+ } RIGHT_BRACE
+ , COMMA
+ . DOT
+ - MINUS
+ + PLUS
+ ; SEMICOLON
+ * STAR
+ != BANG_EQUAL
+ ! BANG
+ == EQUAL_EQUAL
+ = EQUAL
+ <= LESS_EQUAL
+ < LESS
+ >= GREATER_EQUAL
+ > GREATER
+ / SLASH
+ ].each_slice(2).to_h.transform_values(&:to_sym)
+ TOKENS_RE = Regexp.union(TOKENS.keys)
+
+ KEYWORDS = %w[
+ and AND
+ class CLASS
+ else ELSE
+ false FALSE
+ for FOR
+ fun FUN
+ if IF
+ nil NIL
+ or OR
+ print PRINT
+ return RETURN
+ super SUPER
+ this THIS
+ true TRUE
+ var VAR
+ while WHILE
+ ].each_slice(2).to_h.transform_values(&:to_sym)
+
+ class State < Struct.new(:ss, :tokens, :errors, :line)
+ def eos? = ss.eos?
+ def scan(re) = ss.scan(re)
+ def pos = ss.pos
+
+ def initialize(src)
+ super(StringScanner.new(src), [], [], 1)
+ end
+
+ def add_token(type, text: nil, literal: nil)
+ text ||= ss.matched
+ self.tokens << Lox::Token.new(type, text, literal, line)
+ end
+ end
+
+ def scan(src)
+ state = State.new(src)
+
+ until state.eos?
+ case
+ when state.scan(/\/\/(?~\n)/)
+ # ignore line comment
+ when state.scan(/\/\*/)
+ scan_block_comment(state)
+ when matched = state.scan(TOKENS_RE)
+ state.add_token(TOKENS.fetch(matched))
+ when state.scan(/[ \r\t]/)
+ # ignore whitespace
+ when state.scan(/\n/)
+ state.line += 1
+ when state.scan(/"/)
+ scan_str(state)
+ when number = state.scan(/\d+(\.\d+)?/)
+ state.add_token(:NUMBER, literal: number.to_f)
+ when identifier = state.scan(/[a-zA-Z_]\w*/)
+ type = KEYWORDS.fetch(identifier, :IDENTIFIER)
+ state.add_token(type)
+ else
+ state.errors << Error.new(line: state.line, message: "Unexpected character.")
+ state.scan(/./) # keep scanning
+ end
+ end
+
+ fail unless state.errors.empty?
+
+ state.add_token(:EOF, text: "")
+ state.tokens
+ end
+
+ private
+
+ def scan_str(state)
+ text = ?"
+ loop do
+ case
+ when state.scan(/"/)
+ text << ?"
+ state.add_token(:STRING, text:, literal: text[1..-2])
+ return
+ when state.scan(/\n/)
+ text << ?\n
+ state.line += 1
+ when state.eos?
+ state.errors << Error.new(line: state.line, message: "Unterminated string.")
+ return
+ when c = state.scan(/(?~"|\n)/)
+ text << c
+ else
+ fail "unreachable!"
+ end
+ end
+ end
+
+ def scan_block_comment(state)
+ loop do
+ case
+ when state.scan(/\/\*/)
+ scan_block_comment(state)
+ when state.scan(/\*\//)
+ return
+ when state.scan(/\n/)
+ state.line += 1
+ when state.eos?
+ state.errors << Error.new(line: state.line, message: "Unterminated block comment.")
+ return
+ when state.scan(/./)
+ # no-op
+ else
+ fail "unreachable!"
+ end
+ end
+ end
+ end
+end
diff --git a/ruby/lib/lox/token.rb b/ruby/lib/lox/token.rb
new file mode 100644
index 0000000..f1166ad
--- /dev/null
+++ b/ruby/lib/lox/token.rb
@@ -0,0 +1,7 @@
+module Lox
+ Token = Struct.new(:type, :lexeme, :literal, :line) do
+ def to_s
+ "#{type} #{lexeme} #{literal}"
+ end
+ end
+end
diff --git a/ruby/test/lox/test_ast_printer.rb b/ruby/test/lox/test_ast_printer.rb
new file mode 100644
index 0000000..2a0319a
--- /dev/null
+++ b/ruby/test/lox/test_ast_printer.rb
@@ -0,0 +1,22 @@
+require_relative "../test_helper"
+
+require "lox/ast_printer"
+require "lox/expr"
+require "lox/token"
+
+class TestAstPrinter < Lox::Test
+ def test_ast_printer
+ expr = Lox::Expr::Binary.new(
+ Lox::Expr::Unary.new(
+ Lox::Token.new(:MINUS, ?-, nil, 1),
+ Lox::Expr::Literal.new(123),
+ ),
+ Lox::Token.new(:STAR, ?*, nil, 1),
+ Lox::Expr::Grouping.new(
+ Lox::Expr::Literal.new(45.67),
+ ),
+ )
+
+ assert_equal "(* (- 123) (group 45.67))", Lox::AstPrinter.new.print(expr)
+ end
+end
diff --git a/ruby/test/lox/test_scanner.rb b/ruby/test/lox/test_scanner.rb
new file mode 100644
index 0000000..1523a2e
--- /dev/null
+++ b/ruby/test/lox/test_scanner.rb
@@ -0,0 +1,108 @@
+require_relative "../test_helper"
+
+require "lox/scanner"
+
+class TestScanner < Lox::Test
+ def setup
+ @scanner = Lox::Scanner.new
+ end
+
+ def test_basic_tokens
+ %w[( LEFT_PAREN
+ ) RIGHT_PAREN
+ { LEFT_BRACE
+ } RIGHT_BRACE
+ , COMMA
+ . DOT
+ - MINUS
+ + PLUS
+ ; SEMICOLON
+ * STAR
+ != BANG_EQUAL
+ ! BANG
+ == EQUAL_EQUAL
+ = EQUAL
+ <= LESS_EQUAL
+ < LESS
+ >= GREATER_EQUAL
+ > GREATER
+ / SLASH].each_slice(2).to_h.transform_values(&:to_sym).each do |str, token_type|
+ assert_equal token_type.to_sym, @scanner.scan(str)[0].type
+ end
+ end
+
+ def test_comments_and_whitespace
+ tokens = @scanner.scan(<<~SRC)
+ (\t) // here lies a comment
+ . //
+ SRC
+
+ assert_equal %i[LEFT_PAREN RIGHT_PAREN DOT EOF], tokens.map(&:type)
+ end
+
+ def test_line_numbers
+ tokens = @scanner.scan(<<~SRC)
+ (
+ )
+ SRC
+
+ assert_equal [1, 2, 3], tokens.map(&:line)
+ end
+
+ def test_strings
+ assert_equal [
+ Lox::Token.new(:STRING, '""', "", 1),
+ Lox::Token.new(:EOF, "", nil, 1),
+ ], @scanner.scan('""')
+
+ assert_raises do
+ @scanner.scan('"')
+ end
+
+ assert_equal [
+ Lox::Token.new(:STRING, '"foo"', "foo", 1),
+ Lox::Token.new(:EOF, "", nil, 1),
+ ], @scanner.scan('"foo"')
+
+ assert_equal [
+ Lox::Token.new(:STRING, "\"foo\nbar\"", "foo\nbar", 2),
+ Lox::Token.new(:EOF, "", nil, 2),
+ ], @scanner.scan("\"foo\nbar\"")
+ end
+
+ def test_numbers
+ assert_equal [
+ Lox::Token.new(:NUMBER, "123", 123.0, 1),
+ Lox::Token.new(:NUMBER, "123.4", 123.4, 1),
+ Lox::Token.new(:EOF, "", nil, 1),
+ ], @scanner.scan("123 123.4")
+ end
+
+ def test_identifiers
+ assert_equal [
+ Lox::Token.new(:OR, "or", nil, 1),
+ Lox::Token.new(:IDENTIFIER, "orchid", nil, 1),
+ Lox::Token.new(:IDENTIFIER, "o", nil, 1),
+ Lox::Token.new(:EOF, "", nil, 1),
+ ], @scanner.scan("or orchid o")
+ end
+
+ def test_block_comments
+ tokens = @scanner.scan(<<~SRC)
+ foo
+ /* here lies a /* nested */ block comment
+ with newlines */
+ bar
+ SRC
+
+ assert_equal [
+ Lox::Token.new(:IDENTIFIER, "foo", nil, 1),
+ Lox::Token.new(:IDENTIFIER, "bar", nil, 4),
+ Lox::Token.new(:EOF, "", nil, 5),
+ ], tokens
+
+ assert_raises do
+ @scanner.scan("/*")
+ end
+ end
+end
diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb
new file mode 100644
index 0000000..2b59721
--- /dev/null
+++ b/ruby/test/test_helper.rb
@@ -0,0 +1,6 @@
+require "minitest"
+
+module Lox
+ class Test < Minitest::Test
+ end
+end
diff --git a/ruby/test/test_lox.rb b/ruby/test/test_lox.rb
index 1aeff9e..eca0bb0 100644
--- a/ruby/test/test_lox.rb
+++ b/ruby/test/test_lox.rb
@@ -1,13 +1,12 @@
+require_relative "test_helper"
+
require "lox"
-include Lox
require "open3"
-require "minitest/autorun"
require "mocktail"
-require "pry"
-class TestLox < Minitest::Test
+class TestLox < Lox::Test
def test_error_on_more_than_one_arg
lox_path = File.expand_path("../bin/lox", __dir__)
o, s = Open3.capture2(lox_path, "foo", "bar")
@@ -16,7 +15,7 @@ class TestLox < Minitest::Test
end
end
-class TestRunner < Minitest::Test
+class TestRunner < Lox::Test
include Mocktail::DSL
def teardown
@@ -24,8 +23,8 @@ class TestRunner < Minitest::Test
end
def test_returns_tokens
- scanner = Mocktail.of(Scanner)
- runner = Runner.new(scanner:)
+ scanner = Mocktail.of(Lox::Scanner)
+ runner = Lox::Runner.new(scanner:)
stubs { scanner.scan("src") }.with { %w[ some tokens ] }
tokens = runner.run("src")
@@ -34,124 +33,3 @@ class TestRunner < Minitest::Test
end
end
-class TestScanner < Minitest::Test
- def setup
- @scanner = Scanner.new
- end
-
- def test_basic_tokens
- %w[( LEFT_PAREN
- ) RIGHT_PAREN
- { LEFT_BRACE
- } RIGHT_BRACE
- , COMMA
- . DOT
- - MINUS
- + PLUS
- ; SEMICOLON
- * STAR
- != BANG_EQUAL
- ! BANG
- == EQUAL_EQUAL
- = EQUAL
- <= LESS_EQUAL
- < LESS
- >= GREATER_EQUAL
- > GREATER
- / SLASH].each_slice(2).to_h.transform_values(&:to_sym).each do |str, token_type|
- assert_equal token_type.to_sym, @scanner.scan(str)[0].type
- end
- end
-
- def test_comments_and_whitespace
- tokens = @scanner.scan(<<~SRC)
- (\t) // here lies a comment
- . //
- SRC
-
- assert_equal %i[LEFT_PAREN RIGHT_PAREN DOT EOF], tokens.map(&:type)
- end
-
- def test_line_numbers
- tokens = @scanner.scan(<<~SRC)
- (
- )
- SRC
-
- assert_equal [1, 2, 3], tokens.map(&:line)
- end
-
- def test_strings
- assert_equal [
- Token.new(:STRING, '""', "", 1),
- Token.new(:EOF, "", nil, 1),
- ], @scanner.scan('""')
-
- assert_raises do
- @scanner.scan('"')
- end
-
- assert_equal [
- Token.new(:STRING, '"foo"', "foo", 1),
- Token.new(:EOF, "", nil, 1),
- ], @scanner.scan('"foo"')
-
- assert_equal [
- Token.new(:STRING, "\"foo\nbar\"", "foo\nbar", 2),
- Token.new(:EOF, "", nil, 2),
- ], @scanner.scan("\"foo\nbar\"")
- end
-
- def test_numbers
- assert_equal [
- Token.new(:NUMBER, "123", 123.0, 1),
- Token.new(:NUMBER, "123.4", 123.4, 1),
- Token.new(:EOF, "", nil, 1),
- ], @scanner.scan("123 123.4")
- end
-
- def test_identifiers
- assert_equal [
- Token.new(:OR, "or", nil, 1),
- Token.new(:IDENTIFIER, "orchid", nil, 1),
- Token.new(:IDENTIFIER, "o", nil, 1),
- Token.new(:EOF, "", nil, 1),
- ], @scanner.scan("or orchid o")
- end
-
- def test_block_comments
- tokens = @scanner.scan(<<~SRC)
- foo
- /* here lies a /* nested */ block comment
- with newlines */
- bar
- SRC
-
- assert_equal [
- Token.new(:IDENTIFIER, "foo", nil, 1),
- Token.new(:IDENTIFIER, "bar", nil, 4),
- Token.new(:EOF, "", nil, 5),
- ], tokens
-
- assert_raises do
- @scanner.scan("/*")
- end
- end
-end
-
-class TestAstPrinter < Minitest::Test
- def test_ast_printer
- expr = Expr::Binary.new(
- Expr::Unary.new(
- Token.new(:MINUS, ?-, nil, 1),
- Expr::Literal.new(123),
- ),
- Token.new(:STAR, ?*, nil, 1),
- Expr::Grouping.new(
- Expr::Literal.new(45.67),
- ),
- )
-
- assert_equal "(* (- 123) (group 45.67))", AstPrinter.new.print(expr)
- end
-end