748f87636a
checkpoint failing test after fixing tests checkpoint checkpoint checkpoint re-work asd checkpoint checkpoint checkpoint mix proj checkpoint mix first parser impl checkpoint fix tests re-org parser checkpoint strings fix multiline strings tuples checkpoint maps checkpoint checkpoint checkpoint checkpoint fix weird eof expression parse error checkpoint before typing checkpoint checpoint checkpoint checkpoint checkpoint ids in primitive types checkpoint checkpoint fix tests initial annotation checkpoint checkpoint checkpoint union subtyping conventions refactor - split typer typing tuples checkpoint test refactor checkpoint test refactor parsing atoms checkpoint atoms wip lists checkpoint typing lists checkopint checkpoint wip fixing correct list typing map discussion checkpoint map basic typing fix tests checkpoint checkpoint checkpoint checkpoint fix condition typing fix literal keys in map types checkpoint union types checkpoint union type checkpoint row types discussion & bidirectional typecheck checkpoint basic lambdas checkpoint lambdas typing application wip function application checkpoint checkpoint checkpoint cduce checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint checkpoint
869 lines
33 KiB
Elixir
869 lines
33 KiB
Elixir
defmodule Til.ParserTest do
|
|
use ExUnit.Case, async: true
|
|
alias Til.Parser
|
|
alias Til.TestHelpers
|
|
|
|
describe "parse/2 - Basic Atoms" do
|
|
test "parses a simple integer literal" do
|
|
source = "42"
|
|
file_name = "test.tly"
|
|
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source, file_name)
|
|
# file_node + integer_node
|
|
assert map_size(nodes_map) == 2
|
|
# Still need file_node for parent_id check
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert is_integer(node.id)
|
|
assert node.type_id == nil
|
|
assert node.parent_id == file_node.id
|
|
assert node.file == file_name
|
|
# "42"
|
|
assert node.location == [0, 1, 1, 2, 1, 3]
|
|
assert node.raw_string == source
|
|
assert node.ast_node_type == :literal_integer
|
|
assert node.value == 42
|
|
end
|
|
|
|
test "parses a negative integer literal" do
|
|
source = "-123"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
# file_node + integer_node
|
|
assert map_size(nodes_map) == 2
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.parent_id == file_node.id
|
|
# "-123"
|
|
assert node.location == [0, 1, 1, 4, 1, 5]
|
|
assert node.raw_string == source
|
|
assert node.ast_node_type == :literal_integer
|
|
assert node.value == -123
|
|
end
|
|
|
|
test "parses a simple symbol" do
|
|
source = "foo"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
# file_node + symbol_node
|
|
assert map_size(nodes_map) == 2
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.parent_id == file_node.id
|
|
# "foo"
|
|
assert node.location == [0, 1, 1, 3, 1, 4]
|
|
assert node.raw_string == source
|
|
assert node.ast_node_type == :symbol
|
|
assert node.name == "foo"
|
|
end
|
|
|
|
test "parses a symbol with hyphens and numbers" do
|
|
source = "my-var-123"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.parent_id == file_node.id
|
|
assert node.raw_string == source
|
|
assert node.ast_node_type == :symbol
|
|
assert node.name == "my-var-123"
|
|
end
|
|
|
|
test "parses operator-like symbols" do
|
|
source = "+"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.parent_id == file_node.id
|
|
assert node.raw_string == source
|
|
assert node.ast_node_type == :symbol
|
|
assert node.name == "+"
|
|
end
|
|
|
|
test "parses a sequence of integers and symbols" do
|
|
source = "10 foo -20 bar+"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# 4 items + 1 file_node
|
|
assert map_size(nodes_map) == 5
|
|
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
# Children are already sorted by the parser
|
|
[n1_id, n2_id, n3_id, n4_id] = file_node.children
|
|
n1 = TestHelpers.get_node_by_id(nodes_map, n1_id)
|
|
n2 = TestHelpers.get_node_by_id(nodes_map, n2_id)
|
|
n3 = TestHelpers.get_node_by_id(nodes_map, n3_id)
|
|
n4 = TestHelpers.get_node_by_id(nodes_map, n4_id)
|
|
|
|
assert n1.ast_node_type == :literal_integer
|
|
assert n1.value == 10
|
|
assert n1.raw_string == "10"
|
|
assert n1.location == [0, 1, 1, 2, 1, 3]
|
|
assert n1.parent_id == file_node.id
|
|
|
|
assert n2.ast_node_type == :symbol
|
|
assert n2.name == "foo"
|
|
assert n2.raw_string == "foo"
|
|
# after "10 "
|
|
assert n2.location == [3, 1, 4, 6, 1, 7]
|
|
assert n2.parent_id == file_node.id
|
|
|
|
assert n3.ast_node_type == :literal_integer
|
|
assert n3.value == -20
|
|
assert n3.raw_string == "-20"
|
|
# after "foo "
|
|
assert n3.location == [7, 1, 8, 10, 1, 11]
|
|
assert n3.parent_id == file_node.id
|
|
|
|
assert n4.ast_node_type == :symbol
|
|
assert n4.name == "bar+"
|
|
assert n4.raw_string == "bar+"
|
|
# after "-20 "
|
|
assert n4.location == [11, 1, 12, 15, 1, 16]
|
|
assert n4.parent_id == file_node.id
|
|
end
|
|
|
|
test "uses 'unknown' as default file_name" do
|
|
source = "7"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.file == "unknown"
|
|
assert file_node.file == "unknown"
|
|
assert node.parent_id == file_node.id
|
|
end
|
|
end
|
|
|
|
describe "parse/2 - S-expressions" do
|
|
test "parses an empty S-expression" do
|
|
source = "()"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# file_node + s_expr_node
|
|
assert map_size(nodes_map) == 2
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
s_expr_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
assert s_expr_node.ast_node_type == :s_expression
|
|
assert s_expr_node.parent_id == file_node.id
|
|
assert s_expr_node.children == []
|
|
# "()"
|
|
assert s_expr_node.location == [0, 1, 1, 2, 1, 3]
|
|
|
|
# Raw string for S-expressions is tricky, current impl might be placeholder like "()" or "(...)"
|
|
# For now, let's assert it starts with ( and ends with ) if not placeholder
|
|
assert String.starts_with?(s_expr_node.raw_string, "(") &&
|
|
String.ends_with?(s_expr_node.raw_string, ")")
|
|
|
|
# if it's "()"
|
|
assert String.length(s_expr_node.raw_string) == 2
|
|
end
|
|
|
|
test "parses a simple S-expression with integers" do
|
|
source = "(1 22 -3)"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# 1 file_node + 1 S-expr node + 3 integer nodes = 5 nodes
|
|
assert map_size(nodes_map) == 5
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
s_expr_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(s_expr_node)
|
|
assert s_expr_node.ast_node_type == :s_expression
|
|
assert s_expr_node.parent_id == file_node.id
|
|
assert length(s_expr_node.children) == 3
|
|
|
|
children_nodes = Enum.map(s_expr_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
assert Enum.map(children_nodes, & &1.value) == [1, 22, -3]
|
|
|
|
Enum.each(children_nodes, fn child ->
|
|
assert child.parent_id == s_expr_node.id
|
|
assert child.ast_node_type == :literal_integer
|
|
end)
|
|
|
|
# Source "(1 22 -3)" has length 9. Start offset 0, end offset 9. Start col 1, end col 10.
|
|
# "(1 22 -3)"
|
|
assert s_expr_node.location == [0, 1, 1, 9, 1, 10]
|
|
end
|
|
|
|
test "parses a simple S-expression with symbols" do
|
|
source = "(foo bar baz)"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# 1 file_node, 1 s-expr, 3 symbols = 5 nodes
|
|
assert map_size(nodes_map) == 5
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
s_expr_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(s_expr_node)
|
|
assert s_expr_node.ast_node_type == :s_expression
|
|
assert s_expr_node.parent_id == file_node.id
|
|
assert length(s_expr_node.children) == 3
|
|
children_nodes = Enum.map(s_expr_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
assert Enum.map(children_nodes, & &1.name) == ["foo", "bar", "baz"]
|
|
|
|
Enum.each(children_nodes, fn child ->
|
|
assert child.parent_id == s_expr_node.id
|
|
assert child.ast_node_type == :symbol
|
|
end)
|
|
end
|
|
|
|
test "parses nested S-expressions" do
|
|
# outer: a, (b 1), c | inner: b, 1
|
|
source = "(a (b 1) c)"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# Nodes: 1 file_node, outer_s_expr, a, inner_s_expr, c, b, 1 => 7 nodes
|
|
assert map_size(nodes_map) == 7
|
|
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
outer_s_expr = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(outer_s_expr)
|
|
assert outer_s_expr.ast_node_type == :s_expression
|
|
assert outer_s_expr.parent_id == file_node.id
|
|
assert length(outer_s_expr.children) == 3
|
|
|
|
# 'a'
|
|
child1 = TestHelpers.get_nth_child_node(nodes_map, 0, outer_s_expr.id)
|
|
# '(b 1)'
|
|
inner_s_expr = TestHelpers.get_nth_child_node(nodes_map, 1, outer_s_expr.id)
|
|
# 'c'
|
|
child3 = TestHelpers.get_nth_child_node(nodes_map, 2, outer_s_expr.id)
|
|
|
|
assert child1.ast_node_type == :symbol && child1.name == "a"
|
|
assert inner_s_expr.ast_node_type == :s_expression
|
|
assert child3.ast_node_type == :symbol && child3.name == "c"
|
|
|
|
assert inner_s_expr.parent_id == outer_s_expr.id
|
|
assert length(inner_s_expr.children) == 2
|
|
# 'b'
|
|
grandchild1 = TestHelpers.get_nth_child_node(nodes_map, 0, inner_s_expr.id)
|
|
# '1'
|
|
grandchild2 = TestHelpers.get_nth_child_node(nodes_map, 1, inner_s_expr.id)
|
|
|
|
assert grandchild1.ast_node_type == :symbol && grandchild1.name == "b"
|
|
assert grandchild1.parent_id == inner_s_expr.id
|
|
assert grandchild2.ast_node_type == :literal_integer && grandchild2.value == 1
|
|
assert grandchild2.parent_id == inner_s_expr.id
|
|
end
|
|
|
|
test "parses S-expressions with varied spacing" do
|
|
source = "( foo 1\nbar )"
|
|
{s_expr_node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
refute is_nil(s_expr_node)
|
|
assert s_expr_node.ast_node_type == :s_expression
|
|
assert s_expr_node.parent_id == file_node.id
|
|
assert length(s_expr_node.children) == 3
|
|
|
|
children_names_values =
|
|
Enum.map(s_expr_node.children, fn id ->
|
|
node = TestHelpers.get_node_by_id(nodes_map, id)
|
|
if node.ast_node_type == :symbol, do: node.name, else: node.value
|
|
end)
|
|
|
|
assert children_names_values == ["foo", 1, "bar"]
|
|
end
|
|
end
|
|
|
|
describe "parse/2 - Error Handling" do
|
|
test "handles unclosed S-expression" do
|
|
source = "(foo bar"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# Expect 1 file_node, 1 S-expr node (marked with error), 2 symbol nodes = 4 nodes
|
|
assert map_size(nodes_map) == 4
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
s_expr_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(s_expr_node)
|
|
assert s_expr_node.ast_node_type == :s_expression
|
|
assert s_expr_node.parent_id == file_node.id
|
|
assert s_expr_node.parsing_error == "Unclosed S-expression"
|
|
# foo, bar
|
|
assert length(s_expr_node.children) == 2
|
|
# Location should span till end of source
|
|
# "(foo bar"
|
|
assert s_expr_node.location == [0, 1, 1, 8, 1, 9]
|
|
end
|
|
|
|
test "handles unexpected closing parenthesis at top level" do
|
|
# foo, error_node_for_), bar
|
|
source = "foo ) bar"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# 3 items + 1 file_node
|
|
assert map_size(nodes_map) == 4
|
|
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
top_level_children =
|
|
Enum.map(file_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
error_node =
|
|
Enum.find(top_level_children, &(&1.ast_node_type == :unknown && &1.raw_string == ")"))
|
|
|
|
refute is_nil(error_node)
|
|
assert error_node.parent_id == file_node.id
|
|
assert error_node.parsing_error == "Unexpected ')'"
|
|
# location of ")"
|
|
assert error_node.location == [4, 1, 5, 5, 1, 6]
|
|
|
|
symbol_foo =
|
|
Enum.find(top_level_children, &(&1.ast_node_type == :symbol && &1.name == "foo"))
|
|
|
|
symbol_bar =
|
|
Enum.find(top_level_children, &(&1.ast_node_type == :symbol && &1.name == "bar"))
|
|
|
|
refute is_nil(symbol_foo)
|
|
assert symbol_foo.parent_id == file_node.id
|
|
refute is_nil(symbol_bar)
|
|
assert symbol_bar.parent_id == file_node.id
|
|
end
|
|
|
|
test "handles unknown token inside S-expression (partial, basic)" do
|
|
source = "(foo 123"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
assert map_size(nodes_map) == 4
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
s_expr_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(s_expr_node)
|
|
assert s_expr_node.ast_node_type == :s_expression
|
|
assert s_expr_node.parent_id == file_node.id
|
|
assert s_expr_node.parsing_error == "Unclosed S-expression"
|
|
assert length(s_expr_node.children) == 2
|
|
|
|
child1 = TestHelpers.get_nth_child_node(nodes_map, 0, s_expr_node.id)
|
|
child2 = TestHelpers.get_nth_child_node(nodes_map, 1, s_expr_node.id)
|
|
|
|
assert child1.name == "foo"
|
|
assert child2.value == 123
|
|
end
|
|
|
|
test "parses multiple top-level S-expressions" do
|
|
source = "() (1 2) (foo)"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# () -> 1 node
|
|
# (1 2) -> 1 s-expr, 2 ints = 3 nodes
|
|
# (foo) -> 1 s-expr, 1 symbol = 2 nodes
|
|
# Total items = 1 + 3 + 2 = 6 nodes. Plus 1 file_node = 7 nodes.
|
|
assert map_size(nodes_map) == 7
|
|
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
top_level_s_expr_nodes =
|
|
file_node.children
|
|
|> Enum.map(&TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|> Enum.filter(&(&1.ast_node_type == :s_expression))
|
|
|
|
assert length(top_level_s_expr_nodes) == 3
|
|
Enum.each(top_level_s_expr_nodes, fn node -> assert node.parent_id == file_node.id end)
|
|
|
|
# Check children counts or specific content if necessary
|
|
# For example, the S-expression for (1 2)
|
|
s_expr_1_2 =
|
|
Enum.find(top_level_s_expr_nodes, fn node ->
|
|
children = Enum.map(node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
Enum.map(children, & &1.value) == [1, 2]
|
|
end)
|
|
|
|
refute is_nil(s_expr_1_2)
|
|
end
|
|
|
|
# Test for raw_string of S-expression (this is a known tricky part in the impl)
|
|
# The current implementation has a placeholder for S-expression raw_string.
|
|
# This test will likely fail or need adjustment based on how raw_string is actually captured.
|
|
# For now, I'll skip a very precise raw_string test for S-expressions until it's robustly implemented.
|
|
# test "S-expression raw_string is captured correctly" do
|
|
# source = "( add 1 2 )" # Note spaces
|
|
# {:ok, nodes_map} = Parser.parse(source)
|
|
# s_expr_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :s_expression))
|
|
# assert s_expr_node.raw_string == source
|
|
# end
|
|
end
|
|
|
|
describe "parse/2 - String Literals" do
|
|
test "parses a simple single-line string" do
|
|
source = "'hello world'"
|
|
file_name = "test_str.tly"
|
|
{:ok, nodes_map} = Parser.parse(source, file_name)
|
|
# file_node + string_node
|
|
assert map_size(nodes_map) == 2
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.value == "hello world"
|
|
assert node.raw_string == "'hello world'"
|
|
assert node.location == [0, 1, 1, 13, 1, 14]
|
|
assert node.file == file_name
|
|
assert node.parent_id == file_node.id
|
|
assert node.type_id == nil
|
|
end
|
|
|
|
test "parses an empty string" do
|
|
source = "''"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# file_node + string_node
|
|
assert map_size(nodes_map) == 2
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
assert node.value == ""
|
|
assert node.raw_string == "''"
|
|
assert node.location == [0, 1, 1, 2, 1, 3]
|
|
end
|
|
|
|
test "parses a multiline string with no initial indent" do
|
|
# ' is at col 1, strip 0 spaces
|
|
source = "'hello\nworld'"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
assert node.value == "hello\nworld"
|
|
assert node.raw_string == "'hello\nworld'"
|
|
assert node.location == [0, 1, 1, 13, 2, 7]
|
|
end
|
|
|
|
test "parses a multiline string with initial indent" do
|
|
# ' is at col 3, strip 2 spaces
|
|
source = " 'hello\n world\n end'"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
assert node.value == "hello\nworld\nend"
|
|
assert node.raw_string == "'hello\n world\n end'"
|
|
# Location of ' is [2,1,3]. Raw string length is 21.
|
|
# End location: offset 2+21=23. Line 3, Col 7.
|
|
assert node.location == [2, 1, 3, 23, 3, 7]
|
|
end
|
|
|
|
test "parses a multiline string with varied subsequent indents" do
|
|
# ' is at col 3, strip 2 spaces
|
|
source = " 'hello\n world\n more'"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
# " world" -> "world", " more" -> " more"
|
|
assert node.value == "hello\nworld\n more"
|
|
assert node.raw_string == "'hello\n world\n more'"
|
|
# Location of ' is [2,1,3]. Raw string length is 22.
|
|
# End location: offset 2+22=24. Line 3, Col 9.
|
|
assert node.location == [2, 1, 3, 24, 3, 9]
|
|
end
|
|
|
|
test "parses a string containing parentheses and other special chars" do
|
|
source = "' (foo) [bar] - 123 '"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
assert node.value == " (foo) [bar] - 123 "
|
|
assert node.raw_string == "' (foo) [bar] - 123 '"
|
|
end
|
|
|
|
test "parses a string within an S-expression" do
|
|
source = "('str' 1)"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# 1 file_node, s-expr, string, integer = 4 nodes
|
|
assert map_size(nodes_map) == 4
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
s_expr_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(s_expr_node)
|
|
assert s_expr_node.ast_node_type == :s_expression
|
|
assert s_expr_node.parent_id == file_node.id
|
|
children_nodes = Enum.map(s_expr_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
string_node = Enum.find(children_nodes, &(&1.ast_node_type == :literal_string))
|
|
integer_node = Enum.find(children_nodes, &(&1.ast_node_type == :literal_integer))
|
|
|
|
refute is_nil(string_node)
|
|
refute is_nil(integer_node)
|
|
|
|
assert string_node.value == "str"
|
|
assert string_node.raw_string == "'str'"
|
|
assert string_node.parent_id == s_expr_node.id
|
|
|
|
# Location: `(` is [0,1,1]. ` ` is [1,1,2]. `s` is [2,1,3]. `t` is [3,1,4]. `r` is [4,1,5]. ` ` is [5,1,6].
|
|
# Token "'str'" starts at offset 1, line 1, col 2. Length 5. Ends offset 6, line 1, col 7.
|
|
assert string_node.location == [1, 1, 2, 6, 1, 7]
|
|
|
|
assert integer_node.value == 1
|
|
end
|
|
|
|
test "handles unclosed string literal" do
|
|
source = "'abc"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# file_node + string_node
|
|
assert map_size(nodes_map) == 2
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
assert node.parsing_error == "Unclosed string literal"
|
|
# Content up to EOF, processed
|
|
assert node.value == "abc"
|
|
assert node.raw_string == "'abc"
|
|
# Location: ' is [0,1,1]. Raw string "'abc" length 4. Ends offset 4, line 1, col 5.
|
|
assert node.location == [0, 1, 1, 4, 1, 5]
|
|
end
|
|
|
|
test "handles unclosed string literal with newlines" do
|
|
# ' at col 3, strip 2
|
|
source = " 'hello\n world"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
assert node.parsing_error == "Unclosed string literal"
|
|
assert node.value == "hello\nworld"
|
|
assert node.raw_string == "'hello\n world"
|
|
# Location of ' is [2,1,3]. Raw string "'hello\n world" length 14.
|
|
# ' (2,1,3) -> (3,1,4)
|
|
# hello (3,1,4) -> (8,1,9)
|
|
# \n (8,1,9) -> (9,2,1)
|
|
# world (9,2,1) -> (16,2,8)
|
|
# End location: offset 2+14=16. Line 2, Col 8.
|
|
assert node.location == [2, 1, 3, 16, 2, 8]
|
|
end
|
|
|
|
test "string with only newlines and spaces, respecting indent" do
|
|
# ' at col 3, strip 2
|
|
source = " '\n \n '"
|
|
# Content: "\n \n "
|
|
# Lines: ["", " ", " "]
|
|
# Processed: "", " " (strip 2 from " "), "" (strip 2 from " ")
|
|
# Value: "\n \n"
|
|
{node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
assert node.ast_node_type == :literal_string
|
|
assert node.parent_id == file_node.id
|
|
assert node.value == "\n \n"
|
|
assert node.raw_string == "'\n \n '"
|
|
end
|
|
|
|
test "large s-expression parse test" do
|
|
# ' at col 3, strip 2
|
|
source = """
|
|
(defn my-function (x y)
|
|
(= asd 'first line
|
|
second line
|
|
third line
|
|
asdasd')
|
|
(+ x y))
|
|
"""
|
|
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
assert map_size(nodes_map) > 0
|
|
# IO.inspect(nodes_map, limit: :infinity)
|
|
end
|
|
end
|
|
|
|
describe "parse/2 - List Expressions" do
|
|
test "parses an empty list" do
|
|
source = "[]"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# file_node + list_node
|
|
assert map_size(nodes_map) == 2
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
list_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
assert list_node.ast_node_type == :list_expression
|
|
assert list_node.parent_id == file_node.id
|
|
assert list_node.children == []
|
|
# "[]"
|
|
assert list_node.location == [0, 1, 1, 2, 1, 3]
|
|
assert list_node.raw_string == "[]"
|
|
end
|
|
|
|
test "parses a simple list with integers" do
|
|
source = "[1 22 -3]"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# 1 file_node + 1 list_expression node + 3 integer nodes = 5 nodes
|
|
assert map_size(nodes_map) == 5
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
list_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(list_node)
|
|
assert list_node.ast_node_type == :list_expression
|
|
assert list_node.parent_id == file_node.id
|
|
assert length(list_node.children) == 3
|
|
|
|
children_nodes = Enum.map(list_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
assert Enum.map(children_nodes, & &1.value) == [1, 22, -3]
|
|
|
|
Enum.each(children_nodes, fn child ->
|
|
assert child.parent_id == list_node.id
|
|
assert child.ast_node_type == :literal_integer
|
|
end)
|
|
|
|
# "[1 22 -3]"
|
|
assert list_node.location == [0, 1, 1, 9, 1, 10]
|
|
end
|
|
|
|
test "parses a simple list with symbols" do
|
|
source = "[foo bar baz]"
|
|
{list_node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
# 1 file_node, 1 list_expr, 3 symbols = 5 nodes
|
|
assert map_size(nodes_map) == 5
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
refute is_nil(list_node)
|
|
assert list_node.ast_node_type == :list_expression
|
|
assert list_node.parent_id == file_node.id
|
|
assert length(list_node.children) == 3
|
|
children_nodes = Enum.map(list_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
assert Enum.map(children_nodes, & &1.name) == ["foo", "bar", "baz"]
|
|
end
|
|
|
|
test "parses nested lists" do
|
|
source = "[a [b 1] c]"
|
|
{outer_list, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
# Nodes: 1 file_node, outer_list, a, inner_list, c, b, 1 => 7 nodes
|
|
assert map_size(nodes_map) == 7
|
|
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
refute is_nil(outer_list)
|
|
assert outer_list.ast_node_type == :list_expression
|
|
assert outer_list.parent_id == file_node.id
|
|
assert length(outer_list.children) == 3
|
|
|
|
# 'a'
|
|
child1 = TestHelpers.get_nth_child_node(nodes_map, 0, outer_list.id)
|
|
# '[b 1]'
|
|
inner_list = TestHelpers.get_nth_child_node(nodes_map, 1, outer_list.id)
|
|
# 'c'
|
|
child3 = TestHelpers.get_nth_child_node(nodes_map, 2, outer_list.id)
|
|
|
|
assert child1.ast_node_type == :symbol && child1.name == "a"
|
|
assert inner_list.ast_node_type == :list_expression
|
|
assert child3.ast_node_type == :symbol && child3.name == "c"
|
|
|
|
assert inner_list.parent_id == outer_list.id
|
|
assert length(inner_list.children) == 2
|
|
# 'b'
|
|
grandchild1 = TestHelpers.get_nth_child_node(nodes_map, 0, inner_list.id)
|
|
# '1'
|
|
grandchild2 = TestHelpers.get_nth_child_node(nodes_map, 1, inner_list.id)
|
|
|
|
assert grandchild1.ast_node_type == :symbol && grandchild1.name == "b"
|
|
assert grandchild2.ast_node_type == :literal_integer && grandchild2.value == 1
|
|
end
|
|
|
|
test "parses lists with varied spacing" do
|
|
source = "[ foo 1\nbar ]"
|
|
{list_node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
refute is_nil(list_node)
|
|
assert list_node.ast_node_type == :list_expression
|
|
assert list_node.parent_id == file_node.id
|
|
assert length(list_node.children) == 3
|
|
|
|
children_names_values =
|
|
Enum.map(list_node.children, fn id ->
|
|
node = TestHelpers.get_node_by_id(nodes_map, id)
|
|
if node.ast_node_type == :symbol, do: node.name, else: node.value
|
|
end)
|
|
|
|
assert children_names_values == ["foo", 1, "bar"]
|
|
end
|
|
|
|
test "handles unclosed list" do
|
|
source = "[foo bar"
|
|
{list_node, nodes_map} = TestHelpers.parse_and_get_first_node(source)
|
|
# Expect 1 file_node, 1 list_expression node (error), 2 symbol nodes = 4 nodes
|
|
assert map_size(nodes_map) == 4
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
refute is_nil(list_node)
|
|
assert list_node.ast_node_type == :list_expression
|
|
assert list_node.parent_id == file_node.id
|
|
assert list_node.parsing_error == "Unclosed list"
|
|
# foo, bar
|
|
assert length(list_node.children) == 2
|
|
# "[foo bar"
|
|
assert list_node.location == [0, 1, 1, 8, 1, 9]
|
|
end
|
|
|
|
test "handles unexpected closing square bracket at top level" do
|
|
source = "foo ] bar"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
# 1 file_node, foo, error_node_for_], bar = 4 nodes
|
|
assert map_size(nodes_map) == 4
|
|
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
|
|
top_level_children =
|
|
Enum.map(file_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
error_node =
|
|
Enum.find(top_level_children, &(&1.ast_node_type == :unknown && &1.raw_string == "]"))
|
|
|
|
refute is_nil(error_node)
|
|
assert error_node.parent_id == file_node.id
|
|
assert error_node.parsing_error == "Unexpected ']'"
|
|
# location of "]"
|
|
assert error_node.location == [4, 1, 5, 5, 1, 6]
|
|
|
|
symbol_foo =
|
|
Enum.find(top_level_children, &(&1.ast_node_type == :symbol && &1.name == "foo"))
|
|
|
|
refute is_nil(symbol_foo)
|
|
assert symbol_foo.parent_id == file_node.id
|
|
|
|
symbol_bar =
|
|
Enum.find(top_level_children, &(&1.ast_node_type == :symbol && &1.name == "bar"))
|
|
|
|
refute is_nil(symbol_bar)
|
|
assert symbol_bar.parent_id == file_node.id
|
|
end
|
|
|
|
test "parses a list with mixed elements including strings, S-expressions, and other lists" do
|
|
source = "[1 'hello' (a b) [x y] 'end']"
|
|
{:ok, nodes_map} = Parser.parse(source)
|
|
|
|
# Expected items: 1 outer list, 1 int, 1 str, 1 s-expr (with 2 sym children), 1 inner list (with 2 sym children), 1 str
|
|
# Node counts: outer_list (1) + int (1) + str (1) + s-expr (1) + sym_a (1) + sym_b (1) + inner_list (1) + sym_x (1) + sym_y (1) + str_end (1) = 10 nodes
|
|
# Plus 1 file_node = 11 nodes
|
|
assert map_size(nodes_map) == 11
|
|
|
|
file_node = Enum.find(Map.values(nodes_map), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node)
|
|
outer_list_node = TestHelpers.get_first_child_node(nodes_map)
|
|
|
|
refute is_nil(outer_list_node)
|
|
assert outer_list_node.ast_node_type == :list_expression
|
|
assert outer_list_node.parent_id == file_node.id
|
|
assert length(outer_list_node.children) == 5
|
|
|
|
children = Enum.map(outer_list_node.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
# Child 1: Integer 1
|
|
assert Enum.at(children, 0).ast_node_type == :literal_integer
|
|
assert Enum.at(children, 0).value == 1
|
|
|
|
# Child 2: String 'hello'
|
|
assert Enum.at(children, 1).ast_node_type == :literal_string
|
|
assert Enum.at(children, 1).value == "hello"
|
|
|
|
# Child 3: S-expression (a b)
|
|
s_expr_child = Enum.at(children, 2)
|
|
assert s_expr_child.ast_node_type == :s_expression
|
|
assert length(s_expr_child.children) == 2
|
|
|
|
s_expr_children =
|
|
Enum.map(s_expr_child.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
assert Enum.map(s_expr_children, & &1.name) == ["a", "b"]
|
|
|
|
# Child 4: List [x y]
|
|
inner_list_child = Enum.at(children, 3)
|
|
assert inner_list_child.ast_node_type == :list_expression
|
|
assert length(inner_list_child.children) == 2
|
|
|
|
inner_list_children =
|
|
Enum.map(inner_list_child.children, &TestHelpers.get_node_by_id(nodes_map, &1))
|
|
|
|
assert Enum.map(inner_list_children, & &1.name) == ["x", "y"]
|
|
|
|
# Child 5: String 'end'
|
|
assert Enum.at(children, 4).ast_node_type == :literal_string
|
|
assert Enum.at(children, 4).value == "end"
|
|
end
|
|
|
|
test "symbol cannot contain brackets" do
|
|
# This should be parsed as symbol "foo[bar]" with current regex
|
|
source = "foo[bar]"
|
|
# After regex change to exclude brackets from symbols:
|
|
# It should be symbol "foo", then an unclosed list "[", then symbol "bar", then error for "]"
|
|
# Or, if we want `foo[bar]` to be an error or specific construct, tokenizer needs more rules.
|
|
# For now, with `[^\s\(\)\[\]]+`, "foo[bar]" is not a single symbol.
|
|
# It would be: "foo" (symbol), then "[" (start list), then "bar" (symbol), then error (unclosed list).
|
|
# Let's test "foo["
|
|
source1 = "foo["
|
|
{:ok, nodes_map1} = Parser.parse(source1)
|
|
# 1 file_node, "foo" symbol, "[" (unclosed list_expression) = 3 nodes
|
|
assert map_size(nodes_map1) == 3
|
|
file_node1 = Enum.find(Map.values(nodes_map1), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node1)
|
|
children1 = Enum.map(file_node1.children, &TestHelpers.get_node_by_id(nodes_map1, &1))
|
|
|
|
foo_node = Enum.find(children1, &(&1.name == "foo"))
|
|
list_node = Enum.find(children1, &(&1.ast_node_type == :list_expression))
|
|
|
|
refute is_nil(foo_node)
|
|
assert foo_node.parent_id == file_node1.id
|
|
refute is_nil(list_node)
|
|
assert list_node.parent_id == file_node1.id
|
|
assert list_node.parsing_error == "Unclosed list"
|
|
assert list_node.children == []
|
|
|
|
# Test "foo[bar"
|
|
source2 = "foo[bar"
|
|
{:ok, nodes_map2} = Parser.parse(source2)
|
|
# 1 file_node, "foo" symbol, "[" (list_expression), "bar" symbol inside list = 4 nodes
|
|
assert map_size(nodes_map2) == 4
|
|
file_node2 = Enum.find(Map.values(nodes_map2), &(&1.ast_node_type == :file))
|
|
refute is_nil(file_node2)
|
|
children2 = Enum.map(file_node2.children, &TestHelpers.get_node_by_id(nodes_map2, &1))
|
|
|
|
foo_node2 = Enum.find(children2, &(&1.name == "foo"))
|
|
list_node2 = Enum.find(children2, &(&1.ast_node_type == :list_expression))
|
|
|
|
refute is_nil(foo_node2)
|
|
assert foo_node2.parent_id == file_node2.id
|
|
refute is_nil(list_node2)
|
|
assert list_node2.parent_id == file_node2.id
|
|
assert list_node2.parsing_error == "Unclosed list"
|
|
assert length(list_node2.children) == 1
|
|
bar_node_id = hd(list_node2.children)
|
|
bar_node = TestHelpers.get_node_by_id(nodes_map2, bar_node_id)
|
|
assert bar_node.name == "bar"
|
|
assert bar_node.parent_id == list_node2.id
|
|
end
|
|
end
|
|
end
|