kacper/elipl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
elipl/tpl.exs
Kacper Marzecki 8d33ae0172 add new parser script
2026-05-09 01:47:31 +02:00

1166 lines
35 KiB
Elixir
Raw Blame History

Mix.install([:jason])
Code.require_file("casex.exs")
defmodule Tpl do
# improved case with multiple patterns and guards
import Casex
@newline "\n" |> to_charlist() |> hd()
@tab "\t" |> to_charlist() |> hd()
@space " " |> to_charlist() |> hd()
@lparen "(" |> to_charlist() |> hd()
@rparen ")" |> to_charlist() |> hd()
@lbrace "{" |> to_charlist() |> hd()
@rbrace "}" |> to_charlist() |> hd()
@dquote "\"" |> to_charlist() |> hd()
@comma "," |> to_charlist() |> hd()
@dot "." |> to_charlist() |> hd()
@semi ";" |> to_charlist() |> hd()
def ch(str), do: str |> to_charlist() |> hd()
##### DEBUGGING HELPERS #####
def print_chunkd(%{program_output: output} = state) do
print_chunkd(output)
state
end
def print_chunkd(ast, indent \\ 0) do
# pretty print state.program_output here, like a tree
# recursively, with `|` for keeping track of levels
# like so:
# S-expression:
# | Token: a
# | Token: b
# Brace block:
# | Token: x
# | Token: y
# | S-expression:
# | | S-expression:
# | | | Token: a
# | Token: b
#
# ! note the `|` pipe used for levels
Enum.each(ast, fn item ->
print_item(item, indent)
end)
ast
end
def print_item(%{t: :sexp, list: list}, indent) do
IO.puts(String.duplicate("| ", indent) <> "S-expression:")
Enum.each(list, fn item -> print_item(item, indent + 1) end)
end
def print_item(%{t: :brace, list: list, name: %{str: name, t: :token}}, indent) do
IO.puts(String.duplicate("| ", indent) <> "Brace block: (#{name})")
Enum.each(list, fn item -> print_item(item, indent + 1) end)
end
def print_item(%{t: :brace, list: list}, indent) do
IO.puts(String.duplicate("| ", indent) <> "Brace block:")
Enum.each(list, fn item -> print_item(item, indent + 1) end)
end
def print_item(%{t: :token, str: str}, indent) do
IO.puts(String.duplicate("| ", indent) <> "Token: #{str}")
end
def print_item(%{t: :string, chunks: chunks, values: values}, indent) do
IO.puts(String.duplicate("| ", indent) <> "String:")
max_len = max(length(chunks), length(values))
Enum.each(0..(max_len - 1), fn i ->
if i < length(chunks) do
chunk = Enum.at(chunks, i)
IO.puts(String.duplicate("| ", indent + 1) <> "Chunk: \"#{chunk}\"")
end
if i < length(values) do
value = Enum.at(values, i)
IO.puts(String.duplicate("| ", indent + 1) <> "Value:")
print_item(value, indent + 2)
end
end)
end
def print_item(%{t: :comment, str: str}, indent) do
IO.puts(String.duplicate("| ", indent) <> "Comment: #{str}")
end
def print_item(%{t: :comma}, indent) do
IO.puts(String.duplicate("| ", indent) <> "Comma")
end
def print_item(%{t: :newline} = _other, indent) do
IO.puts(String.duplicate("| ", indent) <> "Newline")
end
def print_item(%{t: :dot} = _other, indent) do
IO.puts(String.duplicate("| ", indent) <> "Dot access")
end
def print_item(nil, indent) do
IO.puts(String.duplicate("| ", indent) <> "Unknown (maybe piped?)")
end
def print_item(other, indent) do
IO.puts(String.duplicate("| ", indent) <> "Unknown item: #{inspect(other)}")
end
##### END DEBUGGING HELPERS #####
def add_offset(state, o) do
%{state | offset: state.offset + o}
end
def drop_input(state, n) do
%{state | input: Enum.drop(state.input, n)}
end
# Adds a character to the current expression being built
def add_to_exp(%{exp: exp} = state, char) do
exp = %{exp | chars: [char | exp.chars]}
%{state | exp: exp}
end
# Starts a new expression
def add_to_exp(state, char) do
Map.put(state, :exp, %{start: state.offset, chars: [char]})
end
# Adds a character to the current string chunk being built
def add_to_exp(%{exp: %{t: :string} = exp} = state, char, type) when type == :string do
exp = %{exp | current_chunk_chars: [char | exp.current_chunk_chars]}
%{state | exp: exp}
end
def add_to_exp(%{exp: %{t: :comment} = exp} = state, char, type) when type == :comment do
exp = %{exp | chars: [char | exp.chars]}
%{state | exp: exp}
end
# Starts a new string expression
def add_to_exp(state, char, type) when type == :comment do
Map.put(state, :exp, %{
start: state.offset,
t: :comment,
chars: [char]
})
end
# adds a exp to output
def add_single_token(state, str, type \\ :token) do
comma_exp = %{
start: state.offset,
end: state.offset + 1,
t: type,
str: str
}
Map.update!(state, :output, fn r -> [comma_exp | r] end)
end
# Finalizes the current expression and adds it to the current output list
def end_exp(%{exp: %{t: :string} = exp} = state) do
# Finalize the last chunk if it exists
#
exp =
if exp.current_chunk_chars && exp.current_chunk_chars != [] do
chunk_str = exp.current_chunk_chars |> Enum.reverse() |> to_string
%{exp | chunks: [chunk_str | exp.chunks], current_chunk_chars: []}
else
exp
end
# Reverse chunks and values to maintain original order
str_exp = %{exp | chunks: Enum.reverse(exp.chunks), values: Enum.reverse(exp.values)}
str_exp = str_exp |> Map.put(:end, state.offset) |> Map.delete(:current_chunk_chars)
state
|> Map.update!(:output, fn r -> [str_exp | r] end)
|> Map.delete(:exp)
end
def end_exp(%{exp: %{t: :comment} = exp} = state) do
str = exp.chars |> Enum.reverse() |> to_string
exp =
exp
|> Map.put(:str, str)
|> Map.put(:t, :comment)
|> Map.put(:end, state.offset)
|> Map.delete(:chars)
state
|> Map.update!(:output, fn r -> [exp | r] end)
|> Map.delete(:exp)
end
def end_exp(%{exp: exp} = state) do
str = exp.chars |> Enum.reverse() |> to_string
exp =
exp
|> Map.put(:str, str)
|> Map.put(:t, :token)
|> Map.put(:end, state.offset)
|> Map.delete(:chars)
state
|> Map.update!(:output, fn r -> [exp | r] end)
|> Map.delete(:exp)
end
# If no expression is active, do nothing.
def end_exp(state), do: state
# Finalizes the current expression if one exists
def end_exp_if_exists(state) do
if Map.has_key?(state, :exp), do: end_exp(state), else: state
end
# Reverses the final program_output list
def end_program_output_list(%{output: output} = state) do
%{state | program_output: output |> hd() |> Map.get(:list)}
end
# ends a token and puts it as `:name` in the new brace block
def start_named_brace(state) do
state = end_exp_if_exists(state)
[name_token | rest_output] = state.output
# Push current output onto stack and start a new empty output list for the sublist
# and put th e name_token as :name in the new brace block
%{
state
| sublist_stack: [%{t: :brace, output: rest_output, name: name_token} | state.sublist_stack],
output: []
}
end
# Starts a new sublist (for parentheses or braces)
def start_sublist(state, type) do
# Finalize any current token before starting a sublist
state = end_exp_if_exists(state)
# Push current output onto stack and start a new empty output list for the sublist
%{state | sublist_stack: [%{t: type, output: state.output} | state.sublist_stack], output: []}
end
# Ends the current sublist and adds it to the parent list
# def end_sublist(%{sublist_stack: [%{t: :string_interp} | _sublist_stack]} = _state) do
# # This function should not be called for :string_interp directly.
# # String interpolation end is handled specially in the chunk function.
# raise "end_sublist should not be called for :string_interp directly"
# end
def end_paren(state) do
state = end_exp_if_exists(state)
# Pop the parent output list from the stack
[%{t: :paren, output: parent_output} | sublist_stack] = state.sublist_stack
# Reverse the current output (which is the paren sublist) and add it to the parent output
paren_sublist = Enum.reverse(state.output)
%{
state
| sublist_stack: sublist_stack,
output: [%{t: :sexp, list: paren_sublist} | parent_output]
}
end
def end_brace(state) do
state = end_exp_if_exists(state)
# Pop the parent output list from the stack
[%{t: :brace, output: parent_output} = brace | sublist_stack] = state.sublist_stack
# Reverse the current output (which is the brace sublist) and add it to the parent output
brace_sublist = Enum.reverse(state.output)
brace =
Map.merge(
%{t: :brace, list: brace_sublist},
if(Map.has_key?(brace, :name), do: %{name: brace.name}, else: %{})
)
%{
state
| sublist_stack: sublist_stack,
output: [brace | parent_output]
}
end
# def end_implicit_sexp(state) do
# state = end_exp_if_exists(state)
# # Pop the parent output list from the stack
# [%{t: :implicit_sexp, output: parent_output} | sublist_stack] = state.sublist_stack
# # Reverse the current output (which is the implicit sexp) and add it to the parent output
# sexp = Enum.reverse(state.output)
# # if the only thing in implicit_sexp is a sexp , we add that directly to parent_output
# sexp =
# case sexp do
# [%{t: :token} = token] -> token
# [%{t: :string} = string] -> string
# # [%{t: :sexp, list: [_]} = sexp] -> sexp
# [%{t: :sexp} = sexp] -> sexp
# [] -> []
# values -> %{t: :sexp, list: values}
# end
#
# %{
# state
# | sublist_stack: sublist_stack,
# output:
# case sexp do
# [] -> parent_output
# _ -> [sexp | parent_output]
# end
# }
# end
#
# def maybe_end_implicit_sexp(%{sublist_stack: [%{t: :implicit_sexp} | _]} = state) do
# end_implicit_sexp(state)
# end
#
# def maybe_end_implicit_sexp(state), do: state
def end_interpolation(state) do
state = end_exp_if_exists(state)
# Pop the parent output list from the stack
[%{t: :string_interp, exp: string_exp, output: parent_output} | sublist_stack] =
state.sublist_stack
[brace] = state.output
# The interpolated value is the entire current output list
# interpolated_value = %{t: :sexp, list: state.output |> Enum.reverse()}
# IO.inspect("Interpolated value: #{inspect(interpolated_value)}")
updated_string_exp = %{
string_exp
| values: [brace | string_exp.values]
}
Map.merge(state, %{
sublist_stack: sublist_stack,
exp: updated_string_exp,
# Restore parent output
output: parent_output
})
end
# Helper to finalize a top-level line (current output) and add it to program_output
defp finalize_top_level_line(state) do
# Ensure any pending exp is closed
state = end_exp_if_exists(state)
# Reverse the current line's tokens/sublists and add to program_output
line_content = Enum.reverse(state.output)
new_program_output =
case line_content do
[] ->
state.program_output
_ ->
# The line content is already wrapped in an implicit_sexp,
# which becomes a regular sexp when the sublist is ended.
# So we just prepend it.
line_content ++ state.program_output
end
# Reset output for the next line
%{state | program_output: new_program_output, output: []}
end
def debug_state(state) do
IO.puts("----- -----")
case state.input do
[] ->
IO.puts("CONSUMED ALL INPUT")
IO.puts(state.start_input |> to_string())
input ->
state.start_input
|> String.trim_trailing(input |> to_string())
|> IO.inspect(label: "SO FAR")
end
IO.inspect(state.mode |> Enum.reverse(), limit: :infinity, label: "MODE")
Enum.at(state.input, 0)
|> List.wrap()
|> to_string
|> IO.inspect(label: "NEXT CHAR", charlists: :as_lists)
state
end
# Stage 1 - chunk into lists, tokens, strings, comments, newlines
def chunk(str) when is_binary(str) do
input = str |> String.to_charlist()
%{
offset: -1,
# Accumulates finalized lines
program_output: [],
# Accumulates tokens/sublists for the current line/sublist
output: [],
sublist_stack: [],
input: input,
mode: [],
start_input: str
}
|> start_sublist(:brace)
|> chunk(input, [:brace])
# |> chunk
|> end_brace()
|> end_program_output_list()
end
def chunk(%{input: input, mode: mode} = state) do
# debug_state(state)
if mode == [], do: raise("Mode stack empty!")
casex {input, mode} do
# comments
{[char | rest], _} when char == @semi ->
{state, _} =
Enum.reduce_while(
rest,
# state, mode (either :comment or :after_comment)
{state |> end_exp_if_exists() |> drop_input(1), :comment},
fn
@newline, {st, :comment} ->
st = st |> add_offset(1) |> drop_input(1)
{:cont, {st, :after_comment}}
c, {st, :comment} ->
st = st |> add_to_exp(c, :comment) |> add_offset(1) |> drop_input(1)
{:cont, {st, :comment}}
c, {st, :after_comment} when c in [@space, @tab] ->
st = st |> add_offset(1) |> drop_input(1)
{:cont, {st, :after_comment}}
@semi, {st, :after_comment} ->
st = st |> add_to_exp(@newline, :comment) |> add_offset(1) |> drop_input(1)
{:cont, {st, :comment}}
_c, {st, :after_comment} ->
# finalize comment exp
st = st |> end_exp_if_exists()
{:halt, {st, nil}}
end
)
state |> chunk(state.input, mode)
{[char | rest], [:token | [:paren | prev_mode]]}, {[char | rest], [:paren | prev_mode]}
when char == @rparen ->
state
|> end_exp_if_exists()
|> add_offset(1)
|> end_paren()
|> chunk(rest, prev_mode)
# Handle opening brace inside a string (interpolation)
{[char | rest], [:string | _prev_mode] = mode} when char == @lbrace ->
state =
state
# Finalize the current string chunk before starting interpolation
|> (fn s ->
chunk_str = s.exp.current_chunk_chars |> Enum.reverse() |> to_string
%{s | exp: %{s.exp | chunks: [chunk_str | s.exp.chunks], current_chunk_chars: []}}
end).()
state
|> add_offset(1)
# Push the current string expression AND output onto the stack
|> Map.update!(:sublist_stack, fn stack ->
[%{t: :string_interp, exp: state.exp, output: state.output} | stack]
end)
# Clear current exp and output to chunk the interpolated expression
|> Map.put(:output, [])
|> Map.delete(:exp)
|> start_sublist(:brace)
# Enter top mode to chunk the expression
|> chunk(rest, [:brace | mode])
{[char | rest], [m | _] = mode} when char == @dot and m in [:token, :brace, :paren] ->
state
|> end_exp_if_exists()
|> add_offset(1)
|> add_single_token(".", :dot)
|> chunk(rest, mode)
{[char | rest], [:token | [:brace | _] = mode]}, {[char | rest], [:brace | _] = mode}
when char == @comma ->
# commas separate implicit sexps
state
|> end_exp_if_exists()
|> add_offset(1)
|> add_single_token(",", :comma)
|> chunk(rest, mode)
# Handle closing brace inside string interpolation (with active token)
# Handle closing brace inside string interpolation (no active token)
{[char | rest], [:token, :brace | [:string | _prev_mode] = string_mode] = mode},
{[char | rest], [:brace | [:string | _prev_mode] = string_mode]},
guard([char == @rbrace]) ->
state
# Finalize the interpolated expression
|> end_exp_if_exists()
|> add_offset(1)
|> end_brace()
# Pop the string expression from the stack
|> end_interpolation()
# Return to string mode
|> chunk(rest, string_mode)
# handle closing brace when in :implicit_sexp in brace, token and top
{[char | rest], [:token | [:brace | prev_mode]]}, {[char | rest], [:brace | prev_mode]}
when char == @rbrace ->
state
|> end_exp_if_exists()
|> add_offset(1)
|> end_brace()
|> chunk(rest, prev_mode)
# Handle opening parenthesis, entering paren mode
{[char | rest], _} when char == @lparen ->
state
# Finalize any current token before starting a sublist
|> end_exp_if_exists()
|> add_offset(1)
|> start_sublist(:paren)
|> chunk(rest, [:paren | mode])
# Handle NAMED Brace opening
{[char | rest], [:token | _prev_mode]} when char == @lbrace ->
state
# Finalize any current token before starting a sublist
|> end_exp()
|> add_offset(1)
|> start_named_brace()
|> chunk(rest, [:brace | mode])
# Handle opening brace, entering brace mode
{[char | rest], _} when char == @lbrace ->
state
# Finalize any current token before starting a sublist
|> end_exp_if_exists()
|> add_offset(1)
|> start_sublist(:brace)
|> chunk(rest, [:brace | mode])
# Handle closing double quote, exiting string mode
{[char | rest], [:string | prev_mode]} when char == @dquote ->
state
|> add_offset(1)
# Finalize the string expression
|> end_exp()
|> chunk(rest, prev_mode)
# Handle opening double quote, entering string mode
{[char | rest], _} when char == @dquote ->
state
# Finalize any preceding token
|> end_exp_if_exists()
|> add_offset(1)
|> Map.put(:exp, %{
start: state.offset,
t: :string,
chunks: [],
values: [],
current_chunk_chars: []
})
|> chunk(rest, [:string | mode])
# # Newline after a token in a top-level implicit sexp. Ends the line.
# {[@newline | rest], [:token | [:top | _] = prev_mode]} ->
# state
# |> end_exp()
# |> add_offset(1)
# |> chunk(rest, prev_mode)
# Newline after a token in a brace's implicit sexp. Ends the sexp, not the line.
{[@newline | rest], [:token | [:brace | _] = prev_mode]} ->
state
|> end_exp()
|> add_offset(1)
|> add_single_token(@newline, :newline)
|> chunk(rest, prev_mode)
# Newline after a token in a paren. Just whitespace.
{[@newline | rest], [:token | [:paren | _] = prev_mode]} ->
state
|> end_exp()
|> add_offset(1)
|> add_single_token(@newline, :newline)
|> chunk(rest, prev_mode)
{[@newline | rest], [:brace | _] = mode} ->
state
|> end_exp()
|> add_offset(1)
|> add_single_token(@newline, :newline)
|> chunk(rest, mode)
# # Newline on its own in a top-level implicit sexp. Ends the line.
# {[@newline | rest], [:implicit_sexp, :top | _] = mode} ->
# state
# |> add_offset(1)
# |> end_implicit_sexp()
# |> finalize_top_level_line()
# |> chunk(rest, Enum.drop(mode, 1))
#
# # Newline on its own in a brace's implicit sexp. Ends the sexp.
# {[@newline | rest], [:implicit_sexp, :brace | _] = mode} ->
# state
# |> add_offset(1)
# |> end_implicit_sexp()
# |> chunk(rest, Enum.drop(mode, 1))
# Newline on an empty top-level line.
# {[@newline | rest], [:top] = mode} ->
# state
# |> add_offset(1)
# |> finalize_top_level_line()
# |> chunk(rest, mode)
# Newline as whitespace in other contexts (paren, brace).
{[@newline | rest], _mode} ->
add_offset(state, 1)
|> chunk(rest, mode)
# Whitespace ending a token
{[end_token | rest], [:token | prev_mode]} when end_token in [@tab, @space] ->
state
|> add_offset(1)
|> end_exp()
|> chunk(rest, prev_mode)
# Characters into current string chunk (including newlines and some special chars)
{[char | rest], [:string | _]} ->
add_offset(state, 1)
|> add_to_exp(char, :string)
|> chunk(rest, mode)
# Whitespace (not ending a token)
{[whitespace | rest], _} when whitespace in [@tab, @space] ->
add_offset(state, 1)
|> chunk(rest, mode)
# Characters into current token
{[char | rest], [:token | _]} ->
add_offset(state, 1)
|> add_to_exp(char)
|> chunk(rest, mode)
# A new token (from :top, :paren, :brace, or :implicit_sexp mode)
{[char | rest], [current_mode | _] = mode}
when current_mode in [:paren, :brace] and
char not in [@tab, @space, @newline, @dquote, @rbrace, @rparen, @lbrace, @lparen] ->
state =
state
|> end_exp_if_exists()
|> add_offset(1)
# If in :top or :brace, start an implicit s-expression for the line
# {state, mode} =
# if current_mode in [:top, :brace] do
# {start_sublist(state, :implicit_sexp), [:implicit_sexp | mode]}
# else
# {state, mode}
# end
state
|> add_to_exp(char)
|> chunk(rest, [:token | mode])
# End of input, with an active token
# TODO: better way to collapse open sublists
{[], [:token | _]} ->
state
|> end_exp()
# |> (fn s ->
# Enum.reduce(s.sublist_stack, s, fn %{t: type}, acc ->
# if type == :implicit_sexp, do: end_implicit_sexp(acc), else: acc
# end)
# end).()
|> finalize_top_level_line()
|> (fn s ->
Enum.reduce(s.sublist_stack, s, fn _, acc ->
case acc.sublist_stack do
[] -> acc
# [%{t: :implicit_sexp} | _] -> end_implicit_sexp(acc)
[%{t: :paren} | _] -> end_paren(acc)
[%{t: :brace} | _] -> end_brace(acc)
end
end)
end).()
|> end_program_output_list()
# End of input, no active token
{[], modes} ->
Enum.reverse(modes)
|> Enum.drop(2)
|> Enum.reverse()
|> Enum.reduce(state, fn
# :implicit_sexp, %{output_stack: [%{t: :implicit_sexp} | _]} = acc ->
# end_implicit_sexp(acc)
:paren, %{sublist_stack: [%{t: :paren} | _]} = acc ->
end_paren(acc)
:brace, %{sublist_stack: [%{t: :brace} | _]} = acc ->
end_brace(acc)
_, acc ->
acc
end)
# Catch-all for unhandled characters
{[char | rest], mode} ->
IO.inspect("Unhandled char '#{[char] |> to_string}' in mode #{inspect(mode)}")
add_offset(state, 1) |> chunk(rest, mode)
end
end
def chunk(state, input, mode) do
chunk(%{state | input: input, mode: mode})
end
def run() do
"""
parent elizabeth charles
parent charles william
parent charles harry
- - -
male charles
male william
male harry
female elizabeth
- - -
age charles 75
age william 42
age harry 40
- - -
person X {
male X
}
- - -
grandparent G C {
parent G P
parent P C
(G \= C)
}
- - -
is_adult Person {
age Person A
A >= 18
}
- - -
dcg command (cmd Action Target) {
verb Action
noun Target
}
- - -
dcg verb (action find) { "find" }
dcg noun (obj harry) { "harry" }
- - -
Adults = find P {
person P
is_adult P
}
- - -
A (B
C)
D E F
G H
- - -
( f (a (b kek)
d
e)
f )
- - -
{ this is { (a) b } of {
( braces )
braces 2
} } asd
- - -
"hello
world"
- - -
"a {n} { nf "{ nn1 } b {nn2 nn3}" } c"
- - -
a b c , b, c
{ x x c , y x (asd ) , ( z ) }
( p, q, "r, asdad, ")
- - -
dcg asd X Y [ x ] {
"hello {X
a b} and {Y}"
}
- - -
grandparent G C x{
parent G P
parent P C ; and some comment here
(G = C)
}
; somecomment here
m{ x x c , y x (asd ) , z }
- - -
m{a b, b c d , c}
- - -
map "n" "<leader>ff" f{vim.lsp.buf.format() } t{ desc "Format code" }
- - -
arg1 = implicit sexp ending on a newline
arg2 = another implicit sexp ending on newline
: hof fn elem {
transform elem :someatom
} arg3_to_hof
res = somefun arg1 arg2
: Enum.map fn elem {
transform elem :someatom
}
: length
: plus 2
: IO.inspect k{label "Transformed length"}
: fetch_some_map
: .mapkey
; some comment
; asd
- - -
res : op1 : op2 : op3
: op4 ; asd
: asd ; asd
- - -
res s d = asd x yy : . q : e
: map ( fn i (add i 1) )
: . length, asdklj
res . length . positive?
: IO.println
- - -
; instead of
; map("n", "<leader>srf", function()
; local file_path = vim.api.nvim_buf_get_name(0)
; print(file_path)
; require("grug-far").open({
; transient = true,
; prefills = {
; paths = file_path,
; },
; })
map "n" "<leader>stf" fn {
l timer_minutes = (choose_timer)
if not timer_minutes {
print "No timer selected"
}
require "grug-far"
: .open t{
transient true
prefills t{
timers (timer_minutes : at 1 : tostring)
}
}
}
; thats it folks
"""
|> String.split("- - -\n")
|> Enum.each(fn input ->
IO.puts("===================== Input ====")
IO.puts(input)
# IO.puts("==== chunkd ====")
state =
input
|> chunk()
# |> IO.inspect()
# |> print_chunkd()
chunks = Map.get(state, :program_output)
IO.puts("==== AST ====")
chunks
|> chunks_to_ast()
|> print_chunkd()
# |> IO.inspect(label: "AST")
IO.puts("\n\n")
end)
end
def to_ast(str) do
str
|> chunk()
# |> print_chunkd()
|> chunks_to_ast
end
# ============================================================================
# Pass 2: Structuring Pass (Chunks -> AST)
# ============================================================================
# chunks are currently just lists of tokens / other lists
# among these tokens are special tokens like :comma , :newline, and infix tokens ":", ".", etc
# theses , along with the type of list that contains them, will determine the AST structure
# 1. sexp lists with parens ()
# , / spaces / nelines delimit expressions, expressions are single tokens or sublists, OR binary operations (: . || etc )
# (a b c : d e, g . f (h i) ) -> AST representing function call with arguments and method calls
# elements:
# - a (separated by space from the next element)
# - b (separated by space from the next element)
# - c: d e (binary operation with left = c , operator = : , right = (d e) ) (separated by comma / newline, from the next element)
# - g . f (binary operation with left = g , operator = . , right = f )
# - (h i) (separated by space from the previous element)
# 2. brackets with {}
# spaces dont delimit new elements, only commas and newlines do
# , / spaces / newlines delimit expressions, expressions are single tokens or sublists, OR binary operations (: . || etc )
# { a b
# c : d e
# : x, g . f (h i) }
# - a b (separated by newline from the next element)
# - c piped into d e and then into x (binary operation with left = (c) , operator = : , right = (d e : x ) ) (separated by comma / newline, from the next element)
# - g . f (binary operation with left = g , operator = . , right = f )
#
# also handle infix operations here
# This pass takes the raw nodes from the chunker and applies grammatical rules
# (like operator precedence) to build a meaningful AST.
@operator_table %{
# "token_string" => {precedence, :left | :right}
# Lower precedence number binds less tightly.
# assignment
"=" => {10, :right},
# pipe operator
":" => {50, :right},
# method call / property access
"." => {80, :right}
}
def chunks_to_ast(%{program_output: chunks}) do
chunks_to_ast(chunks)
end
def chunks_to_ast(chunks) do
# The top-level program is a list of expressions delimited by newlines, like a brace block.
structure_expressions(chunks, :brace)
end
defp structure_node(%{t: :brace, list: raw_nodes} = node) do
# For brace blocks, group by newline/comma and process each expression.
structured_list = structure_expressions(raw_nodes, :brace)
%{node | list: structured_list}
end
defp structure_node(%{t: :sexp, list: raw_nodes} = node) do
# For paren s-expressions, the contents are comma-separated expressions.
# We process the groups, but the result is a flat list, not another s-exp.
structured_list =
raw_nodes
# Recurse on sub-nodes first
|> Enum.map(&structure_node/1)
# Group by comma only
|> group_expressions(:paren)
|> Enum.map(&process_expression/1)
|> Enum.reject(&is_nil/1)
# If there's only one resulting expression and it's an sexp, flatten it.
case structured_list do
[%{t: :sexp, list: inner_list}] -> %{node | list: inner_list}
_ -> %{node | list: structured_list}
end
end
defp structure_node(%{t: :string, values: values} = node) do
# Also recurse into string interpolations
structured_values = Enum.map(values, &structure_node/1)
%{node | values: structured_values}
end
defp structure_node(node) do
# It's a simple token, string, comment, or already-processed node. Leave it as is.
node
end
defp structure_expressions(raw_nodes, type) do
raw_nodes
# 1. Recursively structure any sub-blocks first.
|> Enum.map(&structure_node/1)
# 2. Group nodes into expressions based on delimiters.
|> group_expressions(type)
# 3. Process each individual expression group.
|> Enum.map(&process_expression/1)
# 4. Filter out any nil results from empty expressions
|> Enum.reject(&is_nil/1)
# |> dbg()
end
defp group_expressions(nodes, type) do
# Manually group expressions, handling line continuations for pipes.
{groups, last_group} =
Enum.reduce(nodes, {[], []}, fn node, {acc_groups, current_group} ->
case node do
%{t: :comma} ->
# A comma always finishes the current expression.
{[Enum.reverse(current_group) | acc_groups], []}
%{t: :newline} when type == :brace ->
# A newline might finish the expression. Look ahead.
next_significant = find_next_significant(nodes, node)
is_continuation =
case next_significant do
%{t: :token, str: ":"} -> true
%{t: :token, str: "."} -> true
_ -> false
end
if is_continuation do
# It's a line continuation, add newline to current group (as whitespace) and continue.
{acc_groups, [node | current_group]}
else
# It's a delimiter, finish the current expression.
{[Enum.reverse(current_group) | acc_groups], []}
end
_ ->
# Any other token is part of the current expression.
{acc_groups, [node | current_group]}
end
end)
# Add the last group if it's not empty.
final_groups =
if last_group != [], do: [Enum.reverse(last_group) | groups], else: groups
final_groups
|> Enum.reverse()
|> Enum.reject(&(&1 == []))
end
defp find_next_significant(nodes, current_node) do
# This is a simplified lookahead. A more robust implementation might need the index.
# For now, we find the index of the current node and look from there.
case Enum.find_index(nodes, &(&1 == current_node)) do
nil ->
nil
index ->
nodes
|> Enum.drop(index + 1)
|> Enum.find(fn
%{t: :comment} -> false
%{t: :newline} -> false
_ -> true
end)
end
end
# Base case: A single token/item becomes itself.
defp process_expression([item]), do: item
# Base case: An empty list of tokens results in nil.
defp process_expression([]), do: nil
# Updated recursive case without exceptions
defp process_expression(tokens) do
# IO.inspect(tokens, label: "Processing expression")
# 1. Find all potential operators and their indices.
operators_with_indices =
tokens
|> Enum.with_index()
|> Enum.filter(fn {token, _idx} ->
# Ensure token is a map with a `str` key before checking the table
is_map(token) && Map.has_key?(@operator_table, token[:str])
end)
case operators_with_indices do
# Case 1: No operators found. This is a simple s-expression (e.g., function call).
[] ->
%{t: :sexp, list: tokens}
# |> IO.inspect(label: "Simple s-expression (no operators)")
# Case 2: Operators were found. Proceed with precedence logic.
ops ->
# 2. Find the operator with the lowest precedence (the pivot).
{_operator, pivot_index} =
Enum.min(ops, fn {token1, _idx1}, {token2, _idx2} ->
{prec1, assoc1} = @operator_table[token1.str]
{prec2, assoc2} = @operator_table[token2.str]
cond do
prec1 < prec2 ->
true
prec1 > prec2 ->
false
# Precedences are equal, use associativity to break the tie.
# For left-associative, the leftmost (first) operator wins.
# For right-associative, the rightmost (last) operator wins.
assoc1 == :left ->
true
assoc1 == :right && assoc2 == :right ->
false
true ->
false
end
end)
# |> IO.inspect(label: "Chosen pivot operator")
operator = Enum.at(tokens, pivot_index)
# 3. Split the token list into left/right sides around the pivot.
left_tokens = Enum.slice(tokens, 0, pivot_index)
right_tokens = Enum.slice(tokens, pivot_index + 1, length(tokens))
# 4. Recursively process the left and right sides.
# Special case for dot operator: RHS is a literal, not an expression.
{lhs, rhs} =
{process_expression(left_tokens), process_expression(right_tokens)}
# if operator.str == "." do
# {process_expression(left_tokens), List.first(right_tokens)}
# else
# {process_expression(left_tokens), process_expression(right_tokens)}
# end
# 5. Build the final s-expression.
%{t: :sexp, list: [operator, lhs, rhs]}
end
end
end
Tpl.run()
Reference in New Issue View Git Blame Copy Permalink