feat: Handle atom_safe and kw_identifier_safe tokens

add failing cases

Author: lukaszsamson

lib/spitfire.ex

@@ -248,13 +248,16 @@ defmodule Spitfire do
           :alias -> &parse_alias/1
           :"<<" -> &parse_bitstring/1
           :kw_identifier when is_list or is_map -> &parse_kw_identifier/1
+          :kw_identifier_safe when is_list or is_map -> &parse_kw_identifier/1
           :kw_identifier_unsafe when is_list or is_map -> &parse_kw_identifier/1
           :kw_identifier when not is_list and not is_map -> &parse_bracketless_kw_list/1
+          :kw_identifier_safe when not is_list and not is_map -> &parse_bracketless_kw_list/1
           :kw_identifier_unsafe when not is_list and not is_map -> &parse_bracketless_kw_list/1
           :int -> &parse_int/1
           :flt -> &parse_float/1
           :atom -> &parse_atom/1
           :atom_quoted -> &parse_atom/1
+          :atom_safe -> &parse_atom/1
           :atom_unsafe -> &parse_atom/1
           true -> &parse_boolean/1
           false -> &parse_boolean/1
@@ -535,6 +538,9 @@ defmodule Spitfire do
     end
   end
 
+  defp map_kw_identifier_to_atom_token(:kw_identifier_safe), do: :atom_safe
+  defp map_kw_identifier_to_atom_token(:kw_identifier_unsafe), do: :atom_unsafe
+
   defp parse_kw_identifier(%{current_token: {:kw_identifier, meta, token}} = parser) do
     trace "parse_kw_identifier", trace_meta(parser) do
       token = encode_literal(parser, token, meta)
@@ -546,9 +552,10 @@ defmodule Spitfire do
     end
   end
 
-  defp parse_kw_identifier(%{current_token: {:kw_identifier_unsafe, meta, tokens}} = parser) do
-    trace "parse_kw_identifier (unsafe)", trace_meta(parser) do
-      {atom, parser} = parse_atom(%{parser | current_token: {:atom_unsafe, meta, tokens}})
+  defp parse_kw_identifier(%{current_token: {type, meta, tokens}} = parser)
+       when type in [:kw_identifier_safe, :kw_identifier_unsafe] do
+    trace "parse_kw_identifier (#{type})", trace_meta(parser) do
+      {atom, parser} = parse_atom(%{parser | current_token: {map_kw_identifier_to_atom_token(type), meta, tokens}})
       parser = parser |> next_token() |> eat_eol()
 
       {expr, parser} = parse_expression(parser, @kw_identifier, false, false, false)
@@ -584,9 +591,10 @@ defmodule Spitfire do
     end
   end
 
-  defp parse_bracketless_kw_list(%{current_token: {:kw_identifier_unsafe, meta, tokens}} = parser) do
-    trace "parse_bracketless_kw_list (unsafe)", trace_meta(parser) do
-      {atom, parser} = parse_atom(%{parser | current_token: {:atom_unsafe, meta, tokens}})
+  defp parse_bracketless_kw_list(%{current_token: {type, meta, tokens}} = parser)
+       when type in [:kw_identifier_safe, :kw_identifier_unsafe] do
+    trace "parse_bracketless_kw_list (#{type})", trace_meta(parser) do
+      {atom, parser} = parse_atom(%{parser | current_token: {map_kw_identifier_to_atom_token(type), meta, tokens}})
       parser = parser |> next_token() |> eat_eol()
 
       atom =
@@ -1274,11 +1282,18 @@ defmodule Spitfire do
     end
   end
 
-  defp parse_atom(%{current_token: {:atom_unsafe, _, tokens}} = parser) do
-    trace "parse_atom (unsafe)", trace_meta(parser) do
+  defp parse_atom(%{current_token: {type, _, tokens}} = parser) when type in [:atom_safe, :atom_unsafe] do
+    trace "parse_atom (#{type})", trace_meta(parser) do
       meta = current_meta(parser)
       {args, parser} = parse_interpolation(parser, tokens)
-      {{{:., meta, [:erlang, :binary_to_atom]}, [{:delimiter, ~S'"'} | meta], [{:<<>>, meta, args}, :utf8]}, parser}
+
+      binary_to_atom_op =
+        case type do
+          :atom_safe -> :binary_to_existing_atom
+          :atom_unsafe -> :binary_to_atom
+        end
+
+      {{{:., meta, [:erlang, binary_to_atom_op]}, [{:delimiter, ~S'"'} | meta], [{:<<>>, meta, args}, :utf8]}, parser}
     end
   end
 

test/spitfire_test.exs

@@ -2911,6 +2911,78 @@ defmodule SpitfireTest do
     end
   end
 
+  describe "safe atoms" do
+    test "parses quoted atoms" do
+      code = """
+      :"abc#{Enum.random(1..10_000)}"
+      """
+
+      # tokenizer returns error
+      assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}}
+
+      code = """
+      :"abc\#{foo}"
+      """
+
+      # tokenizer returns atom_safe token
+      assert Spitfire.parse(code, existing_atoms_only: true) == s2q(code, existing_atoms_only: true)
+    end
+
+    test "parses quoted kw identifiers" do
+      # TODO this code crashes the parser
+      # ** (FunctionClauseError) no function clause matching in Spitfire.peek_token/1
+
+      # The following arguments were given to Spitfire.peek_token/1:
+
+      #     # 1
+      #     %{tokens: [nil | :eot], literal_encoder: nil, errors: [{[line: 1, column: 1], "missing closing bracket for list"}, {[], "unknown token: eot"}], current_token: {:"]", nil}, fuel: 150, peek_token: nil, nesting: 0}
+
+      # Attempted function clauses (showing 7 out of 7):
+
+      #     defp peek_token(%{peek_token: {:stab_op, _, token}})
+      #     defp peek_token(%{peek_token: {type, _, _, _}}) when type === :list_heredoc or type === :bin_heredoc
+      #     defp peek_token(%{peek_token: {token, _, _}})
+      #     defp peek_token(%{peek_token: {token, _}})
+      #     defp peek_token(%{peek_token: {token, _, _, _, _, _, _}})
+      #     defp peek_token(%{peek_token: :eof})
+      #     defp peek_token(%{tokens: :eot})
+
+      # code: assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}}
+      # stacktrace:
+      #   (spitfire 0.1.5) lib/spitfire.ex:2386: Spitfire.peek_token/1
+      #   (spitfire 0.1.5) lib/spitfire.ex:318: anonymous fn/7 in Spitfire.parse_expression/6
+      #   (spitfire 0.1.5) lib/spitfire/while.ex:49: Spitfire.While.do_while/2
+      #   (spitfire 0.1.5) lib/spitfire.ex:196: anonymous fn/1 in Spitfire.parse_program/1
+      #   (spitfire 0.1.5) lib/spitfire/while.ex:5: Spitfire.While2.recurse/3
+      #   (spitfire 0.1.5) lib/spitfire.ex:195: Spitfire.parse_program/1
+      #   (spitfire 0.1.5) lib/spitfire.ex:140: Spitfire.parse/2
+      #   test/spitfire_test.exs:2936: (test)
+      # code = "[\"abc#{Enum.random(1..10000)}\": 1]"
+
+      # # tokenizer return error
+      # assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}}
+
+      # TODO this code errors with a wrong message
+      # {
+      #   :error,
+      #   {:%{}, [{:closing, []}, {:line, 1}, {:column, 1}], [{:__block__, [error: true], []}]},
+      #   [{[], "unknown token: eot"}, {[], "missing closing brace for map"}]
+      # }
+
+      # code = "%{\"abc#{Enum.random(1..10000)}\": 1}"
+
+      # # tokenizer return error
+      # assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}}
+
+      code = """
+      ["abc\#{foo}": 1]
+      """
+
+      # tokenizer returns kw_identifier_safe token
+      assert Spitfire.parse(code, existing_atoms_only: true) == s2q(code, existing_atoms_only: true)
+    end
+  end
+
   defp s2q(code, opts \\ []) do
     Code.string_to_quoted(code, Keyword.merge([columns: true, token_metadata: true, emit_warnings: false], opts))
   end