Parse strings

This saves some effort and probably some performance for things like integers, but I'm mainly doing this in anticipation of string literals, because it would just be ridiculous to read code that lexes string literals twice.

src/hz_sophia.erl

@@ -4,6 +4,8 @@
 -copyright("Jarvis Carroll <spiveehere@gmail.com>").
 -license("GPL-3.0-or-later").
 
+-export([check_parser/1]).
+
 -include_lib("eunit/include/eunit.hrl").
 
 parse_literal(Type, String) ->
@@ -19,7 +21,7 @@ parse_literal2(Result, Tk, String) ->
     case next_token(Tk, String) of
         {ok, {{eof, _, _, _, _}, _, _}} ->
             {ok, Result};
-        {ok, {{_, S, Row, Start, End}, _, _}} ->
+        {ok, {{_, S, _, Row, Start, End}, _, _}} ->
             {error, {unexpected_token, S, Row, Start, End}};
         {error, Reason} ->
             {error, Reason}
@@ -27,44 +29,120 @@ parse_literal2(Result, Tk, String) ->
 
 %%% Tokenizer
 
+-define(IS_ALPHA(C), ((((C) >= $A) and ((C) =< $Z)) or (((C) >= $a) and ((C) =< $z)) or ((C) == $_))).
+-define(IS_NUM(C), (((C) >= $0) and ((C) =< $9))).
+-define(IS_ALPHANUM(C), (?IS_ALPHA(C) or ?IS_NUM(C))).
+-define(IS_HEX(C), (?IS_NUM(C) or (((C) >= $A) and ((C) =< $F)) or (((C) >= $a) and ((C) =< $f)))).
+
 next_token({tk, Row, Col}, []) ->
     {ok, {{eof, "", Row, Col, Col}, {tk, Row, Col}, []}};
 next_token({tk, Row, Col}, " " ++ Rest) ->
     next_token({tk, Row + 1, Col}, Rest);
 next_token({tk, Row, Col}, "\t" ++ Rest) ->
     next_token({tk, Row + 1, Col}, Rest);
-next_token(Tk, [N | _] = String) when N >= $0, N =< $9 ->
+next_token({tk, _, Col}, "\r\n" ++ Rest) ->
-    num_token(Tk, Tk, String, []);
+    next_token({tk, 1, Col + 1}, Rest);
-next_token(Tk, [N | _] = String) when N >= $A, N =< $Z ->
+next_token({tk, _, Col}, "\r" ++ Rest) ->
-    alphanum_token(Tk, Tk, String, []);
+    next_token({tk, 1, Col + 1}, Rest);
-next_token(Tk, [N | _] = String) when N >= $a, N =< $z ->
+next_token({tk, _, Col}, "\n" ++ Rest) ->
-    alphanum_token(Tk, Tk, String, []);
+    next_token({tk, 1, Col + 1}, Rest);
-next_token(Tk, [$_ | _] = String) ->
+next_token(Tk, [C | _] = String) when ?IS_ALPHA(C) ->
     alphanum_token(Tk, Tk, String, []);
+next_token(Tk, [C | _] = String) when ?IS_NUM(C) ->
+    num_token(Tk, Tk, String, [], 0);
+next_token({tk, Row, Col}, [$#, C | Rest]) when ?IS_HEX(C) ->
+    bytes_token({tk, Row, Col}, {tk, Row + 1, Col}, [C | Rest], "#", []);
+next_token({tk, Row, Col}, "\"" ++ Rest) ->
+    string_token({tk, Row, Col}, {tk, Row + 1, Col}, Rest, "\"", <<>>);
 next_token({tk, Row, Col}, [Char | Rest]) ->
-    Token = {character, [Char], Row, Col, Col},
+    Token = {character, [Char], Char, Row, Col, Col},
     {ok, {Token, {tk, Row + 1, Col}, Rest}}.
 
-num_token(Start, {tk, Row, Col}, [N | Rest], Acc) when N >= $0, N =< $9 ->
+alphanum_token(Start, {tk, Row, Col}, [C | Rest], Acc) when ?IS_ALPHANUM(C) ->
-    num_token(Start, {tk, Row + 1, Col}, Rest, [N | Acc]);
-num_token({tk, _, Start}, {tk, Row, End}, String, Acc) ->
-    NumString = lists:reverse(Acc),
-    Token = {integer, NumString, Row, Start, End},
-    {ok, {Token, {tk, Row, End}, String}}.
-
-alphanum_token(Start, {tk, Row, Col}, [C | Rest], Acc) when C >= $A, C =< $Z ->
     alphanum_token(Start, {tk, Row, Col}, Rest, [C | Acc]);
-alphanum_token(Start, {tk, Row, Col}, [C | Rest], Acc) when C >= $a, C =< $z ->
-    alphanum_token(Start, {tk, Row, Col}, Rest, [C | Acc]);
-alphanum_token(Start, {tk, Row, Col}, [C | Rest], Acc) when C >= $0, C =< $9 ->
-    alphanum_token(Start, {tk, Row, Col}, Rest, [C | Acc]);
-alphanum_token(Start, {tk, Row, Col}, [$_ | Rest], Acc) ->
-    alphanum_token(Start, {tk, Row, Col}, Rest, [$_ | Acc]);
 alphanum_token({tk, _, Start}, {tk, Row, End}, String, Acc) ->
     AlphaString = lists:reverse(Acc),
-    Token = {alphanum, AlphaString, Row, Start, End},
+    Token = {alphanum, AlphaString, AlphaString, Row, Start, End},
     {ok, {Token, {tk, Row, End}, String}}.
 
+num_token(Start, {tk, Row, Col}, [C | Rest], Chars, Value) when ?IS_NUM(C) ->
+    NewValue = Value * 10 + (C - $0),
+    num_token(Start, {tk, Row + 1, Col}, Rest, [C | Chars], NewValue);
+num_token(Start, {tk, Row, Col}, [$_, C | Rest], Chars, Value) when ?IS_NUM(C) ->
+    NewValue = Value * 10 + (C - $0),
+    num_token(Start, {tk, Row + 2, Col}, Rest, [C, $_ | Chars], NewValue);
+num_token({tk, _, Start}, {tk, Row, End}, String, Chars, Value) ->
+    NumString = lists:reverse(Chars),
+    Token = {integer, NumString, Value, Row, Start, End},
+    {ok, {Token, {tk, Row, End}, String}}.
+
+bytes_token(Start, {tk, Row, Col}, [C | Rest], Chars, Digits) when ?IS_HEX(C) ->
+    Digit = convert_digit(C),
+    bytes_token(Start, {tk, Row + 1, Col}, Rest, [C | Chars], [Digit | Digits]);
+bytes_token(Start, {tk, Row, Col}, [$_, C | Rest], Chars, Digits) when ?IS_HEX(C) ->
+    Digit = convert_digit(C),
+    bytes_token(Start, {tk, Row + 1, Col}, Rest, [C, $_ | Chars], [Digit | Digits]);
+bytes_token({tk, _, Start}, {tk, Row, End}, String, Chars, Digits) ->
+    BytesString = lists:reverse(Chars),
+    Value = reverse_combine_nibbles(Digits, <<>>),
+    Token = {bytes, BytesString, Value, Row, Start, End},
+    {ok, {Token, {tk, Row, End}, String}}.
+
+convert_digit(C) when C >= $0, C =< $9 ->
+    C - $0;
+convert_digit(C) when C >= $A, C =< $Z ->
+    C - $A + 10;
+convert_digit(C) when C >= $a, C =< $z ->
+    C - $a + 10.
+
+reverse_combine_nibbles([D1, D2 | Rest], Acc) ->
+    NewAcc = <<D2:4, D1:4, Acc/binary>>,
+    reverse_combine_nibbles(Rest, NewAcc);
+reverse_combine_nibbles([D1], Acc) ->
+    <<0:4, D1:4, Acc/binary>>;
+reverse_combine_nibbles([], Acc) ->
+    Acc.
+
+string_token(Start, {tk, Row, Col}, [$\\, $x, A, B | Rest], SourceChars, Value) ->
+    case escape_hex_code(A, B) of
+        {ok, ByteVal} ->
+            string_token(Start, {tk, Row + 4, Col}, Rest, [B, A, $x, $\ | SourceChars], <<Value/binary, ByteVal>>);
+        error ->
+            {error, {invalid_escape_code, [$\\, $x, A, B], Row, Col}}
+    end;
+string_token(Start, {tk, Row, Col}, [$\\, C | Rest], SourceChars, Value) ->
+    case escape_char(C) of
+        {ok, ByteVal} ->
+            string_token(Start, {tk, Row + 2, Col}, Rest, [C, $\ | SourceChars], <<Value/binary, ByteVal>>);
+        error ->
+            {error, {invalid_escape_code, [C], Row, Col}}
+    end;
+string_token({tk, _, Start}, {tk, Row, End}, [$" | Rest], SourceChars, Value) ->
+    SourceStr = lists:reverse([$" | SourceChars]),
+    Token = {string, SourceStr, Value, Row, Start, End},
+    {ok, {Token, {tk, Row, End}, Rest}};
+string_token(Start, {tk, Row, Col}, [C | Rest], SourceChars, Value) ->
+    string_token(Start, {tk, Row + 1, Col}, Rest, [C | SourceChars], <<Value/binary, C>>).
+
+escape_hex_code(A, B) when ?IS_HEX(A), ?IS_HEX(B) ->
+    % As of writing this, the Sophia compiler will convert this byte from
+    % extended ASCII to unicode... But it really shouldn't. The literal parser
+    % does what the compiler should do.
+    Byte = convert_digit(A) * 16 + convert_digit(B),
+    {ok, Byte};
+escape_hex_code(_, _) ->
+    error.
+
+escape_char($b)  -> {ok, $\b};
+escape_char($e)  -> {ok, $\e};
+escape_char($f)  -> {ok, $\f};
+escape_char($n)  -> {ok, $\n};
+escape_char($r)  -> {ok, $\r};
+escape_char($t)  -> {ok, $\t};
+escape_char($v)  -> {ok, $\v};
+escape_char($")  -> {ok, $\"};
+escape_char($\\) -> {ok, $\\};
+escape_char(_)   -> error.
 
 %%% Sophia Literal Parser
 
@@ -85,11 +163,14 @@ alphanum_token({tk, _, Start}, {tk, Row, End}, String, Acc) ->
 %%% pushdown automaton that we want.
 
 parse_expression(Type, Tk, String) ->
-    {ok, {Token, NewTk, NewString}} = next_token(Tk, String),
+    case next_token(Tk, String) of
-    parse_expression2(Type, NewTk, NewString, Token).
+        {ok, {Token, NewTk, NewString}} ->
+            parse_expression2(Type, NewTk, NewString, Token);
+        {error, Reason} ->
+            {error, Reason}
+    end.
 
-parse_expression2(Type, Tk, String, {integer, S, Row, Start, End}) ->
+parse_expression2(Type, Tk, String, {integer, _, Value, Row, Start, End}) ->
-    Value = list_to_integer(S),
     case Type of
         {_, _, integer} ->
             {ok, {Value, Tk, String}};
@@ -98,15 +179,39 @@ parse_expression2(Type, Tk, String, {integer, S, Row, Start, End}) ->
         {O, N, _} ->
             {error, {wrong_type, O, N, integer, Row, Start, End}}
     end;
-parse_expression2(Type, Tk, String, {character, "[", Row, Start, _}) ->
+parse_expression2(Type, Tk, String, {bytes, _, Value, Row, Start, End}) ->
+    Len = byte_size(Value),
+    Result = {bytes, Value},
+    case Type of
+        {_, _, {bytes, [any]}} ->
+            {ok, {Result, Tk, String}};
+        {_, _, {bytes, [Len]}} ->
+            {ok, {Result, Tk, String}};
+        {_, _, {bytes, [ExpectedLen]}} ->
+            {error, {bytes_wrong_size, ExpectedLen, Len, Row, Start, End}};
+        {_, _, unknown_type} ->
+            {ok, {Result, Tk, String}};
+        {O, N, _} ->
+            {error, {wrong_type, O, N, {bytes, [Len]}, Row, Start, End}}
+    end;
+parse_expression2(Type, Tk, String, {string, _, Value, Row, Start, End}) ->
+    case Type of
+        {_, _, string} ->
+            {ok, {Value, Tk, String}};
+        {_, _, unknown_type} ->
+            {ok, {Value, Tk, String}};
+        {O, N, _} ->
+            {error, {wrong_type, O, N, string, Row, Start, End}}
+    end;
+parse_expression2(Type, Tk, String, {character, "[", _, Row, Start, _}) ->
     parse_list(Type, Tk, String, Row, Start);
-parse_expression2(Type, Tk, String, {character, "(", Row, Start, _}) ->
+parse_expression2(Type, Tk, String, {character, "(", _, Row, Start, _}) ->
     parse_tuple(Type, Tk, String, Row, Start);
-parse_expression2(Type, Tk, String, {character, "{", Row, Start, _}) ->
+parse_expression2(Type, Tk, String, {character, "{", _, Row, Start, _}) ->
     parse_record_or_map(Type, Tk, String, Row, Start);
-parse_expression2(Type, Tk, String, {alphanum, Ident, Row, Start, End}) ->
+parse_expression2(Type, Tk, String, {alphanum, Ident, _, Row, Start, End}) ->
     parse_variant(Type, Tk, String, Ident, Row, Start, End);
-parse_expression2(_, _, _, {_, S, Row, Start, End}) ->
+parse_expression2(_, _, _, {_, S, _, Row, Start, End}) ->
     {error, {unexpected_token, S, Row, Start, End}}.
 
 unknown_type() ->
@@ -116,9 +221,9 @@ expect_tokens([], Tk, String) ->
     {ok, {Tk, String}};
 expect_tokens([Str | Rest], Tk, String) ->
     case next_token(Tk, String) of
-        {ok, {{_, Str, _, _, _}, NewTk, NewString}} ->
+        {ok, {{_, Str, _, _, _, _}, NewTk, NewString}} ->
             expect_tokens(Rest, NewTk, NewString);
-        {ok, {{_, Actual, Row, Start, End}}} ->
+        {ok, {{_, Actual, _, Row, Start, End}}} ->
             {error, {unexpected_token, Actual, Row, Start, End}}
     end.
 
@@ -133,7 +238,7 @@ parse_list({O, N, _}, _, _, Row, Start) ->
 
 parse_list_loop(Inner, Tk, String, CloseChar, Row, Start, Acc) ->
     case next_token(Tk, String) of
-        {ok, {{character, CloseChar, _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, CloseChar, _, _, _, _}, NewTk, NewString}} ->
             {ok, {lists:reverse(Acc), NewTk, NewString}};
         {ok, {Token, NewTk, NewString}} ->
             parse_list_loop2(Inner, NewTk, NewString, CloseChar, Row, Start, Acc, Token)
@@ -152,9 +257,9 @@ parse_list_loop2(Inner, Tk, String, CloseChar, Row, Start, Acc, Token) ->
 
 parse_list_loop3(Inner, Tk, String, CloseChar, Row, Start, Acc) ->
     case next_token(Tk, String) of
-        {ok, {{character, CloseChar, _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, CloseChar, _, _, _, _}, NewTk, NewString}} ->
             {ok, {lists:reverse(Acc), NewTk, NewString}};
-        {ok, {{character, ",", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, ",", _, _, _, _}, NewTk, NewString}} ->
             parse_list_loop(Inner, NewTk, NewString, CloseChar, Row, Start, Acc);
         {error, Reason} ->
             {error, Reason}
@@ -207,14 +312,14 @@ parse_multivalue2([Next | Rest], Tk, String, Row, Start, Acc, Token) ->
     end;
 parse_multivalue2([], Tk, String, _, _, Acc, {character, ")", _, _, _}) ->
     {ok, {lists:reverse(Acc), Tk, String}};
-parse_multivalue2([], _, _, _, _, _, {_, S, Row, Start, End}) ->
+parse_multivalue2([], _, _, _, _, _, {_, S, _, Row, Start, End}) ->
     {error, {unexpected_token, S, Row, Start, End}}.
 
 parse_multivalue3(ElemTypes, Tk, String, Row, Start, Acc) ->
     case next_token(Tk, String) of
-        {ok, {{character, ")", Row2, Start2, _}, NewTk, NewString}} ->
+        {ok, {{character, ")", _, Row2, Start2, _}, NewTk, NewString}} ->
             check_multivalue_long_enough(ElemTypes, NewTk, NewString, Row2, Start2, Acc);
-        {ok, {{character, ",", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, ",", _, _, _, _}, NewTk, NewString}} ->
             parse_multivalue(ElemTypes, NewTk, NewString, Row, Start, Acc);
         {error, Reason} ->
             {error, Reason}
@@ -257,9 +362,9 @@ parse_variant3(Arities, Tag, [], Tk, String) ->
     {ok, {Result, Tk, String}};
 parse_variant3(Arities, Tag, ElemTypes, Tk, String) ->
     case next_token(Tk, String) of
-        {ok, {{character, "(", Row, Start, _}, NewTk, NewString}} ->
+        {ok, {{character, "(", _, Row, Start, _}, NewTk, NewString}} ->
             parse_variant4(Arities, Tag, ElemTypes, NewTk, NewString, Row, Start);
-        {ok, {{_, Actual, Row, Start, End}}} ->
+        {ok, {{_, Actual, _, Row, Start, End}}} ->
             {error, {unexpected_token, Actual, Row, Start, End}}
     end.
 
@@ -287,13 +392,13 @@ parse_record_or_map({_, _, {record, Fields}}, Tk, String, _, _) ->
     parse_record(Fields, Tk, String, #{});
 parse_record_or_map({_, _, unknown_type}, Tk, String, _, _) ->
     case next_token(Tk, String) of
-        {ok, {{character, "}", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, "}", _, _, _, _}, NewTk, NewString}} ->
             {ok, {#{}, NewTk, NewString}};
-        {ok, {{character, "[", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, "[", _, _, _, _}, NewTk, NewString}} ->
             parse_map2(unknown_type(), unknown_type(), NewTk, NewString, #{});
-        {ok, {{alphanum, _, Row, Start, End}, _, _}} ->
+        {ok, {{alphanum, _, _, Row, Start, End}, _, _}} ->
             {error, {unresolved_record, Row, Start, End}};
-        {ok, {{_, S, Row, Start, End}, _, _}} ->
+        {ok, {{_, S, _, Row, Start, End}, _, _}} ->
             {error, {unexpected_token, S, Row, Start, End}}
     end;
 parse_record_or_map({O, N, _}, _, _, Row, Start) ->
@@ -301,11 +406,11 @@ parse_record_or_map({O, N, _}, _, _, Row, Start) ->
 
 parse_record(Fields, Tk, String, Acc) ->
     case next_token(Tk, String) of
-        {ok, {{alphanum, Ident, Row, Start, End}, NewTk, NewString}} ->
+        {ok, {{alphanum, Ident, _, Row, Start, End}, NewTk, NewString}} ->
             parse_record2(Fields, NewTk, NewString, Acc, Ident, Row, Start, End);
-        {ok, {{character, "}", Row, Start, End}, NewTk, NewString}} ->
+        {ok, {{character, "}", _, Row, Start, End}, NewTk, NewString}} ->
             parse_record_end(Fields, NewTk, NewString, Acc, Row, Start, End);
-        {ok, {{_, S, Row, Start, End}, _, _}} ->
+        {ok, {{_, S, _, Row, Start, End}, _, _}} ->
             {error, {unexpected_token, S, Row, Start, End}};
         {error, Reason} ->
             {error, Reason}
@@ -346,11 +451,11 @@ parse_record5(Fields, Tk, String, Acc, Ident, Type) ->
 
 parse_record6(Fields, Tk, String, Acc) ->
     case next_token(Tk, String) of
-        {ok, {{character, ",", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, ",", _, _, _, _}, NewTk, NewString}} ->
             parse_record(Fields, NewTk, NewString, Acc);
-        {ok, {{character, "}", Row, Start, End}, NewTk, NewString}} ->
+        {ok, {{character, "}", _, Row, Start, End}, NewTk, NewString}} ->
             parse_record_end(Fields, NewTk, NewString, Acc, Row, Start, End);
-        {ok, {{_, S, Row, Start, End}, _, _}} ->
+        {ok, {{_, S, _, Row, Start, End}, _, _}} ->
             {error, {unexpected_token, S, Row, Start, End}};
         {error, Reason} ->
             {error, Reason}
@@ -381,11 +486,11 @@ parse_record_final_loop([], _, FieldsReverse) ->
 
 parse_map(KeyType, ValueType, Tk, String, Acc) ->
     case next_token(Tk, String) of
-        {ok, {{character, "[", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, "[", _, _, _, _}, NewTk, NewString}} ->
             parse_map2(KeyType, ValueType, NewTk, NewString, Acc);
-        {ok, {{character, "}", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, "}", _, _, _, _}, NewTk, NewString}} ->
             {ok, {Acc, NewTk, NewString}};
-        {ok, {{_, S, Row, Start, End}}} ->
+        {ok, {{_, S, _, Row, Start, End}}} ->
             {error, {unexpected_token, S, Row, Start, End}}
     end.
 
@@ -416,11 +521,11 @@ parse_map4(KeyType, ValueType, Tk, String, Acc, Key) ->
 
 parse_map5(KeyType, ValueType, Tk, String, Acc) ->
     case next_token(Tk, String) of
-        {ok, {{character, ",", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, ",", _, _, _, _}, NewTk, NewString}} ->
             parse_map(KeyType, ValueType, NewTk, NewString, Acc);
-        {ok, {{character, "}", _, _, _}, NewTk, NewString}} ->
+        {ok, {{character, "}", _, _, _, _}, NewTk, NewString}} ->
             {ok, {Acc, NewTk, NewString}};
-        {ok, {{_, S, Row, Start, End}}} ->
+        {ok, {{_, S, _, Row, Start, End}}} ->
             {error, {unexpected_token, S, Row, Start, End}}
     end.
 
@@ -430,12 +535,13 @@ wrap_error(Reason, _) -> Reason.
 %%% Tests
 
 check_sophia_to_fate(Type, Sophia, Fate) ->
-    {ok, FateActual} = parse_literal(Type, Sophia),
+    case parse_literal(Type, Sophia) of
-    case FateActual of
+        {ok, Fate} ->
-        Fate ->
             ok;
-        _ ->
+        {ok, FateActual} ->
-            erlang:error({to_fate_failed, Fate, FateActual})
+            erlang:error({to_fate_failed, Sophia, Fate, {ok, FateActual}});
+        {error, Reason} ->
+            erlang:error({to_fate_failed, Sophia, Fate, {error, Reason}})
     end.
 
 compile_entrypoint_code_and_type(Source, Entrypoint) ->
@@ -487,20 +593,27 @@ check_parser_with_typedef(Typedef, Sophia) ->
     % definitions in the AACI, so untyped parses probably don't work.
     check_sophia_to_fate(Type, Sophia, Fate).
 
-int_test() ->
+anon_types_test() ->
-    check_parser("123").
+    % Integers.
+    check_parser("123"),
+    check_parser("1_2_3"),
+    % Bytes.
+    check_parser("#DEAD000BEEF"),
+    check_parser("#DE_AD0_00B_EEF"),
+    % Strings.
+    check_parser("\"hello world\""),
+    check_parser("\"  \\b\\e\\f\\n\\r\\t\\v\\\"\\\\  \""),
+    check_parser("\"\\x00\\x11\\x77\""),
+    % List of integers.
+    check_parser("[1, 2, 3]"),
+    % List of lists.
+    check_parser("[[], [1], [2, 3]]"),
+    % Tuple.
+    check_parser("(1, [2, 3], (4, 5))"),
+    % Map.
+    check_parser("{[1] = 2, [3] = 4}"),
 
-list_test() ->
+    ok.
-    check_parser("[1, 2, 3]").
-
-list_of_lists_test() ->
-    check_parser("[[], [1], [2, 3]]").
-
-tuple_test() ->
-    check_parser("(1, [2, 3], (4, 5))").
-
-maps_test() ->
-    check_parser("{[1] = 2, [3] = 4}").
 
 records_test() ->
     TypeDef = "record pair = {x: int, y: int}",