Merge pull request #26 from aeternity/PT-165180296-fix-fate-scanner

Pt 165180296 fix fate scanner

.gitignore

@@ -20,3 +20,5 @@ src/aeb_fate_pp.erl
 *.erl~
 *.hrl~
 *.aes~
+doc
+cover

quickcheck/aefate_eqc.erl

@@ -0,0 +1,107 @@
+%%% @author Thomas Arts
+%%% @copyright (C) 2018, Thomas Arts
+%%% @doc Use `rebar3 as eqc shell` to run properties in the shell
+%%%
+%%%
+%%% @end
+%%% Created : 13 Dec 2018 by Thomas Arts <thomas@SpaceGrey.lan>
+
+-module(aefate_eqc).
+
+-include_lib("eqc/include/eqc.hrl").
+
+-compile([export_all, nowarn_export_all]).
+
+prop_roundtrip() ->
+    ?FORALL(FateData, fate_type(),
+            measure(bytes, size(term_to_binary(FateData)),
+                    begin
+                        Serialized = aeb_fate_encoding:serialize(FateData),
+                        ?WHENFAIL(eqc:format("Serialized ~p to ~p~n", [FateData, Serialized]),
+                                  equals(aeb_fate_encoding:deserialize(Serialized), FateData))
+                    end
+                   )).
+
+prop_format() ->
+    ?FORALL(FateData, fate_type(),
+            ?WHENFAIL(eqc:format("Trying to format ~p failed~n",[FateData]),
+                      begin
+                          String = aeb_fate_data:format(FateData),
+                          collect([FateData, unicode:characters_to_binary(String, latin1)], true)
+                      end)).
+
+prop_format_scan() ->
+    ?FORALL(FateData, fate_type(),
+            ?WHENFAIL(eqc:format("Trying to format ~p failed~n~p~n", [FateData, catch unicode:characters_to_list(aeb_fate_data:format(FateData), utf8) ]),
+                      begin
+                          String = aeb_fate_data:format(FateData),
+                          {ok, _Scanned, _} = aeb_fate_asm_scan:scan(unicode:characters_to_list(String)),
+                          true
+                      end)).
+
+fate_type() ->
+    ?SIZED(Size, fate_type(Size, [map])).
+
+fate_type(0, _Options) ->
+    ?LAZY(
+       oneof([fate_integer(),
+              fate_boolean(),
+              fate_nil(),
+              fate_unit(),
+              fate_string(),
+              fate_address(),
+              fate_hash(),
+              fate_signature(),
+              fate_contract(),
+              fate_oracle(),
+              fate_name(),
+              fate_bits(),
+              fate_channel()]));
+fate_type(Size, Options) ->
+    ?LETSHRINK([Smaller], [?LAZY(fate_type(Size div 5, Options))],
+    oneof([?LAZY(fate_type(Size - 1, Options)),
+           ?LAZY(fate_list( Smaller )),
+           ?LAZY(fate_tuple( list( Smaller ))),
+           ?LAZY(fate_variant(?LET(L, list( Smaller), list_to_tuple(L))))] ++
+              [
+               ?LAZY(fate_map( fate_type(Size div 3, Options -- [map]),
+                               Smaller))
+              || lists:member(map, Options)
+              ])).
+
+
+fate_integer()    -> oneof([int(), largeint()]).
+fate_bits()       -> {bits, oneof([int(), largeint()])}.
+fate_boolean()    -> elements([true, false]).
+fate_nil()        -> [].
+fate_unit()       -> {tuple, {}}.
+fate_string()     -> ?SUCHTHAT(S, utf8(), string:find(S, "\"") == nomatch).
+fate_address()    -> {address, non_zero_binary(256 div 8)}.
+fate_hash()       -> {hash, non_zero_binary(32)}.
+fate_signature()  -> {signature, non_zero_binary(64)}.
+fate_contract()   -> {contract, non_zero_binary(256 div 8)}.
+fate_oracle()     -> {oracle, non_zero_binary(256 div 8)}.
+fate_name()       -> {name, non_zero_binary(256 div 8)}.
+fate_channel()    -> {channel, non_zero_binary(256 div 8)}.
+
+%% May shrink to fate_unit
+fate_tuple(ListGen) ->
+    {tuple, ?LET(Elements, ListGen, list_to_tuple(Elements))}.
+
+fate_variant(TupleGen) ->
+    ?LET({L1, L2, Tuple}, {list(choose(0, 255)), list(choose(0,255)), TupleGen},
+         {variant, L1 ++ [size(Tuple)] ++ L2, length(L1), Tuple}).
+
+fate_list(Gen) ->
+    oneof([fate_nil(), ?SHRINK(list(Gen), [fate_nil()])]).
+
+fate_map(KeyGen, ValGen) ->
+    map(KeyGen, ValGen).
+
+
+non_zero_binary(N) ->
+    Bits = N*8,
+    ?SUCHTHAT(Bin, binary(N), begin <<V:Bits>> = Bin, V =/= 0 end).
+
+char() ->
+    choose(1, 255).

src/aeb_fate_asm_scan.template

@@ -13,12 +13,12 @@ HEXDIGIT = [0-9a-fA-F]
 LOWER    = [a-z_]
 UPPER    = [A-Z]
 BASE58   = [123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz]
-BASE64   = [ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxy0123456789+/=]
+BASE64   = [A-Za-z0-9+/=]
 INT      = {DIGIT}+
 HEX      = 0x{HEXDIGIT}+
 OBJECT   = @[a-z][a-z]_{BASE58}+
 HASH     = #{BASE64}+
-SIG      = \$\sg_{BASE58}+
+SIG      = \$sg_{BASE58}+
 WS       = [\000-\s]
 ID       = {LOWER}[a-zA-Z0-9_]*
 STRING   = "[^"]*"
@@ -51,8 +51,11 @@ FUNCTION       : {token, {function, TokenLine, 'FUNCTION' }}.
  {token, {int, TokenLine, parse_int(TokenChars)}}.
 -{INT} :
  {token, {int, TokenLine, parse_int(TokenChars)}}.
+
+%% Due to the definition of STRING the tokens start and end with a quote ".
 {STRING} :
- {token, {string, TokenLine, list_to_binary(TokenChars)}}.
+ {token, {string, TokenLine, unicode:characters_to_binary(
+                lists:sublist(TokenChars, 2, length(TokenChars) - 2))}}.
 {BITS} :
  {token, {bits, TokenLine, bits(TokenChars)}}.
 
@@ -112,7 +115,7 @@ parse_hash("#" ++ Chars) ->
     base64:decode(Chars).
 
 parse_object([_|Chars]) ->
-    case aeser_api_encoder:decode(list_to_binary(Chars)) of
+    case aeser_api_encoder:decode(unicode:characters_to_binary(Chars)) of
        {account_pubkey, Bin}  -> {address, Bin};
        {contract_pubkey, Bin} -> {contract, Bin};
        {oracle_pubkey, Bin}   -> {oracle, Bin};

src/aeb_fate_data.erl

@@ -20,6 +20,7 @@
 -type fate_signature() :: ?FATE_SIGNATURE_T.
 -type fate_variant()   :: ?FATE_VARIANT_T.
 -type fate_tuple()     :: ?FATE_TUPLE_T.
+-type fate_bits()      :: ?FATE_BITS_T.
 
 -type fate_type_type() :: integer
                         | boolean
@@ -34,7 +35,7 @@
                         | name
                         | channel
                         | bits
-                        | {variant, list(), integer()}.
+                        | {variant, list()}.
 
 
 -type fate_type() ::
@@ -54,7 +55,7 @@
       | fate_channel()
       | fate_variant()
       | fate_map()
-      | fate_type_type().
+      | fate_bits().
 
 -export_type([fate_type/0
              , fate_boolean/0
@@ -73,6 +74,7 @@
              , fate_channel/0
              , fate_variant/0
              , fate_map/0
+             , fate_bits/0
              , fate_type_type/0
              ]).
 
@@ -120,6 +122,9 @@ make_string(S)  when is_list(S) ->
     ?FATE_STRING(list_to_binary(lists:flatten(S)));
 make_string(S)  when is_binary(S) -> ?FATE_STRING(S).
 
+%% Tag points to the selected variant (zero based)
+%% The arity of this variant is read from the list of provided arities
+%% and should match the size of the given tuple.
 make_variant(Arities, Tag, Values) ->
     Arities = [A || A <- Arities, is_integer(A), A < 256],
     Size = length(Arities),
@@ -205,7 +210,7 @@ format(L) when ?IS_FATE_LIST(L) -> format_list(?FATE_LIST_VALUE(L));
 format(?FATE_UNIT) -> "()";
 format(?FATE_TUPLE(T)) ->
     ["( ", lists:join(", ", [ format(E) || E <- erlang:tuple_to_list(T)]), " )"];
-format(S) when ?IS_FATE_STRING(S) -> [S];
+format(S) when ?IS_FATE_STRING(S) -> ["\"", S, "\""];
 format(?FATE_BITS(B)) when B >= 0 ->
     ["<", format_bits(B, "") , ">"];
 format(?FATE_BITS(B)) when B < 0 ->
@@ -213,7 +218,7 @@ format(?FATE_BITS(B)) when B < 0 ->
 format(?FATE_VARIANT(Arities, Tag, T)) ->
     ["(| ",
       lists:join("| ",
-                 [io_lib:format("~p", [Arities]),
+                 [format_arities(Arities),
                   integer_to_list(Tag) |
                   [format(make_tuple(T))]]),
      " |)"];
@@ -244,6 +249,9 @@ format_nbits(N, Acc) ->
     Bit = $1 - (N band 1),
     format_nbits(N bsr 1, [Bit|Acc]).
 
+format_arities(Arities) ->
+    ["[ ", lists:join(", ", [integer_to_list(E) || E <- Arities]), " ]"].
+
 format_list(List) ->
     ["[ ", lists:join(", ", [format(E) || E <- List]), " ]"].
 

src/aebytecode.app.src

@@ -1,6 +1,6 @@
 {application, aebytecode,
  [{description, "Bytecode definitions, serialization and deserialization for aeternity."},
-  {vsn, "2.0.1"},
+  {vsn, "2.1.0"},
   {registered, []},
   {applications,
    [kernel,