Map 'hash' as known api type, add some more tests

Reviewed-on: #55

README.md

@@ -2,6 +2,8 @@
 
 Serialization helpers for the Gajumaru.
 
+For an overview of the static serializer, see [this document](doc/static.md).
+
 ## Build
 
     $ rebar3 compile
@@ -30,6 +32,7 @@ how the type information is represented. The fully serialized form is
 produced by the `serialize` functions.
 
 The basic types supported by the encoder are:
+* `integer()`         (`anyint`, code: 246)
 * `neg_integer()`     (`negint`, code: 247)
 * `non_neg_integer()` (`int`   , code: 248)
 * `binary()`          (`binary`, code: 249)
@@ -40,11 +43,31 @@ The basic types supported by the encoder are:
 * `gmser_id:id()`     (`id`    , code: 254)
 * `atom()`            (`label` , code: 255)
 
+(The range of codes is chosen because the `gmser_chain_objects` codes
+range from 10 to 200, and also to stay within 1 byte.)
+
 When encoding `map` types, the map elements are first sorted.
 
 When specifying a map type for template-driven encoding, use 
-the `#{items => [{Key, Value}]}` construct.
+the `#{items => [{Key, ValueType} | {opt, Key, ValueType}]}` construct.
+The key names are included in the encoding, and are match against the item
+specs during decoding. If the key names don't match, the decoding fails, unless
+for an `{opt, K, V}` item, in which case that item spec is skipped.
 
+```erlang
+T = #{items => [{a,int},{opt,b,int},{c,int}]}
+E1 = gmser_dyn:encode_typed(T, #{a => 1, b => 2, c => 3}) ->
+    [<<0>>,<<1>>,[<<252>>,
+     [[[<<255>>,<<97>>],[<<248>>,<<1>>]],
+     [[<<255>>,<<98>>],[<<248>>,<<2>>]],
+     [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]]
+E2 = gmser_dyn:encode_typed(T, #{a => 1, c => 3}) ->
+    [<<0>>,<<1>>,[<<252>>,
+     [[[<<255>>,<<97>>],[<<248>>,<<1>>]],
+     [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]]
+gmser_dyn:decode_typed(T,E2) ->
+    #{c => 3,a => 1}
+```
 
 ## Labels
 
@@ -52,6 +75,24 @@ Labels correspond to (existing) atoms in Erlang.
 Decoding of a label results in a call to `binary_to_existing_atom/2`, so will
 fail if the corresponding atom does not already exist.
 
+This behavior can be modified using the option `#{missing_labels => fail | create | convert}`,
+where `fail` is the default, as described above, `convert` means that missing atoms are
+converted to binaries, and `create` means that the atom is created dynamically.
+
+The option can be passed e.g.:
+```erlang
+gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert}))
+```
+
+or
+```erlang
+gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert}, Types))
+```
+
+By calling `gmser_dyn:register_types/1`, after having added options to the type map,
+the options can be made to take effect automatically.
+
+
 It's possible to cache labels for more compact encoding.
 Note that when caching labels, the same cache mapping needs to be used on the
 decoder side.
@@ -90,31 +131,70 @@ Templates can be provided to the encoder by either naming an already registered
 type, or by passing a template directly. In both cases, the encoder will enforce
 the type information in the template.
 
-If the template has been registered, the encoder omits inner type tags (still
-inserting the top-level tag), leading to some compression of the output.
-This also means that the serialized term cannot be decoded without the same
-schema information on the decoder side.
-
-In the case of a directly provided template, all type information is inserted,
-such that the serialized term can be decoded without any added type information.
-The template types are still enforced during encoding.
+If the template has been registered, the encoder uses the registered type specification
+to drive the encoding. The code of the registered template is embedded in the encoded
+output:
 
 ```erlang
-ET = fun(Type,Term) -> io:fwrite("~w~n", [gmser_dyn:encode_typed(Type,Term)]) end.
+gmser_dyn:encode_typed({int,int,int}, {1,2,3}) ->
+    [<<0>>,<<1>>,[<<253>>,
+     [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]]
 
-ET([{int,int}], [{1,2}]) -> [<<0>>,<<1>>,[<<251>>,[[[<<248>>,<<1>>],[<<248>>,<<2>>]]]]]
-
-gmser_dyn:register_type(1000,lt2i,[{int,int}]).
-ET(lt2i, [{1,2}]) -> [<<0>>,<<1>>,[<<3,232>>,[[<<1>>,<<2>>]]]]
+Types = gmser_dyn_types:add_type(t3,1013,{int,int,int}).
+gmser_dyn:encode_typed(t3, {1,2,3}, Types) ->
+    [<<0>>,<<1>>,[[<<3,245>>,<<253>>],
+     [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]]
 ```
 
+Note that the original `<<253>>` type code is wrapped as `[<<3,245>>,<<253>>]`,
+where `<<3,245>>` corresponds to the custom code `1013`.
+
+Using the default option `#{strict => true}`, the decoder will extract the custom
+type spec, and validate the encoded data against it. If the custom code is missing,
+the decoder aborts. Using `#{strict => false}`, the custom code is used if it exists,
+but otherwise, it's ignored, and the encoded data is decoded using the dynamic type
+info.
+
 ### Alternative types
 
-The dynamic encoder supports two additions to the `gmserialization` template
-language: `any` and `#{alt => [AltTypes]}`.
+The dynamic encoder supports a few additions to the `gmserialization` template
+language: `any`, `#{list => Type}`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`.
+
+#### `any`
 
 The `any` type doesn't have an associated code, but enforces dynamic encoding.
 
+#### `list`
+
+The original list type notation expects a key-value list, e.g.
+
+`[{name, binary}, {age, int}]`
+
+```erlang
+EL = gmser_dyn:encode_typed([{name,binary},{age,int}], [{name,<<"Ulf">>},{age,29}]) ->
+    [<<0>>,<<1>>,[<<251>>,
+     [[<<253>>,[[<<255>>,<<110,97,109,101>>],[<<249>>,<<85,108,102>>]]],
+      [<<253>>,[[<<255>>,<<97,103,101>>],[<<248>>,<<29>>]]]]]]
+```
+Note that the encoding explicitly lays out a `[{Key, Value}]` structure, all
+dynamically typed. This means it can be dynamically decoded without templates.
+
+```erlang
+gmser_dyn:decode(EL).
+[{name,<<"Ulf">>},{age,29}]
+```
+
+In order to specify something like Erlang's `[integer()]` type, we can use
+the following:
+
+```erlang
+gmser_dyn:encode_typed(#{list => int}, [1,2,3,4]) ->
+    [<<0>>,<<1>>,[<<251>>,
+     [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>],[<<248>>,<<4>>]]]]
+```
+
+#### `alt`
+
 The `#{alt => [Type]}` construct also enforces dynamic encoding, and will try
 to encode as each type in the list, in the specified order, until one matches.
 
@@ -122,8 +202,66 @@ to encode as each type in the list, in the specified order, until one matches.
 gmser_dyn:encode_typed(#{alt => [negint,int]}, 5) -> [<<0>>,<<1>>,[<<247>>,<<5>>]]
 gmser_dyn:encode_typed(#{alt => [negint,int]}, 5) -> [<<0>>,<<1>>,[<<248>>,<<5>>]]
 
-gmser_dyn:register_type(246, anyint, #{alt => [negint, int]})
-
 gmser_dyn:encode_typed(anyint,-5) -> [<<0>>,<<1>>,[<<246>>,[<<247>>,<<5>>]]]
 gmser_dyn:encode_typed(anyint,5)  -> [<<0>>,<<1>>,[<<246>>,[<<248>>,<<5>>]]]
 ```
+
+#### `switch`
+
+The `switch` type allows for encoding a 'tagged' object, where the tag determines
+the type.
+
+```erlang
+E1 = gmser_dyn:encode_typed(#{switch => #{name => binary, age => int}}, #{age => 29}) ->
+    [<<0>>,<<1>>,[<<252>>,[[[<<255>>,<<97,103,101>>],[<<248>>,<<29>>]]]]]
+gmser_dyn:decode_typed(#{switch => #{name => binary, age => int}}, E1) ->
+    #{age => 29}
+E2 = gmser_dyn:encode_typed(#{switch => #{name => binary, age => int}}, #{name => <<"Ulf">>}) ->
+    [<<0>>,<<1>>,[<<252>>,[[[<<255>>,<<110,97,109,101>>],[<<249>>,<<85,108,102>>]]]]]
+gmser_dyn:decode_typed(#{switch => #{name => binary, age => int}}, E1) ->
+    #{name => <<"Ulf">>}
+```
+
+A practical use of `switch` would be in a protocol schema:
+
+```erlang
+t_msg(_) ->
+    #{switch => #{ call         => t_call
+                 , reply        => t_reply
+                 , notification => t_notification }}.
+
+t_call(_) ->
+    #{items => [ {id, anyint}
+               , {req, t_req} ]}.
+
+t_reply(_) ->
+    #{alt => [#{items => [ {id, anyint}
+                         , {result, t_result} ]},
+              #{items => [ {id, anyint}
+                         , {code, anyint}
+                         , {message, binary} ]}
+             ]}.
+```
+
+In this scenario, messages are 'taggged' as 1-element maps, e.g.:
+
+```erlang
+async_request(Msg) ->
+    Id = erlang:unique_integer(),
+    gmmp_cp:to_server(
+      whereis(gmmp_core_connector),
+      #{call => #{ id => Id
+                 , req => Msg }}),
+    Id.
+```
+
+### Notes
+
+Note that `anyint` is a standard type. The static serializer supports only
+positive integers (`int`), as negative numbers are forbidden on-chain.
+For dynamic encoding e.g. in messaging protocols, handling negative numbers can
+be useful, so the `negint` type was added as a dynamic type. To encode a full-range
+integer, the `alt` construct is needed.
+
+(Floats are not supported, as they are non-deterministic. Rationals and fixed-point
+numbers could easily be handled as high-level types, e.g. as `{int,int}`.)

doc/static.md

@@ -0,0 +1,83 @@
+
+# Static Serialization
+
+The `gmserialization` and `gmser_chain_objects` modules implement the
+static serialization support used in the Gajumaru blockchain.
+
+The purpose is to produce fully deterministic serialization, in order
+to maintain predictable hashing.
+
+Example:
+
+```erlang
+%% deterministic canonical serialization.
+-spec serialize_to_binary(signed_tx()) -> binary_signed_tx().
+serialize_to_binary(#signed_tx{tx = Tx, signatures = Sigs}) ->
+    gmser_chain_objects:serialize(
+      ?SIG_TX_TYPE,
+      ?SIG_TX_VSN,
+      serialization_template(?SIG_TX_VSN),
+      [ {signatures, lists:sort(Sigs)}
+      , {transaction, aetx:serialize_to_binary(Tx)}
+      ]).
+
+-spec deserialize_from_binary(binary()) -> signed_tx().
+deserialize_from_binary(SignedTxBin) when is_binary(SignedTxBin) ->
+    [ {signatures, Sigs}
+    , {transaction, TxBin}
+    ] = gmser_chain_objects:deserialize(
+          ?SIG_TX_TYPE,
+          ?SIG_TX_VSN,
+          serialization_template(?SIG_TX_VSN),
+          SignedTxBin),
+    assert_sigs_size(Sigs),
+    #signed_tx{ tx = aetx:deserialize_from_binary(TxBin)
+              , signatures = Sigs
+              }.
+
+serialization_template(?SIG_TX_VSN) ->
+    [ {signatures, [binary]}
+    , {transaction, binary}
+    ].
+```
+
+The terms that can be encoded using these templates are given by
+this type in `gmserialization.erl`:
+
+```erlang
+-type encodable_term() :: non_neg_integer()
+                        | binary()
+                        | boolean()
+                        | [encodable_term()] %% Of any length
+                        | #{atom() => encodable_term()}
+                        | tuple()  %% Any arity, containing encodable_term().
+                        | gmser_id:id().
+```
+
+The template 'language' is defined by these types:
+
+```erlang
+-type template() :: [{field_name(), type()}].
+-type field_name() :: atom().
+-type type() :: 'int'
+              | 'bool'
+              | 'binary'
+              | 'id'     %% As defined in aec_id.erl
+              | [type()] %% Length one in the type. This means a list of any length.
+              | #{items := [{field_name(), type()}]} %% Record with named fields
+                                                     %% represented as a map.
+                                                     %% Encoded as a list in the given
+                                                     %% order.
+              | tuple(). %% Any arity, containing type(). This means a static size array.
+```
+
+The `gmser_chain_objects.erl` module specifies a serialization code for each
+object that can go on-chain. E.g.:
+
+```erlang
+tag(signed_tx) -> 11;
+...
+rev_tag(11) -> signed_tx;
+```
+
+The `tag` and `vsn` are laid out in the beginning of the serialized object.

rebar.config

@@ -7,4 +7,11 @@
   {enacl,
    {git,
     "https://git.qpq.swiss/QPQ-AG/enacl.git", 
-    {ref, "4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}}]}.
+    {ref, "4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}},
+  {eblake2, "1.0.0"}
+ ]}.
+
+{dialyzer,
+  [ {plt_apps, all_deps},
+    {base_plt_apps, [erts, kernel, stdlib, enacl, base58, eblake2]}
+  ]}.

rebar.lock

@@ -1,8 +1,16 @@
+{"1.2.0",
 [{<<"base58">>,
   {git,"https://git.qpq.swiss/QPQ-AG/erl-base58.git",
        {ref,"e6aa62eeae3d4388311401f06e4b939bf4e94b9c"}},
   0},
+ {<<"eblake2">>,{pkg,<<"eblake2">>,<<"1.0.0">>},0},
  {<<"enacl">>,
   {git,"https://git.qpq.swiss/QPQ-AG/enacl.git",
        {ref,"4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}},
-  0}].
+  0}]}.
+[
+{pkg_hash,[
+ {<<"eblake2">>, <<"EC8AD20E438AAB3F2E8D5D118C366A0754219195F8A0F536587440F8F9BCF2EF">>}]},
+{pkg_hash_ext,[
+ {<<"eblake2">>, <<"3C4D300A91845B25D501929A26AC2E6F7157480846FAB2347A4C11AE52E08A99">>}]}
+].

src/gmser_api_encoder.erl

@@ -13,7 +13,8 @@
          safe_decode/2,
          byte_size_for_type/1]).
 
--export_type([encoded/0]).
+-export_type([encoded/0,
+             known_type/0]).
 
 -type known_type() :: key_block_hash
                     | micro_block_hash
@@ -38,6 +39,7 @@
                     | native_token
                     | commitment
                     | peer_pubkey
+                    | hash
                     | state
                     | poi
                     | state_trees

src/gmser_dyn_types.erl

@@ -0,0 +1,71 @@
+-module(gmser_dyn_types).
+
+-export([ add_type/3    %% (Tag, Code, Template) -> Types1
+        , add_type/4    %% (Tag, Code, Template, Types) -> Types1
+        , from_list/2
+        , expand/1 ]).
+-export([ next_code/1 ]).
+
+next_code(#{codes := Codes}) ->
+    lists:max(maps:keys(Codes)) + 1.
+
+-spec add_type(Tag, Code, Template) -> Types
+              when Tag :: gmser_dyn:tag()
+                   , Code :: gmser_dyn:code()
+                   , Template :: gmser_dyn:template()
+                   , Types    :: gmser_dyn:types().
+add_type(Tag, Code, Template) ->
+    add_type(Tag, Code, Template, gmser_dyn:registered_types()).
+
+add_type(Tag, Code, Template, Types) ->
+    elem_to_type({Tag, Code, Template}, Types).
+
+from_list(L, Types) ->
+    lists:foldl(fun elem_to_type/2, Types, L).
+
+expand(#{vsn := V, templates := Templates0} = Types) ->
+    Templates =
+        maps:map(
+          fun(_, F) when is_function(F, 0) ->
+                  F();
+             (_, F) when is_function(F, 1) ->
+                  F(V);
+             (_, T) ->
+                  T
+          end, Templates0),
+    Types#{templates := Templates}.
+
+elem_to_type({Tag, Code, Template}, Acc) when is_atom(Tag), is_integer(Code) ->
+    #{codes := Codes, rev := Rev, templates := Temps} = Acc,
+    case {is_map_key(Tag, Rev), is_map_key(Code, Codes)} of
+        {false, false} ->
+            Acc#{ codes := Codes#{Code => Tag}
+                , rev := Rev#{Tag => Code}
+                , templates => Temps#{Tag => Template}
+                };
+        {true, _} -> error({duplicate_tag, Tag});
+        {_, true} -> error({duplicate_code, Code})
+    end;
+elem_to_type({modify, {Tag, Template}}, Acc) ->
+    #{codes := _, rev := Rev, templates := Templates} = Acc,
+    _ = maps:get(Tag, Rev),
+    Templates1 = Templates#{Tag := Template},
+    Acc#{templates := Templates1};
+elem_to_type({labels, Lbls}, Acc) ->
+    lists:foldl(fun add_label/2, Acc, Lbls);
+elem_to_type({vsn, V}, Acc) ->
+    Acc#{vsn => V};
+elem_to_type(Elem, _) ->
+    error({invalid_type, Elem}).
+
+add_label({L, Code}, #{labels := Lbls, rev_labels := RevLbls} = Acc)
+  when is_atom(L), is_integer(Code), Code > 0 ->
+    case {is_map_key(L, Lbls), is_map_key(Code, RevLbls)} of
+        {false, false} ->
+            Acc#{labels := Lbls#{L => Code},
+                 rev_labels := RevLbls#{Code => L}};
+        {true, _} -> error({duplicate_label, L});
+        {_, true} -> error({duplicate_label_code, Code})
+    end;
+add_label(Elem, _) ->
+    error({invalid_label, Elem}).

src/gmser_id.erl

@@ -17,6 +17,9 @@
         , is_id/1
         ]).
 
+-export([ t_id/1
+        ]).
+
 %% For aec_serialization
 -export([ encode/1
         , decode/1
@@ -26,11 +29,18 @@
             , val
             }).
 
--type tag() :: 'account' | 'name'
-             | 'commitment' | 'contract' | 'channel'
-             | 'associate_chain' | 'entry' .
+-type tag() :: 'account'
+             | 'associate_chain'
+             | 'channel'
+             | 'commitment'
+             | 'contract'
+             | 'contract_source'
+             | 'name'
+             | 'native_token'
+             | 'entry'.
+
 -type val() :: <<_:256>>.
--opaque(id() :: #id{}).
+-type id() :: #id{}.
 
 -export_type([ id/0
              , tag/0
@@ -94,6 +104,9 @@ decode(<<Tag:?TAG_SIZE/unit:8, Val:?PUB_SIZE/binary>>) ->
     #id{ tag = decode_tag(Tag)
        , val = Val}.
 
+-spec t_id(any()) -> id().
+t_id(#id{} = Id) -> Id.
+
 %%%===================================================================
 %%% Internal functions
 %%%===================================================================

test/gmser_api_encoder_tests.erl

@@ -22,10 +22,39 @@
                , {native_token      , 32}
                , {commitment        , 32}
                , {peer_pubkey       , 32}
+               , {hash              , 32}
                , {state             , 32}
                , {poi               , not_applicable}]).
 
 encode_decode_test_() ->
+    encode_decode_test_(?TYPES).
+
+encode_decode_known_types_test_() ->
+    KnownTypes = known_types(),
+    SizedTypes = [{T, ?TEST_MODULE:byte_size_for_type(T)} || T <- KnownTypes],
+    encode_decode_test_(SizedTypes).
+
+prefixes_are_known_types_test() ->
+    MappedPfxs = mapped_prefixes(),
+    KnownTypes = known_types(),
+    lists:foreach(
+      fun({Pfx, Type}) ->
+              case lists:member(Type, KnownTypes) of
+                  true -> ok;
+                  false ->
+                      error({not_a_known_type, Pfx, Type})
+              end
+      end, MappedPfxs),
+    lists:foreach(
+      fun(Type) ->
+              case lists:keyfind(Type, 2, MappedPfxs) of
+                  {_, _} -> ok;
+                  false ->
+                      error({has_no_mapped_prefix, Type})
+              end
+      end, KnownTypes).
+
+encode_decode_test_(Types) ->
     [{"Byte sizes are correct",
       fun() ->
           lists:foreach(
@@ -33,7 +62,7 @@ encode_decode_test_() ->
                   {_Type, _, ByteSize} = {Type, ByteSize,
                                           ?TEST_MODULE:byte_size_for_type(Type)}
               end,
-              ?TYPES)
+              Types)
       end
      },
      {"Serialize/deserialize known types",
@@ -50,7 +79,7 @@ encode_decode_test_() ->
                     {Type, Key} = ?TEST_MODULE:decode(EncodedKey),
                     {ok, Key} = ?TEST_MODULE:safe_decode(Type, EncodedKey)
               end,
-              ?TYPES)
+              Types)
       end
      },
      {"Key size check works",
@@ -68,7 +97,7 @@ encode_decode_test_() ->
                     CheckIlligalSize(ByteSize - 1),
                     CheckIlligalSize(ByteSize + 1)
               end,
-              ?TYPES)
+              Types)
       end
      },
      {"Missing prefix",
@@ -91,7 +120,7 @@ encode_decode_test_() ->
                     <<_WholePrefix:3/unit:8, RestOfKey2/binary>> = EncodedKey,
                     {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, RestOfKey2)
               end,
-              ?TYPES)
+              Types)
       end
      },
      {"Piece of encoded key",
@@ -110,7 +139,7 @@ encode_decode_test_() ->
                     {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, HalfKey),
                     {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, RestOfKey)
               end,
-              ?TYPES)
+              Types)
       end
      },
      {"Encode/decode binary with only zeros",
@@ -131,8 +160,35 @@ encode_decode_test_() ->
                       Encoded1 = base58:binary_to_base58(Bin),
                       Decoded1 = base58:base58_to_binary(Encoded1),
                       ?assertEqual(Bin, Decoded1)
-                  end, ?TYPES)
+                  end, Types)
             end,
             Bins)
       end}
      ].
+
+known_types() ->
+    Forms = get_forms(),
+    [{type, _, union, Types}] =
+        [Def || {attribute, _, type, {known_type, Def, []}} <- Forms],
+    [Name || {atom,_, Name} <- Types].
+
+mapped_prefixes() ->
+    Forms = get_forms(),
+    [Clauses] = [Cs || {function,_,pfx2type,1,Cs} <- Forms],
+    Abst = [{B, A} || {clause,_,[B],[],[A]} <- Clauses],
+    lists:map(
+      fun({B, A}) ->
+              {eval_expr(B), eval_expr(A)}
+      end, Abst).
+
+get_forms() ->
+    get_forms(code:which(?TEST_MODULE)).
+
+get_forms(Beam) ->
+    {ok, {_, [{abstract_code, {raw_abstract_v1, Forms}}]}} =
+        beam_lib:chunks(Beam, [abstract_code]),
+    Forms.
+
+eval_expr(Expr) ->
+    {value, Val, []} = erl_eval:expr(Expr, []),
+    Val.