Merge pull request 'uw-dyn-options' (#50) from uw-dyn-options into master

Reviewed-on: #50

README.md

@@ -2,6 +2,8 @@
 
 Serialization helpers for the Gajumaru.
 
+For an overview of the static serializer, see [this document](doc/static.md).
+
 ## Build
 
     $ rebar3 compile
@@ -30,6 +32,7 @@ how the type information is represented. The fully serialized form is
 produced by the `serialize` functions.
 
 The basic types supported by the encoder are:
+* `integer()`         (`anyint`, code: 246)
 * `neg_integer()`     (`negint`, code: 247)
 * `non_neg_integer()` (`int`   , code: 248)
 * `binary()`          (`binary`, code: 249)
@@ -40,6 +43,9 @@ The basic types supported by the encoder are:
 * `gmser_id:id()`     (`id`    , code: 254)
 * `atom()`            (`label` , code: 255)
 
+(The range of codes is chosen because the `gmser_chain_objects` codes
+range from 10 to 200, and also to stay within 1 byte.)
+
 When encoding `map` types, the map elements are first sorted.
 
 When specifying a map type for template-driven encoding, use 
@@ -52,6 +58,24 @@ Labels correspond to (existing) atoms in Erlang.
 Decoding of a label results in a call to `binary_to_existing_atom/2`, so will
 fail if the corresponding atom does not already exist.
 
+This behavior can be modified using the option `#{missing_labels => fail | create | convert}`,
+where `fail` is the default, as described above, `convert` means that missing atoms are
+converted to binaries, and `create` means that the atom is created dynamically.
+
+The option can be passed e.g.:
+```erlang
+gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert}))
+```
+
+or
+```erlang
+gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert}, Types))
+```
+
+By calling `gmser_dyn:register_types/1`, after having added options to the type map,
+the options can be made to take effect automatically.
+
+
 It's possible to cache labels for more compact encoding.
 Note that when caching labels, the same cache mapping needs to be used on the
 decoder side.
@@ -122,8 +146,17 @@ to encode as each type in the list, in the specified order, until one matches.
 gmser_dyn:encode_typed(#{alt => [negint,int]}, 5) -> [<<0>>,<<1>>,[<<247>>,<<5>>]]
 gmser_dyn:encode_typed(#{alt => [negint,int]}, 5) -> [<<0>>,<<1>>,[<<248>>,<<5>>]]
 
-gmser_dyn:register_type(246, anyint, #{alt => [negint, int]})
-
 gmser_dyn:encode_typed(anyint,-5) -> [<<0>>,<<1>>,[<<246>>,[<<247>>,<<5>>]]]
 gmser_dyn:encode_typed(anyint,5)  -> [<<0>>,<<1>>,[<<246>>,[<<248>>,<<5>>]]]
 ```
+
+### Notes
+
+Note that `anyint` is a standard type. The static serializer supports only
+positive integers (`int`), as negative numbers are forbidden on-chain.
+For dynamic encoding e.g. in messaging protocols, handling negative numbers can
+be useful, so the `negint` type was added as a dynamic type. To encode a full-range
+integer, the `alt` construct is needed.
+
+(Floats are not supported, as they are non-deterministic. Rationals and fixed-point
+numbers could easily be handled as high-level types, e.g. as `{int,int}`.)

doc/static.md

@@ -0,0 +1,83 @@
+
+# Static Serialization
+
+The `gmserialization` and `gmser_chain_objects` modules implement the
+static serialization support used in the Gajumaru blockchain.
+
+The purpose is to produce fully deterministic serialization, in order
+to maintain predictable hashing.
+
+Example:
+
+```erlang
+%% deterministic canonical serialization.
+-spec serialize_to_binary(signed_tx()) -> binary_signed_tx().
+serialize_to_binary(#signed_tx{tx = Tx, signatures = Sigs}) ->
+    gmser_chain_objects:serialize(
+      ?SIG_TX_TYPE,
+      ?SIG_TX_VSN,
+      serialization_template(?SIG_TX_VSN),
+      [ {signatures, lists:sort(Sigs)}
+      , {transaction, aetx:serialize_to_binary(Tx)}
+      ]).
+
+-spec deserialize_from_binary(binary()) -> signed_tx().
+deserialize_from_binary(SignedTxBin) when is_binary(SignedTxBin) ->
+    [ {signatures, Sigs}
+    , {transaction, TxBin}
+    ] = gmser_chain_objects:deserialize(
+          ?SIG_TX_TYPE,
+          ?SIG_TX_VSN,
+          serialization_template(?SIG_TX_VSN),
+          SignedTxBin),
+    assert_sigs_size(Sigs),
+    #signed_tx{ tx = aetx:deserialize_from_binary(TxBin)
+              , signatures = Sigs
+              }.
+
+serialization_template(?SIG_TX_VSN) ->
+    [ {signatures, [binary]}
+    , {transaction, binary}
+    ].
+```
+
+The terms that can be encoded using these templates are given by
+this type in `gmserialization.erl`:
+
+```erlang
+-type encodable_term() :: non_neg_integer()
+                        | binary()
+                        | boolean()
+                        | [encodable_term()] %% Of any length
+                        | #{atom() => encodable_term()}
+                        | tuple()  %% Any arity, containing encodable_term().
+                        | gmser_id:id().
+```
+
+The template 'language' is defined by these types:
+
+```erlang
+-type template() :: [{field_name(), type()}].
+-type field_name() :: atom().
+-type type() :: 'int'
+              | 'bool'
+              | 'binary'
+              | 'id'     %% As defined in aec_id.erl
+              | [type()] %% Length one in the type. This means a list of any length.
+              | #{items := [{field_name(), type()}]} %% Record with named fields
+                                                     %% represented as a map.
+                                                     %% Encoded as a list in the given
+                                                     %% order.
+              | tuple(). %% Any arity, containing type(). This means a static size array.
+```
+
+The `gmser_chain_objects.erl` module specifies a serialization code for each
+object that can go on-chain. E.g.:
+
+```erlang
+tag(signed_tx) -> 11;
+...
+rev_tag(11) -> signed_tx;
+```
+
+The `tag` and `vsn` are laid out in the beginning of the serialized object.

src/gmser_dyn.erl

@@ -17,6 +17,9 @@
 %% register a type schema, inspect existing schema
 -export([ register_types/1
         , registered_types/0
+        , get_opts/1
+        , set_opts/1
+        , set_opts/2
         , types_from_list/1
         , revert_to_default_types/0
         , dynamic_types/0 ]).
@@ -86,7 +89,8 @@ decode_tag_and_vsn([TagBin, VsnBin, Fields]) ->
 dynamic_types() ->
     #{ vsn   => ?VSN
      , codes =>
-           #{ 247 => negint
+           #{ 246 => anyint
+            , 247 => negint
             , 248 => int
             , 249 => binary
             , 250 => bool
@@ -96,7 +100,8 @@ dynamic_types() ->
             , 254 => id
             , 255 => label}
      , rev =>
-           #{ negint => 247
+           #{ anyint => 246
+            , negint => 247
             , int    => 248
             , binary => 249
             , bool   => 250
@@ -108,7 +113,8 @@ dynamic_types() ->
      , labels => #{}
      , rev_labels => #{}
      , templates =>
-           #{ negint => negint
+           #{ anyint => #{alt => [negint, int]}
+            , negint => negint
             , int    => int
             , binary => binary
             , bool   => bool
@@ -118,6 +124,7 @@ dynamic_types() ->
             , id     => id
             , label  => label
             }
+     , options => #{}
        }.
 
 vsn(Types) ->
@@ -197,11 +204,6 @@ encode_typed_(Tag, Term, E, Vsn, #{templates := Ts, rev := Rev} = Types)
 encode_typed_(MaybeTemplate, Term, _, Vsn, Types) ->
     encode_maybe_template(MaybeTemplate, Term, Vsn, Types).
 
-maybe_emit(E, Code, Enc) when E > 0 ->
-    [encode_basic(int, Code), Enc];
-maybe_emit(0, _, Enc) ->
-    Enc.
-
 encode_maybe_template(#{items := _} = Type, Term, Vsn, Types) ->
     case is_map(Term) of
         true ->
@@ -401,15 +403,29 @@ emit(0, _, _, Enc) ->
 emit_code(Tag, #{rev := Tags}) ->
     encode_basic(int, maps:get(Tag, Tags)).
 
-decode_basic(Type, [Tag,V], #{codes := Codes}) ->
+decode_basic(Type, [Tag,V], #{codes := Codes} = Types) ->
     case decode_basic(int, Tag) of
         Code when map_get(Code, Codes) == Type ->
-            decode_basic(Type, V);
+            decode_basic_(Type, V, Types);
         _ ->
             error(illegal)
     end;
-decode_basic(Type, V, _) ->
-    decode_basic(Type, V).
+decode_basic(Type, V, Types) ->
+    decode_basic_(Type, V, Types).
+
+decode_basic_(label, Fld, #{options := #{missing_labels := Opt}}) ->
+    Bin = decode_basic(binary, Fld),
+    case Opt of
+        create  -> binary_to_atom(Bin, utf8);
+        fail    -> binary_to_existing_atom(Bin, utf8);
+        convert ->
+            try binary_to_existing_atom(Bin, utf8)
+            catch
+                error:_ -> Bin
+            end
+    end;
+decode_basic_(label, Fld, _) ->
+    decode_basic(label, Fld).
 
 decode_basic(label, Fld) ->
     binary_to_existing_atom(decode_basic(binary, Fld), utf8);
@@ -475,6 +491,15 @@ register_type(Code, Tag, Template) when is_integer(Code), Code >= 0 ->
         {_, true} -> error(tag_exists)
     end.
 
+set_opts(Opts) ->
+    set_opts(Opts, registered_types()).
+
+set_opts(Opts, Types) ->
+    Types#{options => Opts}.
+
+get_opts(#{options := Opts}) ->
+    Opts.
+
 cache_label(Code, Label) when is_integer(Code), Code >= 0, is_atom(Label) ->
     #{labels := Lbls, rev_labels := RevLbls} = Types = registered_types(),
     case {is_map_key(Label, Lbls), is_map_key(Code, RevLbls)} of
@@ -631,6 +656,7 @@ dynamic_types_test_() ->
     [ ?_test(revert_to_default_types())
     , ?_test(t_typed_map())
     , ?_test(t_alts())
+    , ?_test(t_anyints())
     ].
 
 t_round_trip(T) ->
@@ -734,4 +760,9 @@ t_alts() ->
     t_round_trip_typed(#{alt => [negint, int]}, 4),
     ok.
 
+t_anyints() ->
+    t_round_trip_typed(anyint, -5),
+    t_round_trip_typed(anyint, 5),
+    ok.
+
 -endif.