Compare commits
No commits in common. "dd1c2455f06aef1fc96d2c942131c0dcc26e2ed8" and "4663a0f57ee9c0608ffd639ff38681587a382712" have entirely different histories.
dd1c2455f0
...
4663a0f57e
99
README.md
99
README.md
@ -13,102 +13,3 @@ Test
|
|||||||
----
|
----
|
||||||
|
|
||||||
$ rebar3 eunit
|
$ rebar3 eunit
|
||||||
|
|
||||||
Dynamic encoding
|
|
||||||
----
|
|
||||||
|
|
||||||
The module `gmser_dyn` offers dynamic encoding support, encoding most 'regular'
|
|
||||||
Erlang data types into an internal RLP representation.
|
|
||||||
|
|
||||||
Main API:
|
|
||||||
* `encode(term()) -> iolist()`
|
|
||||||
* `encode_typed(template(), term()) -> iolist()`
|
|
||||||
* `decode(iolist()) -> term()`
|
|
||||||
|
|
||||||
* `serialize(term()) -> binary()`
|
|
||||||
* `serialize_typed(template(), term()) -> binary()`
|
|
||||||
* `deserialize(binary()) -> term()`
|
|
||||||
|
|
||||||
In the examples below, we use the `decode` functions, to illustrate
|
|
||||||
how the type information is represented. The fully serialized form is
|
|
||||||
produced by the `serialize` functions.
|
|
||||||
|
|
||||||
The basic types supported by the encoder are:
|
|
||||||
* `non_neg_integer()` (`int` , code: 248)
|
|
||||||
* `binary()` (`binary`, code: 249)
|
|
||||||
* `boolean()` (`bool` , code: 250)
|
|
||||||
* `list()` (`list` , code: 251)
|
|
||||||
* `map()` (`map` , code: 252)
|
|
||||||
* `tuple()` (`tuple` , code: 253)
|
|
||||||
* `gmser_id:id()` (`id` , code: 254)
|
|
||||||
* `atom()` (`label` , code: 255)
|
|
||||||
|
|
||||||
When encoding `map` types, the map elements are first sorted.
|
|
||||||
|
|
||||||
When specifying a map type for template-driven encoding, use
|
|
||||||
the `#{items => [{Key, Value}]}` construct.
|
|
||||||
|
|
||||||
Labels
|
|
||||||
----
|
|
||||||
|
|
||||||
Labels correspond to (existing) atoms in Erlang.
|
|
||||||
Decoding of a label results in a call to `binary_to_existing_atom/2`, so will
|
|
||||||
fail if the corresponding atom does not already exist.
|
|
||||||
|
|
||||||
It's possible to cache labels for more compact encoding.
|
|
||||||
Note that when caching labels, the same cache mapping needs to be used on the
|
|
||||||
decoder side.
|
|
||||||
|
|
||||||
Labels are encoded as `[<<255>>, << AtomToBinary/binary >>]`.
|
|
||||||
If a cached label is used, the encoding becomes `[<<255>, [Ix]]`, where
|
|
||||||
`Ix` is the integer-encoded index value of the cached label.
|
|
||||||
|
|
||||||
Examples
|
|
||||||
----
|
|
||||||
|
|
||||||
Dynamically encoded objects have the basic structure `[<<0>>,V,Obj]`, where `V` is the
|
|
||||||
integer-coded version, and `Obj` is the top-level encoding on the form `[Tag,Data]`.
|
|
||||||
|
|
||||||
```erlang
|
|
||||||
E = fun(T) -> io:fwrite("~w~n", [gmser_dyn:encode(T)]) end.
|
|
||||||
|
|
||||||
E(17) -> [<<0>>,<<1>>,[<<248>>,<<17>>]]
|
|
||||||
E(<<"abc">>) -> [<<0>>,<<1>>,[<<249>>,<<97,98,99>>]]
|
|
||||||
E(true) -> [<<0>>,<<1>>,[<<250>>,<<1>>]]
|
|
||||||
E(false) -> [<<0>>,<<1>>,[<<250>>,<<0>>]]
|
|
||||||
E([1,2]) -> [<<0>>,<<1>>,[<<251>>,[[<<248>>,<<1>>],[<<248>>,<<2>>]]]]
|
|
||||||
E({1,2}) -> [<<0>>,<<1>>,[<<253>>,[[<<248>>,<<1>>],[<<248>>,<<2>>]]]]
|
|
||||||
E(#{a=>1, b=>2}) ->
|
|
||||||
[<<0>>,<<1>>,[<<252>>,[[[<<255>>,<<97>>],[<<248>>,<<1>>]],[[<<255>>,<<98>>],[<<248>>,<<2>>]]]]]
|
|
||||||
E(gmser_id:create(account,<<1:256>>)) ->
|
|
||||||
[<<0>>,<<1>>,[<<254>>,<<1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>>]]
|
|
||||||
```
|
|
||||||
|
|
||||||
Note that tuples and list are encoded the same way, except for the initial type tag.
|
|
||||||
Maps are encoded as `[<Map>, [KV1, KV2, ...]]`, where `[KV1, KV2, ...]` is the sorted
|
|
||||||
list of key-value tuples from `map:to_list(Map)`, but with the `tuple` type tag omitted.
|
|
||||||
|
|
||||||
Template-driven encoding
|
|
||||||
----
|
|
||||||
|
|
||||||
Templates can be provided to the encoder by either naming an already registered
|
|
||||||
type, or by passing a template directly. In both cases, the encoder will enforce
|
|
||||||
the type information in the template.
|
|
||||||
|
|
||||||
If the template has been registered, the encoder omits inner type tags (still
|
|
||||||
inserting the top-level tag), leading to some compression of the output.
|
|
||||||
This also means that the serialized term cannot be decoded without the same
|
|
||||||
schema information on the decoder side.
|
|
||||||
|
|
||||||
In the case of a directly provided template, all type information is inserted,
|
|
||||||
such that the serialized term can be decoded without any added type information.
|
|
||||||
The template types are still enforced during encoding.
|
|
||||||
|
|
||||||
```erlang
|
|
||||||
ET = fun(Type,Term) -> io:fwrite("~w~n", [gmser_dyn:encode_typed(Type,Term)]) end.
|
|
||||||
|
|
||||||
ET([{int,int}], [{1,2}]) -> [<<0>>,<<1>>,[<<251>>,[[[<<248>>,<<1>>],[<<248>>,<<2>>]]]]]
|
|
||||||
|
|
||||||
gmser_dyn:register_type(1000,lt2i,[{int,int}]).
|
|
||||||
ET(lt2i, [{1,2}]) -> [<<0>>,<<1>>,[<<3,232>>,[[<<1>>,<<2>>]]]]
|
|
||||||
```
|
|
||||||
|
@ -14,17 +14,11 @@
|
|||||||
, deserialize/1
|
, deserialize/1
|
||||||
, deserialize/2 ]).
|
, deserialize/2 ]).
|
||||||
|
|
||||||
%% register a type schema, inspect existing schema
|
|
||||||
-export([ register_types/1
|
-export([ register_types/1
|
||||||
, registered_types/0
|
, registered_types/0
|
||||||
, types_from_list/1
|
|
||||||
, revert_to_default_types/0
|
, revert_to_default_types/0
|
||||||
, dynamic_types/0 ]).
|
, dynamic_types/0 ]).
|
||||||
|
|
||||||
%% Register individual types, or cache labels
|
|
||||||
-export([ register_type/3
|
|
||||||
, cache_label/2 ]).
|
|
||||||
|
|
||||||
-import(gmserialization, [ decode_field/2 ]).
|
-import(gmserialization, [ decode_field/2 ]).
|
||||||
|
|
||||||
-define(VSN, 1).
|
-define(VSN, 1).
|
||||||
@ -94,7 +88,7 @@ dynamic_types() ->
|
|||||||
, 252 => map
|
, 252 => map
|
||||||
, 253 => tuple
|
, 253 => tuple
|
||||||
, 254 => id
|
, 254 => id
|
||||||
, 255 => label}
|
, 255 => label }
|
||||||
, rev =>
|
, rev =>
|
||||||
#{ int => 248
|
#{ int => 248
|
||||||
, binary => 249
|
, binary => 249
|
||||||
@ -104,8 +98,6 @@ dynamic_types() ->
|
|||||||
, tuple => 253
|
, tuple => 253
|
||||||
, id => 254
|
, id => 254
|
||||||
, label => 255}
|
, label => 255}
|
||||||
, labels => #{}
|
|
||||||
, rev_labels => #{}
|
|
||||||
, templates =>
|
, templates =>
|
||||||
#{ int => int
|
#{ int => int
|
||||||
, binary => binary
|
, binary => binary
|
||||||
@ -121,14 +113,67 @@ dynamic_types() ->
|
|||||||
vsn(Types) ->
|
vsn(Types) ->
|
||||||
maps:get(vsn, Types, ?VSN).
|
maps:get(vsn, Types, ?VSN).
|
||||||
|
|
||||||
registered_types() ->
|
register_types(Types) when is_map(Types) ->
|
||||||
case persistent_term:get({?MODULE, types}, undefined) of
|
Codes = maps:get(codes, Types, #{}),
|
||||||
undefined ->
|
Rev = rev_codes(Codes),
|
||||||
dynamic_types();
|
Templates = maps:get(templates, Types, #{}),
|
||||||
Types when is_map(Types) ->
|
#{codes := Codes0, rev := Rev0, templates := Templates0} =
|
||||||
Types
|
dynamic_types(),
|
||||||
|
Merged = #{ codes => maps:merge(Codes0, Codes)
|
||||||
|
, rev => maps:merge(Rev0, Rev)
|
||||||
|
, templates => maps:merge(Templates0, Templates) },
|
||||||
|
assert_sizes(Merged),
|
||||||
|
assert_mappings(Merged),
|
||||||
|
persistent_term:put({?MODULE, types}, Merged).
|
||||||
|
|
||||||
|
revert_to_default_types() ->
|
||||||
|
persistent_term:put({?MODULE, types}, dynamic_types()).
|
||||||
|
|
||||||
|
assert_sizes(#{codes := Codes, rev := Rev, templates := Ts} = Types) ->
|
||||||
|
assert_sizes(map_size(Codes), map_size(Rev), map_size(Ts), Types).
|
||||||
|
|
||||||
|
assert_sizes(Sz, Sz, Sz, _) ->
|
||||||
|
ok;
|
||||||
|
assert_sizes(Sz, RSz, Sz, Types) when RSz =/= Sz ->
|
||||||
|
%% Wrong size reverse mapping must mean duplicate mappings
|
||||||
|
%% We auto-generate the reverse-mappings, so we know there aren't
|
||||||
|
%% too many of them
|
||||||
|
?LOG_ERROR("Reverse mapping size doesn't match codes size", []),
|
||||||
|
Codes = maps:get(codes, Types),
|
||||||
|
CodeVals = maps:values(Codes),
|
||||||
|
Duplicates = CodeVals -- lists:usort(CodeVals),
|
||||||
|
error({duplicate_mappings, Duplicates, Types});
|
||||||
|
assert_sizes(Sz, _, TSz, Types) when Sz > TSz ->
|
||||||
|
?LOG_ERROR("More codes than templates", []),
|
||||||
|
Tags = maps:keys(maps:get(rev, Types)),
|
||||||
|
Templates = maps:get(templates, Types),
|
||||||
|
Missing = [T || T <- Tags,
|
||||||
|
not is_map_key(T, Templates)],
|
||||||
|
error({missing_mappings, Missing, Types});
|
||||||
|
assert_sizes(Sz, _, TSz, Types) when TSz > Sz ->
|
||||||
|
%% More mappings than codes. May not be horrible.
|
||||||
|
%% We check that all codes have mappings elsewhere.
|
||||||
|
?LOG_WARNING("More templates than codes in ~p", [Types]),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
assert_mappings(#{rev := Rev, templates := Ts} = Types) ->
|
||||||
|
Tags = maps:keys(Rev),
|
||||||
|
case [T || T <- Tags,
|
||||||
|
not is_map_key(T, Ts)] of
|
||||||
|
[] ->
|
||||||
|
ok;
|
||||||
|
Missing ->
|
||||||
|
?LOG_ERROR("Missing templates for ~p", [Missing]),
|
||||||
|
error({missing_templates, Missing, Types})
|
||||||
end.
|
end.
|
||||||
|
|
||||||
|
rev_codes(Codes) ->
|
||||||
|
L = maps:to_list(Codes),
|
||||||
|
maps:from_list([{V, K} || {K, V} <- L]).
|
||||||
|
|
||||||
|
registered_types() ->
|
||||||
|
persistent_term:get({?MODULE, types}, dynamic_types()).
|
||||||
|
|
||||||
template(TagOrCode, Vsn, Types) ->
|
template(TagOrCode, Vsn, Types) ->
|
||||||
{Tag, Template} = get_template(TagOrCode, Types),
|
{Tag, Template} = get_template(TagOrCode, Types),
|
||||||
{Tag, dyn_template_(Template, Vsn)}.
|
{Tag, dyn_template_(Template, Vsn)}.
|
||||||
@ -145,9 +190,6 @@ dyn_template_(F, Vsn) ->
|
|||||||
true -> F
|
true -> F
|
||||||
end.
|
end.
|
||||||
|
|
||||||
find_cached_label(Lbl, #{labels := Lbls}) ->
|
|
||||||
maps:find(Lbl, Lbls).
|
|
||||||
|
|
||||||
decode_(Fields, Vsn, Types, Acc) ->
|
decode_(Fields, Vsn, Types, Acc) ->
|
||||||
{_Tag, Term, Rest} = decode_field_(Fields, Vsn, Types),
|
{_Tag, Term, Rest} = decode_field_(Fields, Vsn, Types),
|
||||||
Acc1 = [Term | Acc],
|
Acc1 = [Term | Acc],
|
||||||
@ -182,33 +224,21 @@ encode_(Term, Emit, Vsn, Types) ->
|
|||||||
Enc
|
Enc
|
||||||
end.
|
end.
|
||||||
|
|
||||||
encode_typed_(Type, Term, Vsn, Types) ->
|
encode_typed_(Code, Term, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) ->
|
||||||
encode_typed_(Type, Term, true, Vsn, Types).
|
|
||||||
|
|
||||||
encode_typed_(any, Term, _, Vsn, Types) ->
|
|
||||||
encode_(Term, true, Vsn, Types);
|
|
||||||
encode_typed_(Code, Term, Emit, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) ->
|
|
||||||
{_Tag, Template} = template(Code, Vsn, Types),
|
{_Tag, Template} = template(Code, Vsn, Types),
|
||||||
maybe_emit(Emit, Code, encode_from_template(Template, Term, false, Vsn, Types));
|
[encode_basic(int, Code), encode_from_template(Template, Term, Vsn, Types)];
|
||||||
encode_typed_(Tag, Term, Emit, Vsn, #{templates := Ts, rev := Rev} = Types)
|
encode_typed_(Tag, Term, Vsn, #{templates := Ts} = Types) when is_map_key(Tag, Ts) ->
|
||||||
when is_map_key(Tag, Ts) ->
|
Template = maps:get(Tag, Ts),
|
||||||
Template = dyn_template_(maps:get(Tag, Ts), Vsn),
|
[emit_code(Tag, Types), encode_from_template(Template, Term, Vsn, Types)];
|
||||||
Code = maps:get(Tag, Rev),
|
encode_typed_(MaybeTemplate, Term, Vsn, Types) ->
|
||||||
maybe_emit(Emit, Code, encode_from_template(Template, Term, false, Vsn, Types));
|
|
||||||
encode_typed_(MaybeTemplate, Term, _, Vsn, Types) ->
|
|
||||||
encode_maybe_template(MaybeTemplate, Term, Vsn, Types).
|
encode_maybe_template(MaybeTemplate, Term, Vsn, Types).
|
||||||
|
|
||||||
maybe_emit(true, Code, Enc) ->
|
|
||||||
[encode_basic(int, Code), Enc];
|
|
||||||
maybe_emit(false, _, Enc) ->
|
|
||||||
Enc.
|
|
||||||
|
|
||||||
encode_maybe_template(Pat, Term, Vsn, Types) when is_list(Pat);
|
encode_maybe_template(Pat, Term, Vsn, Types) when is_list(Pat);
|
||||||
is_tuple(Pat);
|
is_tuple(Pat);
|
||||||
is_map(Pat) ->
|
is_map(Pat) ->
|
||||||
{Tag, _} = auto_template(Pat),
|
{Tag, _} = auto_template(Pat),
|
||||||
[emit_code(Tag, Types),
|
[emit_code(Tag, Types),
|
||||||
encode_from_template(Pat, Term, true, Vsn, Types)];
|
encode_from_template(Pat, Term, Vsn, Types)];
|
||||||
encode_maybe_template(Other, Term, _Vsn, _Types) ->
|
encode_maybe_template(Other, Term, _Vsn, _Types) ->
|
||||||
error({illegal_template, Other, Term}).
|
error({illegal_template, Other, Term}).
|
||||||
|
|
||||||
@ -256,77 +286,56 @@ decode_from_template(Type, V, Vsn, Types) when is_tuple(Type), is_list(V) ->
|
|||||||
Zipped = lists:zip(tuple_to_list(Type), V),
|
Zipped = lists:zip(tuple_to_list(Type), V),
|
||||||
Items = [decode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped],
|
Items = [decode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped],
|
||||||
list_to_tuple(Items);
|
list_to_tuple(Items);
|
||||||
decode_from_template(label, [C], _, #{rev_labels := RLbls}) ->
|
decode_from_template(Type, Fld, _, _) when Type == int
|
||||||
Code = decode_basic(int, C),
|
; Type == binary
|
||||||
maps:get(Code, RLbls);
|
; Type == bool
|
||||||
decode_from_template(Type, Fld, _, Types) when Type == int
|
; Type == id
|
||||||
; Type == binary
|
; Type == label ->
|
||||||
; Type == bool
|
decode_basic(Type, Fld).
|
||||||
; Type == id
|
|
||||||
; Type == label ->
|
|
||||||
decode_basic(Type, Fld, Types).
|
|
||||||
|
|
||||||
encode_from_template(Type, V, Vsn, Types) ->
|
encode_from_template(Type, V, Vsn, Types) ->
|
||||||
encode_from_template(Type, V, true, Vsn, Types).
|
encode_from_template(Type, V, true, Vsn, Types).
|
||||||
|
|
||||||
encode_from_template(any, V, _, Vsn, Types) ->
|
|
||||||
encode_(V, true, Vsn, Types);
|
|
||||||
encode_from_template(list, L, _, Vsn, Types) when is_list(L) ->
|
encode_from_template(list, L, _, Vsn, Types) when is_list(L) ->
|
||||||
assert_type(is_list(L), list, L),
|
|
||||||
[encode_(V, Vsn, Types) || V <- L];
|
[encode_(V, Vsn, Types) || V <- L];
|
||||||
encode_from_template(map, M, _, Vsn, Types) ->
|
encode_from_template(map, M, _, Vsn, Types) when is_map(M) ->
|
||||||
assert_type(is_map(M), map, M),
|
|
||||||
[encode_({K,V}, false, Vsn, Types)
|
[encode_({K,V}, false, Vsn, Types)
|
||||||
|| {K, V} <- lists:sort(maps:to_list(M))];
|
|| {K, V} <- lists:sort(maps:to_list(M))];
|
||||||
encode_from_template(tuple, T, Emit, Vsn, Types) ->
|
encode_from_template(tuple, T, _, Vsn, Types) when is_tuple(T) ->
|
||||||
assert_type(is_tuple(T), tuple, T),
|
[encode_(V, Vsn, Types) || V <- tuple_to_list(T)];
|
||||||
[encode_(V, Emit, Vsn, Types) || V <- tuple_to_list(T)];
|
encode_from_template(T, V, _, Vsn, Types) when tuple_size(T) =:= tuple_size(V) ->
|
||||||
encode_from_template(T, V, Emit, Vsn, Types) when is_tuple(T) ->
|
|
||||||
assert_type(is_tuple(V), T, V),
|
|
||||||
assert_type(tuple_size(T) =:= tuple_size(V), T, V),
|
|
||||||
Zipped = lists:zip(tuple_to_list(T), tuple_to_list(V)),
|
Zipped = lists:zip(tuple_to_list(T), tuple_to_list(V)),
|
||||||
[encode_from_template(T1, V1, Emit, Vsn, Types) || {T1, V1} <- Zipped];
|
[encode_from_template(T1, V1, false, Vsn, Types) || {T1, V1} <- Zipped];
|
||||||
encode_from_template([Type] = T, List, Emit, Vsn, Types) ->
|
encode_from_template([Type], List, _, Vsn, Types) ->
|
||||||
assert_type(is_list(List), T, List),
|
[encode_from_template(Type, V, false, Vsn, Types) || V <- List];
|
||||||
[encode_from_template(Type, V, Emit, Vsn, Types) || V <- List];
|
encode_from_template(Type, List, _, Vsn, Types) when is_list(Type), is_list(List) ->
|
||||||
encode_from_template(Type, List, Emit, Vsn, Types) when is_list(Type), is_list(List) ->
|
encode_fields(Type, List, Vsn, Types);
|
||||||
encode_fields(Type, List, Emit, Vsn, Types);
|
encode_from_template(Type, V, _, _, _Types) when Type == id
|
||||||
encode_from_template(label, V, Emit, _, Types) ->
|
; Type == binary
|
||||||
assert_type(is_atom(V), label, V),
|
; Type == bool
|
||||||
case find_cached_label(V, Types) of
|
; Type == int
|
||||||
error ->
|
; Type == label ->
|
||||||
encode_basic(label, V, Emit, Types);
|
encode_basic(Type, V);
|
||||||
{ok, Code} when is_integer(Code) ->
|
encode_from_template(Type, V, _, _, _) ->
|
||||||
[encode_basic(int, Code)]
|
error({illegal, Type, V}).
|
||||||
end;
|
|
||||||
encode_from_template(Type, V, Emit, _, Types) when Type == id
|
|
||||||
; Type == binary
|
|
||||||
; Type == bool
|
|
||||||
; Type == int
|
|
||||||
; Type == label ->
|
|
||||||
encode_basic(Type, V, Emit, Types);
|
|
||||||
encode_from_template(Type, V, Emit, Vsn, Types) ->
|
|
||||||
encode_typed_(Type, V, Emit, Vsn, Types).
|
|
||||||
%% error({illegal, Type, V}).
|
|
||||||
|
|
||||||
assert_type(true, _, _) -> ok;
|
|
||||||
assert_type(_, Type, V) -> error({illegal, Type, V}).
|
|
||||||
|
|
||||||
|
|
||||||
%% Basically, dynamically encoding a statically defined object
|
%% Basically, dynamically encoding a statically defined object
|
||||||
encode_fields([{Field, Type}|TypesLeft],
|
encode_fields([{Field, Type}|TypesLeft],
|
||||||
[{Field, Val}|FieldsLeft], Emit, Vsn, Types) ->
|
[{Field, Val}|FieldsLeft], Vsn, Types) ->
|
||||||
[ encode_from_template(Type, Val, Emit, Vsn, Types)
|
[ encode_from_template(Type, Val, Vsn, Types)
|
||||||
| encode_fields(TypesLeft, FieldsLeft, Emit, Vsn, Types)];
|
| encode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
|
||||||
encode_fields([{_Field, _Type} = FT|_TypesLeft],
|
encode_fields([{_Field, Type}|TypesLeft],
|
||||||
[Val |_FieldsLeft], _Emit, _Vsn, _Types) ->
|
[Val |FieldsLeft], Vsn, Types) ->
|
||||||
error({illegal_field, FT, Val});
|
%% Not sure if we want to try this ...
|
||||||
|
[ encode_from_template(Type, Val, Vsn, Types)
|
||||||
|
| encode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
|
||||||
encode_fields([Type|TypesLeft],
|
encode_fields([Type|TypesLeft],
|
||||||
[Val |FieldsLeft], Emit, Vsn, Types) when is_atom(Type) ->
|
[Val |FieldsLeft], Vsn, Types) when is_atom(Type) ->
|
||||||
%% Not sure about this ...
|
%% Not sure about this either ...
|
||||||
[ encode_from_template(Type, Val, Emit, Vsn, Types)
|
[ encode_from_template(Type, Val, Vsn, Types)
|
||||||
| encode_fields(TypesLeft, FieldsLeft, Emit, Vsn, Types)];
|
| encode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
|
||||||
encode_fields([], [], _, _, _) ->
|
encode_fields([], [], _, _) ->
|
||||||
[].
|
[].
|
||||||
|
|
||||||
decode_fields([{Tag, Type}|TypesLeft],
|
decode_fields([{Tag, Type}|TypesLeft],
|
||||||
@ -340,26 +349,11 @@ decode_fields([], [], _, _) ->
|
|||||||
emit_code(Tag, #{rev := Tags}) ->
|
emit_code(Tag, #{rev := Tags}) ->
|
||||||
encode_basic(int, maps:get(Tag, Tags)).
|
encode_basic(int, maps:get(Tag, Tags)).
|
||||||
|
|
||||||
decode_basic(Type, [Tag,V], #{codes := Codes}) ->
|
|
||||||
case decode_basic(int, Tag) of
|
|
||||||
Code when map_get(Code, Codes) == Type ->
|
|
||||||
decode_basic(Type, V);
|
|
||||||
_ ->
|
|
||||||
error(illegal)
|
|
||||||
end;
|
|
||||||
decode_basic(Type, V, _) ->
|
|
||||||
decode_basic(Type, V).
|
|
||||||
|
|
||||||
decode_basic(label, Fld) ->
|
decode_basic(label, Fld) ->
|
||||||
binary_to_existing_atom(decode_basic(binary, Fld), utf8);
|
binary_to_existing_atom(decode_basic(binary, Fld), utf8);
|
||||||
decode_basic(Type, Fld) ->
|
decode_basic(Type, Fld) ->
|
||||||
gmserialization:decode_field(Type, Fld).
|
gmserialization:decode_field(Type, Fld).
|
||||||
|
|
||||||
encode_basic(Tag, V, true, Types) ->
|
|
||||||
[emit_code(Tag, Types), encode_basic(Tag, V)];
|
|
||||||
encode_basic(Tag, V, false, _) ->
|
|
||||||
encode_basic(Tag, V).
|
|
||||||
|
|
||||||
encode_basic(label, A) when is_atom(A) ->
|
encode_basic(label, A) when is_atom(A) ->
|
||||||
encode_basic(binary, atom_to_binary(A, utf8));
|
encode_basic(binary, atom_to_binary(A, utf8));
|
||||||
encode_basic(Type, Fld) ->
|
encode_basic(Type, Fld) ->
|
||||||
@ -371,152 +365,6 @@ rlp_decode(Bin) ->
|
|||||||
rlp_encode(Fields) ->
|
rlp_encode(Fields) ->
|
||||||
gmser_rlp:encode(Fields).
|
gmser_rlp:encode(Fields).
|
||||||
|
|
||||||
%% ===========================================================================
|
|
||||||
%% Type registration and validation code
|
|
||||||
|
|
||||||
register_types(Types) when is_map(Types) ->
|
|
||||||
Codes = maps:get(codes, Types, #{}),
|
|
||||||
Rev = rev_codes(Codes),
|
|
||||||
Templates = maps:get(templates, Types, #{}),
|
|
||||||
Labels = maps:get(labels, Types, #{}),
|
|
||||||
#{codes := Codes0, rev := Rev0, labels := Labels0, templates := Templates0} =
|
|
||||||
dynamic_types(),
|
|
||||||
Merged = #{ codes => maps:merge(Codes0, Codes)
|
|
||||||
, rev => maps:merge(Rev0, Rev)
|
|
||||||
, templates => maps:merge(Templates0, Templates)
|
|
||||||
, labels => maps:merge(Labels0, Labels) },
|
|
||||||
assert_sizes(Merged),
|
|
||||||
assert_mappings(Merged),
|
|
||||||
Merged1 = assert_label_cache(Merged),
|
|
||||||
put_types(Merged1).
|
|
||||||
|
|
||||||
put_types(Types) ->
|
|
||||||
persistent_term:put({?MODULE, types}, Types).
|
|
||||||
|
|
||||||
types_from_list(L) ->
|
|
||||||
lists:foldl(fun elem_to_type/2, dynamic_types(), L).
|
|
||||||
|
|
||||||
register_type(Code, Tag, Template) when is_integer(Code), Code >= 0 ->
|
|
||||||
#{codes := Codes, rev := Rev, templates := Temps} = Types = registered_types(),
|
|
||||||
case {is_map_key(Code, Codes), is_map_key(Tag, Rev)} of
|
|
||||||
{false, false} ->
|
|
||||||
New = Types#{ codes := Codes#{Code => Tag}
|
|
||||||
, rev := Rev#{Tag => Code}
|
|
||||||
, templates := Temps#{Tag => Template} },
|
|
||||||
put_types(New),
|
|
||||||
New;
|
|
||||||
{true, _} -> error(code_exists);
|
|
||||||
{_, true} -> error(tag_exists)
|
|
||||||
end.
|
|
||||||
|
|
||||||
cache_label(Code, Label) when is_integer(Code), Code >= 0, is_atom(Label) ->
|
|
||||||
#{labels := Lbls, rev_labels := RevLbls} = Types = registered_types(),
|
|
||||||
case {is_map_key(Label, Lbls), is_map_key(Code, RevLbls)} of
|
|
||||||
{false, false} ->
|
|
||||||
New = Types#{ labels := Lbls#{Label => Code}
|
|
||||||
, rev_labels := RevLbls#{Code => Label} },
|
|
||||||
put_types(New),
|
|
||||||
New;
|
|
||||||
{true,_} -> error(label_exists);
|
|
||||||
{_,true} -> error(code_exists)
|
|
||||||
end.
|
|
||||||
|
|
||||||
elem_to_type({Tag, Code, Template}, Acc) when is_atom(Tag), is_integer(Code) ->
|
|
||||||
#{codes := Codes, rev := Rev, templates := Temps} = Acc,
|
|
||||||
case {is_map_key(Tag, Rev), is_map_key(Code, Codes)} of
|
|
||||||
{false, false} ->
|
|
||||||
Acc#{ codes := Codes#{Code => Tag}
|
|
||||||
, rev := Rev#{Tag => Code}
|
|
||||||
, templates => Temps#{Tag => Template}
|
|
||||||
};
|
|
||||||
{true, _} -> error({duplicate_tag, Tag});
|
|
||||||
{_, true} -> error({duplicate_code, Code})
|
|
||||||
end;
|
|
||||||
elem_to_type({labels, Lbls}, Acc) ->
|
|
||||||
lists:foldl(fun add_label/2, Acc, Lbls);
|
|
||||||
elem_to_type(Elem, _) ->
|
|
||||||
error({invalid_type_list_element, Elem}).
|
|
||||||
|
|
||||||
add_label({L, Code}, #{labels := Lbls, rev_labels := RevLbls} = Acc)
|
|
||||||
when is_atom(L), is_integer(Code), Code > 0 ->
|
|
||||||
case {is_map_key(L, Lbls), is_map_key(Code, RevLbls)} of
|
|
||||||
{false, false} ->
|
|
||||||
Acc#{labels := Lbls#{L => Code},
|
|
||||||
rev_labels := RevLbls#{Code => L}};
|
|
||||||
{true, _} -> error({duplicate_label, L});
|
|
||||||
{_, true} -> error({duplicate_label_code, Code})
|
|
||||||
end;
|
|
||||||
add_label(Elem, _) ->
|
|
||||||
error({invalid_label_elem, Elem}).
|
|
||||||
|
|
||||||
|
|
||||||
revert_to_default_types() ->
|
|
||||||
persistent_term:put({?MODULE, types}, dynamic_types()).
|
|
||||||
|
|
||||||
assert_sizes(#{codes := Codes, rev := Rev, templates := Ts} = Types) ->
|
|
||||||
assert_sizes(map_size(Codes), map_size(Rev), map_size(Ts), Types).
|
|
||||||
|
|
||||||
assert_sizes(Sz, Sz, Sz, _) ->
|
|
||||||
ok;
|
|
||||||
assert_sizes(Sz, RSz, Sz, Types) when RSz =/= Sz ->
|
|
||||||
%% Wrong size reverse mapping must mean duplicate mappings
|
|
||||||
%% We auto-generate the reverse-mappings, so we know there aren't
|
|
||||||
%% too many of them
|
|
||||||
?LOG_ERROR("Reverse mapping size doesn't match codes size", []),
|
|
||||||
Codes = maps:get(codes, Types),
|
|
||||||
CodeVals = maps:values(Codes),
|
|
||||||
Duplicates = CodeVals -- lists:usort(CodeVals),
|
|
||||||
error({duplicate_mappings, Duplicates, Types});
|
|
||||||
assert_sizes(Sz, _, TSz, Types) when Sz > TSz ->
|
|
||||||
?LOG_ERROR("More codes than templates", []),
|
|
||||||
Tags = maps:keys(maps:get(rev, Types)),
|
|
||||||
Templates = maps:get(templates, Types),
|
|
||||||
Missing = [T || T <- Tags,
|
|
||||||
not is_map_key(T, Templates)],
|
|
||||||
error({missing_mappings, Missing, Types});
|
|
||||||
assert_sizes(Sz, _, TSz, Types) when TSz > Sz ->
|
|
||||||
%% More mappings than codes. May not be horrible.
|
|
||||||
%% We check that all codes have mappings elsewhere.
|
|
||||||
?LOG_WARNING("More templates than codes in ~p", [Types]),
|
|
||||||
ok.
|
|
||||||
|
|
||||||
assert_mappings(#{rev := Rev, templates := Ts} = Types) ->
|
|
||||||
Tags = maps:keys(Rev),
|
|
||||||
case [T || T <- Tags,
|
|
||||||
not is_map_key(T, Ts)] of
|
|
||||||
[] ->
|
|
||||||
ok;
|
|
||||||
Missing ->
|
|
||||||
?LOG_ERROR("Missing templates for ~p", [Missing]),
|
|
||||||
error({missing_templates, Missing, Types})
|
|
||||||
end.
|
|
||||||
|
|
||||||
assert_label_cache(#{labels := Labels} = Types) ->
|
|
||||||
Ls = maps:keys(Labels),
|
|
||||||
case [L || L <- Ls, not is_atom(L)] of
|
|
||||||
[] -> ok;
|
|
||||||
_NonAtoms ->
|
|
||||||
error(non_atoms_in_label_cache)
|
|
||||||
end,
|
|
||||||
Rev = [{C,L} || {L,C} <- maps:to_list(Labels)],
|
|
||||||
case [C || {C,_} <- Rev, not is_integer(C)] of
|
|
||||||
[] -> ok;
|
|
||||||
_NonInts -> error(non_integer_label_cache_codes)
|
|
||||||
end,
|
|
||||||
RevLabels = maps:from_list(Rev),
|
|
||||||
case map_size(RevLabels) == map_size(Labels) of
|
|
||||||
true ->
|
|
||||||
Types#{rev_labels => RevLabels};
|
|
||||||
false ->
|
|
||||||
error(non_unique_label_cache_codes)
|
|
||||||
end.
|
|
||||||
|
|
||||||
rev_codes(Codes) ->
|
|
||||||
L = maps:to_list(Codes),
|
|
||||||
maps:from_list([{V, K} || {K, V} <- L]).
|
|
||||||
|
|
||||||
%% ===========================================================================
|
|
||||||
%% Unit tests
|
|
||||||
|
|
||||||
-ifdef(TEST).
|
-ifdef(TEST).
|
||||||
|
|
||||||
@ -554,10 +402,6 @@ user_types_test_() ->
|
|||||||
end,
|
end,
|
||||||
[ ?_test(t_reg_typed_tuple())
|
[ ?_test(t_reg_typed_tuple())
|
||||||
, ?_test(t_reg_chain_objects_array())
|
, ?_test(t_reg_chain_objects_array())
|
||||||
, ?_test(t_reg_template_fun())
|
|
||||||
, ?_test(t_reg_template_vsnd_fun())
|
|
||||||
, ?_test(t_reg_label_cache())
|
|
||||||
, ?_test(t_reg_label_cache2())
|
|
||||||
]}.
|
]}.
|
||||||
|
|
||||||
t_round_trip(T) ->
|
t_round_trip(T) ->
|
||||||
@ -596,56 +440,4 @@ t_reg_chain_objects_array() ->
|
|||||||
Enc = encode_typed(coa, Values),
|
Enc = encode_typed(coa, Values),
|
||||||
Values = decode(Enc).
|
Values = decode(Enc).
|
||||||
|
|
||||||
t_reg_template_fun() ->
|
|
||||||
Template = fun() -> {int,int} end,
|
|
||||||
New = register_type(1010, tup2f0, Template),
|
|
||||||
?debugFmt("New = ~p", [New]),
|
|
||||||
E = encode_typed(tup2f0, {3,4}),
|
|
||||||
{3,4} = decode(E),
|
|
||||||
ok.
|
|
||||||
|
|
||||||
t_reg_template_vsnd_fun() ->
|
|
||||||
Template = fun(1) -> {int,int} end,
|
|
||||||
New = register_type(1011, tup2f1, Template),
|
|
||||||
?debugFmt("New = ~p", [New]),
|
|
||||||
E = encode_typed(tup2f1, {3,4}),
|
|
||||||
{3,4} = decode(E),
|
|
||||||
ok.
|
|
||||||
|
|
||||||
|
|
||||||
t_reg_label_cache() ->
|
|
||||||
Enc0 = gmser_dyn:encode('1'),
|
|
||||||
?debugFmt("Enc0 (no cache): ~w", [Enc0]),
|
|
||||||
MyTypes1 = #{codes => #{1003 => lbl_tup2}, templates => #{ lbl_tup2 => {label,label} }},
|
|
||||||
register_types(MyTypes1),
|
|
||||||
Enc0a = gmser_dyn:encode_typed(lbl_tup2, {'1','1'}),
|
|
||||||
?debugFmt("Enc0a (no cache): ~w", [Enc0a]),
|
|
||||||
{'1','1'} = gmser_dyn:decode(Enc0a),
|
|
||||||
MyTypes2 = MyTypes1#{labels => #{'1' => 49}}, % atom_to_list('1') == [49]
|
|
||||||
register_types(MyTypes2),
|
|
||||||
Enc1 = gmser_dyn:encode('1'),
|
|
||||||
Enc1a = gmser_dyn:encode_typed(lbl_tup2, {'1','1'}),
|
|
||||||
?debugFmt("Enc1 (w/ cache): ~w", [Enc1]),
|
|
||||||
?debugFmt("Enc1a (w/ cache): ~w", [Enc1a]),
|
|
||||||
{'1','1'} = gmser_dyn:decode(Enc1a),
|
|
||||||
true = Enc0 =/= Enc1,
|
|
||||||
Enc2 = gmser_dyn:encode_typed(label, '1'),
|
|
||||||
?debugFmt("Enc2 (typed): ~w", [Enc2]),
|
|
||||||
?assertEqual(Enc2, Enc1),
|
|
||||||
?assertNotEqual(Enc0a, Enc1a).
|
|
||||||
|
|
||||||
t_reg_label_cache2() ->
|
|
||||||
TFromL = gmser_dyn:types_from_list(
|
|
||||||
[ {lbl_tup2, 1003, {label, label}}
|
|
||||||
, {labels,
|
|
||||||
[{'1', 49}]}
|
|
||||||
]),
|
|
||||||
?debugFmt("TFromL = ~w", [TFromL]),
|
|
||||||
register_types(TFromL),
|
|
||||||
Tup = {'1', '1'},
|
|
||||||
Enc = gmser_dyn:encode_typed(lbl_tup2, Tup),
|
|
||||||
[<<0>>,<<1>>,[<<3,235>>,[[<<49>>],[<<49>>]]]] = Enc,
|
|
||||||
Tup = gmser_dyn:decode(Enc).
|
|
||||||
|
|
||||||
|
|
||||||
-endif.
|
-endif.
|
||||||
|
Loading…
x
Reference in New Issue
Block a user