QPQ-AG/gmserialization
11
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 2 Wiki Activity

Fix OTP 28 dialyzer warnings, rewrite gmser_dyn decoder #55

Open
uwiger wants to merge 2 commits from uw-gmser_dyn-rewrite into master
pull from: uw-gmser_dyn-rewrite
merge into: QPQ-AG:master
Conversation 0 Commits 2 Files Changed 6 +491 -262
3 changed files with 491 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 00699b08b7 - Show all commits

79
README.md
View File

@ -59,12 +59,12 @@ T = #{items => [{a,int},{opt,b,int},{c,int}]}
E1 = gmser_dyn:encode_typed(T, #{a => 1, b => 2, c => 3}) ->
[<<0>>,<<1>>,[<<252>>,
[[[<<255>>,<<97>>],[<<248>>,<<1>>]],
[[<<255>>,<<98>>],[<<248>>,<<2>>]],
[[<<255>>,<<99>>],[<<248>>,<<3>>]]]]]
[[<<255>>,<<98>>],[<<248>>,<<2>>]],
[[<<255>>,<<99>>],[<<248>>,<<3>>]]]]]
E2 = gmser_dyn:encode_typed(T, #{a => 1, c => 3}) ->
[<<0>>,<<1>>,[<<252>>,
[[[<<255>>,<<97>>],[<<248>>,<<1>>]],
[[<<255>>,<<99>>],[<<248>>,<<3>>]]]]]
[[<<255>>,<<99>>],[<<248>>,<<3>>]]]]]
gmser_dyn:decode_typed(T,E2) ->
#{c => 3,a => 1}
```
@ -81,12 +81,12 @@ converted to binaries, and `create` means that the atom is created dynamically.
The option can be passed e.g.:
```erlang
gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert}))
gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert}))
```
or
```erlang
gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert}, Types))
gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert}, Types))
```
By calling `gmser_dyn:register_types/1`, after having added options to the type map,
@ -131,37 +131,68 @@ Templates can be provided to the encoder by either naming an already registered
type, or by passing a template directly. In both cases, the encoder will enforce
the type information in the template.
If the template has been registered, the encoder omits inner type tags (still
inserting the top-level tag), leading to some compression of the output.
This also means that the serialized term cannot be decoded without the same
schema information on the decoder side.
In some cases, the type tags will still be emitted. These are when alternative types
appear, and for enumerated map types (`#{items => ...}`). In the latter case, it is
due to the support for optional items.
In the case of a directly provided template, all type information is inserted,
such that the serialized term can be decoded without any added type information.
The template types are still enforced during encoding.
If the template has been registered, the encoder uses the registered type specification
to drive the encoding. The code of the registered template is embedded in the encoded
output:
```erlang
ET = fun(Type,Term) -> io:fwrite("~w~n", [gmser_dyn:encode_typed(Type,Term)]) end.
gmser_dyn:encode_typed({int,int,int}, {1,2,3}) ->
[<<0>>,<<1>>,[<<253>>,
[[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]]
ET([{int,int}], [{1,2}]) -> [<<0>>,<<1>>,[<<251>>,[[[<<248>>,<<1>>],[<<248>>,<<2>>]]]]]
gmser_dyn:register_type(1000,lt2i,[{int,int}]).
ET(lt2i, [{1,2}]) -> [<<0>>,<<1>>,[<<3,232>>,[[<<1>>,<<2>>]]]]
Types = gmser_dyn_types:add_type(t3,1013,{int,int,int}).
gmser_dyn:encode_typed(t3, {1,2,3}, Types) ->
[<<0>>,<<1>>,[[<<3,245>>,<<253>>],
[[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]]
```
Note that the original `<<253>>` type code is wrapped as `[<<3,245>>,<<253>>]`,
where `<<3,245>>` corresponds to the custom code `1013`.
Using the default option `#{strict => true}`, the decoder will extract the custom
type spec, and validate the encoded data against it. If the custom code is missing,
the decoder aborts. Using `#{strict => false}`, the custom code is used if it exists,
but otherwise, it's ignored, and the encoded data is decoded using the dynamic type
info.
### Alternative types
The dynamic encoder supports two additions to the `gmserialization` template
language: `any`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`.
The dynamic encoder supports a few additions to the `gmserialization` template
language: `any`, `#{list => Type}`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`.
#### `any`
The `any` type doesn't have an associated code, but enforces dynamic encoding.
#### `list`
The original list type notation expects a key-value list, e.g.
`[{name, binary}, {age, int}]`
```erlang
EL = gmser_dyn:encode_typed([{name,binary},{age,int}], [{name,<<"Ulf">>},{age,29}]) ->
[<<0>>,<<1>>,[<<251>>,
[[<<253>>,[[<<255>>,<<110,97,109,101>>],[<<249>>,<<85,108,102>>]]],
[<<253>>,[[<<255>>,<<97,103,101>>],[<<248>>,<<29>>]]]]]]
```
Note that the encoding explicitly lays out a `[{Key, Value}]` structure, all
dynamically typed. This means it can be dynamically decoded without templates.
```erlang
gmser_dyn:decode(EL).
[{name,<<"Ulf">>},{age,29}]
```
In order to specify something like Erlang's `[integer()]` type, we can use
the following:
```erlang
gmser_dyn:encode_typed(#{list => int}, [1,2,3,4]) ->
[<<0>>,<<1>>,[<<251>>,
[[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>],[<<248>>,<<4>>]]]]
```
#### `alt`
The `#{alt => [Type]}` construct also enforces dynamic encoding, and will try

663
src/gmser_dyn.erl
View File

@ -29,6 +29,7 @@
, registered_types/1 %% -"-
, add_types/1 %% Returns updated types
, add_types/2 %% -"-
, add_types/3 %% -"-
, latest_vsn/0 %% -"-
, get_opts/1
, set_opts/1
@ -44,12 +45,65 @@
-import(gmserialization, [ decode_field/2 ]).
-type tag() :: atom().
-type code() :: pos_integer().
-type basic_type() :: anyint
| negint
| int
| binary
| bool
| list
| map
| tuple
| id
| label.
-type key() :: term().
-type alt_type() :: #{alt := template()}.
-type list_type() :: #{list := template()}.
-type switch_type() :: #{switch := [{any(), template()}]}.
-type tuple_type() :: tuple().
-type items_type() :: #{items := [{key(), template()} | {opt, key(), template()}]}.
-type fields_type() :: [{key(), template()}].
-type template() :: basic_type()
| alt_type()
| list_type()
| switch_type()
| tuple_type()
| items_type()
| fields_type().
-type types() :: #{ vsn := pos_integer()
, codes := #{code() => tag()}
, rev := #{tag() => code()}
, templates := #{tag() => template()}
, labels := #{atom() => code()}
, rev_labels := #{code() => atom()}
, options := #{atom() => any()}
}.
-export_type([ tag/0
, code/0
, basic_type/0
, key/0
, alt_type/0
, switch_type/0
, list_type/0
, tuple_type/0
, items_type/0
, fields_type/0
, template/0
, types/0
]).
-define(VSN, 1).
-include_lib("kernel/include/logger.hrl").
-ifdef(TEST).
-compile([export_all, nowarn_export_all]).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-endif.
@ -169,30 +223,80 @@ decode_typed(Type, Fields0, Vsn, Types0) ->
error({illegal_encoding, Other})
end.
decode_typed_(Type, [CodeBin, Fields], Vsn, Types) when is_map(Type) ->
decode_([[TemplateCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) ->
TemplateCode = decode_basic(int, TemplateCodeBin),
Code = decode_basic(int, CodeBin),
{Tag, Template} = template(Code, Vsn, Types),
Dyn = is_dyn_template(Tag),
case is_map_key(Code, Codes) of
true ->
Template = template(Code, Vsn, Types),
decode_from_template(Template, Code, Fld, Vsn, Types);
false ->
case option(strict, Types) of
true -> error({unknown_template, TemplateCode});
false ->
Template = template(Code, Vsn, Types),
decode_from_template(Template, Code, Fld, Vsn, Types)
end
end;
decode_([CodeBin, Flds], Vsn, Types) when is_binary(CodeBin) ->
Code = decode_basic(int, CodeBin),
Template = template(Code, Vsn, Types),
decode_from_template(Template, Code, Flds, Vsn, Types).
decode_typed_(Type, [[TCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) ->
TCode = decode_basic(int, TCodeBin),
Code = decode_basic(int, CodeBin),
case maps:find(TCode, Codes) of
{ok, Type} ->
TTemplate = template(TCode, Vsn, Types),
decode_from_template(TTemplate, Code, Fld, Vsn, Types);
{ok, TType} ->
Template = type_to_template(Type, Vsn, Types),
case is_subtype(TType, Template) of
true ->
decode_from_template(Template, Code, Fld, Vsn, Types);
false ->
TTemplate = template(TCode, Vsn, Types),
T1 = decode_from_template(TTemplate, Code, Fld, Vsn, Types),
T2 = decode_from_template(Template, Code, Fld, Vsn, Types),
if T1 =:= T2 -> T1;
true ->
error(badarg)
end
end;
error ->
case option(strict, Types) of
true ->
error(missing_template);
false ->
decode_typed_(Type, Code, Fld, Vsn, Types)
end
end;
decode_typed_(Type, [CodeBin, Fld], Vsn, Types) when is_binary(CodeBin) ->
Code = decode_basic(int, CodeBin),
decode_typed_(Type, Code, Fld, Vsn, Types).
decode_typed_(Type, Code, Fld, Vsn, Types) when is_map(Type) ->
Template = template(Code, Vsn, Types),
case {Type, Template} of
{#{items := _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{items := _}, items} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{alt := _}, _} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{switch:= _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{items := _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{items := _}, items} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{alt := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{switch:= _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{list := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types);
_ ->
error(badarg)
end;
decode_typed_(Type, [CodeBin, Fields], Vsn, Types) ->
Code = decode_basic(int, CodeBin),
{_, Template} = template(Type, Vsn, Types),
decode_typed_(Type, Code, Fld, Vsn, Types) when is_tuple(Type); is_list(Type) ->
decode_from_template(Type, Code, Fld, Vsn, Types);
decode_typed_(Type, Code, Fld, Vsn, Types) ->
Template = template(Type, Vsn, Types),
case template(Code, Vsn, Types) of
{Tag, Template} ->
Dyn = is_dyn_template(Tag),
decode_from_template(Template, Code, Fields, Dyn, Vsn, Types);
Template ->
decode_from_template(Template, Code, Fld, Vsn, Types);
_ ->
error(badarg)
end;
decode_typed_(_, _, _, _) ->
error(illegal_encoding).
end.
decode_tag_and_vsn([TagBin, VsnBin, Fields]) ->
{decode_basic(int, TagBin),
@ -227,52 +331,54 @@ assert_vsn(V, #{} = Types ) -> Types#{vsn => V}.
-define(ID, 254).
-define(LABEL, 255).
-define(CODES, #{ 246 => anyint
, 247 => negint
, 248 => int
, 249 => binary
, 250 => bool
, 251 => list
, 252 => map
, 253 => tuple
, 254 => id
, 255 => label}).
-define(REV, #{ anyint => ?ANYINT
, negint => ?NEGINT
, int => ?INT
, binary => ?BINARY
, bool => ?BOOL
, list => ?LIST
, map => ?MAP
, tuple => ?TUPLE
, id => ?ID
, label => ?LABEL }).
-define(TEMPLATES, #{ anyint => #{alt => [negint, int]}
, negint => negint
, int => int
, binary => binary
, bool => bool
, list => list
, map => map
, tuple => tuple
, id => id
, label => label
}).
-define(OPTIONS, #{ missing_labels => fail
, strict => true }).
dynamic_types() ->
#{ vsn => ?VSN
, codes =>
#{ 246 => anyint
, 247 => negint
, 248 => int
, 249 => binary
, 250 => bool
, 251 => list
, 252 => map
, 253 => tuple
, 254 => id
, 255 => label}
, rev =>
#{ anyint => ?ANYINT
, negint => ?NEGINT
, int => ?INT
, binary => ?BINARY
, bool => ?BOOL
, list => ?LIST
, map => ?MAP
, tuple => ?TUPLE
, id => ?ID
, label => ?LABEL }
, codes => ?CODES
, rev => ?REV
, labels => #{}
, rev_labels => #{}
, templates =>
#{ anyint => #{alt => [negint, int]}
, negint => negint
, int => int
, binary => binary
, bool => bool
, list => list
, map => map
, tuple => tuple
, id => id
, label => label
}
, options => #{}
, templates => ?TEMPLATES
, options => ?OPTIONS
}.
is_dyn_template(T) ->
is_dyn_template_(T, dynamic_types()).
is_custom_template(T) ->
not is_core_template(T).
is_dyn_template_(T, #{templates := Ts}) ->
is_map_key(T, Ts).
is_core_template(T) -> is_map_key(T, ?TEMPLATES).
registered_types() ->
registered_types(latest_vsn()).
@ -287,87 +393,123 @@ registered_types(Vsn) ->
dynamic_types()
end.
type_to_template(T, Vsn, Types) when is_atom(T); is_integer(T) ->
template(T, Vsn, Types);
type_to_template(Type, _Vsn, _Types) ->
Type.
is_subtype(T, T) -> true;
is_subtype(map, #{items := _}) -> true;
is_subtype(map, #{switch := _}) -> true;
is_subtype(tuple, T) when is_tuple(T) -> true;
is_subtype(list, L) when is_list(L) -> true;
is_subtype(_, #{alt := _}) -> true;
is_subtype(anyint, int) -> true;
is_subtype(_, _) ->
false.
template(any, _, _) -> any;
template(TagOrCode, Vsn, Types) ->
{Tag, Template} = get_template(TagOrCode, Types),
{Tag, dyn_template_(Template, Vsn)}.
Template = get_template(TagOrCode, Types),
dyn_template_(Template, Vsn).
get_template(Code, #{codes := Codes, templates := Ts}) when is_integer(Code) ->
Tag = maps:get(Code, Codes),
{Tag, maps:get(Tag, Ts)};
maps:get(Tag, Ts);
get_template(Tag, #{templates := Ts}) when is_atom(Tag) ->
{Tag, maps:get(Tag, Ts)}.
maps:get(Tag, Ts).
dyn_template_(F, Vsn) ->
if is_function(F, 0) -> F();
is_function(F, 1) -> F(Vsn);
true -> F
end.
dyn_template_(T, Vsn) ->
T1 = if is_function(T, 0) -> T();
is_function(T, 1) -> T(Vsn);
true -> T
end,
resolved_template(T1).
resolved_template(T) ->
if
is_map(T) -> T;
is_list(T) -> T;
is_tuple(T) -> T;
is_atom(T) ->
case is_core_template(T) of
true -> T;
false ->
error(unresolved_template)
end
end.
find_cached_label(Lbl, #{labels := Lbls}) ->
maps:find(Lbl, Lbls).
decode_([CodeBin, Flds], Vsn, Types) ->
Code = decode_basic(int, CodeBin),
{Tag, Template} = template(Code, Vsn, Types),
Dyn = is_dyn_template(Tag),
decode_from_template(Template, Code, Flds, Dyn, Vsn, Types).
encode_(Term, Vsn, Types) ->
encode_(Term, dyn(emit()), Vsn, Types).
encode_(Term, E, Vsn, Types) ->
{_Tag, Template} = auto_template(Term),
encode_from_template(Template, Term, E, Vsn, Types).
Template = auto_template(Term),
encode_from_template(Template, Term, Vsn, Types).
%% To control when to emit type codes:
%% If the template is predefined, it's 'not dynamic' (nodyn(E)).
%% If we are encoding against a type that's part of a predefined template,
%% we typically don't emit the type code, except at the very top.
%% So: emit type codes if the 'emit' bit is set, or if the 'dyn' bit is set.
emit() -> 2#01.
dyn() -> 2#10.
emit(E) -> E bor 2#01.
noemit(E) -> E band 2#10.
dyn(E) -> E bor 2#10.
nodyn(E) -> E band 2#01.
%% emit() -> 2#01.
%% dyn() -> 2#10.
%% emit(E) -> E bor 2#01.
%% noemit(E) -> E band 2#10.
%% dyn(E) -> E bor 2#10.
%% nodyn(E) -> E band 2#01.
encode_typed_(Type, Term, Vsn, #{codes := Codes, rev := Rev} = Types) ->
case (is_map_key(Type, Codes) orelse is_map_key(Type, Rev)) of
true ->
encode_typed_(Type, Term, nodyn(emit()), Vsn, Types);
encode_typed_1(Type, Term, Vsn, Types);
false ->
encode_maybe_template(Type, Term, Vsn, Types)
end.
encode_typed_(any, Term, _, Vsn, Types) ->
encode_(Term, dyn(emit()), Vsn, Types);
encode_typed_(Code, Term, E, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) ->
{_Tag, Template} = template(Code, Vsn, Types),
[encode_basic(int,Code),
encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)];
encode_typed_(Tag, Term, E, Vsn, #{templates := Ts, rev := Rev} = Types)
encode_typed_1(any, Term, Vsn, Types) ->
encode_(Term, Vsn, Types);
encode_typed_1(Code, Term, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) ->
Tag = maps:get(Code, Codes),
Template = template(Code, Vsn, Types),
Fld = encode_from_template(Template, Term, Vsn, Types),
case is_custom_template(Tag) of
true ->
[CodeI, FldI] = Fld,
[[encode_basic(int, Code), CodeI], FldI];
false ->
encode_from_template(Template, Term, Vsn, Types)
end;
encode_typed_1(Tag, Term, Vsn, #{templates := Ts, rev := Rev} = Types)
when is_map_key(Tag, Ts) ->
Template = dyn_template_(maps:get(Tag, Ts), Vsn),
Code = maps:get(Tag, Rev),
[encode_basic(int,Code),
encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)];
encode_typed_(MaybeTemplate, Term, _, Vsn, Types) ->
Fld = encode_from_template(Template, Term, Vsn, Types),
case is_custom_template(Tag) of
true ->
[CodeI, FldI] = Fld,
[[encode_basic(int,Code), CodeI], FldI];
false ->
Fld
end;
encode_typed_1(MaybeTemplate, Term, Vsn, Types) ->
encode_maybe_template(MaybeTemplate, Term, Vsn, Types).
encode_maybe_template(#{items := _} = Type, Term, Vsn, Types) ->
case is_map(Term) of
true ->
encode_from_template(Type, Term, emit(dyn()), Vsn, Types);
encode_from_template(Type, Term, Vsn, Types);
false ->
error({invalid, Type, Term})
end;
encode_maybe_template(#{alt := _} = Type, Term, Vsn, Types) ->
encode_from_template(Type, Term, Vsn, emit(dyn()), Types);
encode_from_template(Type, Term, Vsn, Types);
encode_maybe_template(#{switch := _} = Type, Term, Vsn, Types) ->
encode_from_template(Type, Term, Vsn, emit(dyn()), Types);
encode_from_template(Type, Term, Vsn, Types);
encode_maybe_template(#{list := _} = Type, Term, Vsn, Types) ->
encode_from_template(Type, Term, Vsn, Types);
encode_maybe_template(Pat, Term, Vsn, Types) when is_list(Pat);
is_tuple(Pat) ->
encode_from_template(Pat, Term, emit(dyn()), Vsn, Types);
encode_from_template(Pat, Term, Vsn, Types);
encode_maybe_template(Other, Term, _Vsn, _Types) ->
error({illegal_template, Other, Term}).
@ -378,67 +520,72 @@ auto_template({id,Tag,V}) when Tag == account
; Tag == channel
; Tag == associate_chain
; Tag == entry ->
if is_binary(V) -> {id, id};
if is_binary(V) -> id;
true ->
%% close, but no cigar
{tuple, tuple}
tuple
end;
auto_template(T) ->
if is_map(T) -> {map, map};
is_list(T) -> {list, list};
is_tuple(T) -> {tuple, tuple};
is_binary(T) -> {binary, binary};
is_boolean(T) -> {bool, bool};
is_atom(T) -> {label, label}; % binary_to_existing_atom()
if is_map(T) -> map;
is_list(T) -> list;
is_tuple(T) -> tuple;
is_binary(T) -> binary;
is_boolean(T) -> bool;
is_atom(T) -> label; % binary_to_existing_atom()
is_integer(T),
T >= 0 -> {int, int};
T >= 0 -> int;
is_integer(T),
T < 0 -> {negint, negint};
T < 0 -> negint;
true ->
error({invalid_type, T})
end.
decode_from_template(any, _, Fld, _, Vsn, Types) ->
decode_from_template(any, _Code, Fld, Vsn, Types) ->
decode_(Fld, Vsn, Types);
decode_from_template(#{items := Items}, _, Fld, _Dyn, Vsn, Types) when is_list(Fld) ->
decode_from_template(#{items := Items}, _, Fld, Vsn, Types) when is_list(Fld) ->
Zipped = dec_zip_items(Items, Fld, Vsn, Types),
lists:foldl(
fun({K, Type, Code, V}, Map) ->
maps:is_key(K, Map) andalso error(badarg, duplicate_key),
Map#{K => decode_from_template(Type, Code, V, true, Vsn, Types)}
fun({K, Type, V}, Map) ->
case maps:is_key(K, Map) of
true -> error(duplicate_key);
false ->
Map#{K => decode_typed_(Type, V, Vsn, Types)}
end
end, #{}, Zipped);
decode_from_template(#{alt := Alts} = T, Code, Fld, _, Vsn, Types) when is_list(Alts) ->
decode_from_template(#{alt := Alts} = T, Code, Fld, Vsn, Types) when is_list(Alts) ->
decode_alt(Alts, Code, Fld, T, Vsn, Types);
decode_from_template(#{switch := Alts} = T, Code, Fld, Dyn, Vsn, Types) when is_map(Alts) ->
decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types);
decode_from_template(list, _, Flds, _, Vsn, Types) ->
decode_from_template(#{switch := Alts} = T, Code, Fld, Vsn, Types) when is_map(Alts) ->
decode_switch(Alts, Code, Fld, T, Vsn, Types);
decode_from_template(#{list := Type}, ?LIST, Fld, Vsn, Types) ->
[decode_typed_(Type, F, Vsn, Types) || F <- Fld];
decode_from_template(list, _, Flds, Vsn, Types) ->
[decode_(F, Vsn, Types) || F <- Flds];
decode_from_template(map, _, Fld, _, Vsn, Types) ->
TupleFields = [F || F <- Fld],
Items = [decode_from_template({any,any}, ?TUPLE, T, true, Vsn, Types)
|| T <- TupleFields],
decode_from_template(map, ?MAP, TupleFields, Vsn, Types) ->
Items = lists:map(fun([Ke, Ve]) ->
{decode_(Ke, Vsn, Types), decode_(Ve, Vsn, Types)}
end, TupleFields),
maps:from_list(Items);
decode_from_template(tuple, _, Fld, _, Vsn, Types) ->
decode_from_template(tuple, _, Fld, Vsn, Types) ->
Items = [decode_(F, Vsn, Types) || F <- Fld],
list_to_tuple(Items);
decode_from_template([Type], _, Fields, _Dyn, Vsn, Types) ->
[decode_typed_(Type, F, Vsn, Types)
|| F <- Fields];
decode_from_template(Type, _, V, Dyn, Vsn, Types) when is_list(Type), is_list(V) ->
decode_fields(Type, V, Dyn, Vsn, Types);
decode_from_template(Type, _, V, _Dyn, Vsn, Types) when is_tuple(Type), is_list(V) ->
decode_from_template(Type, _, Fields, Vsn, Types) when is_list(Type) ->
decode_fields(Type, Fields, Vsn, Types);
%% Zipped = lists:zip(Type, Fields, fail),
%% [decode_typed_(T, F, Vsn, Types)
%% || {T, F} <- Zipped];
decode_from_template(Type, _, V, Vsn, Types) when is_tuple(Type) ->
Zipped = lists:zip(tuple_to_list(Type), V),
Items = [decode_typed_(T1, V1, Vsn, Types) || {T1, V1} <- Zipped],
list_to_tuple(Items);
decode_from_template(label, _, [C], _, _, #{rev_labels := RLbls}) ->
decode_from_template(label, ?LABEL, [C], _, #{rev_labels := RLbls}) ->
Code = decode_basic(int, C),
maps:get(Code, RLbls);
decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int
; Type == negint
; Type == binary
; Type == bool
; Type == id
; Type == label ->
decode_from_template(Type, Code, Fld, Vsn, Types) when Type == int
; Type == negint
; Type == binary
; Type == bool
; Type == id
; Type == label ->
case template(Code, Vsn, Types) of
Type ->
decode_basic(Type, Fld, Types);
@ -446,45 +593,44 @@ decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int
error(badarg)
end.
dec_zip_items([{K, T}|Is], [{K1, KEnc, VEnc}|Fs], Vsn, Types) ->
dec_zip_items([{K, T}|Is], [{K1, VEnc}|Fs], Vsn, Types) ->
if K =:= K1 ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
true ->
error(illegal_map)
end;
dec_zip_items([{K, T}|Is], [[KEnc, VEnc]|Fs], Vsn, Types) ->
case decode_(KEnc, Vsn, Types) of
K ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
_ ->
error(illegal_map)
end;
dec_zip_items([{opt, K, T}|Is], [{K, KEnc, VEnc}|Fs], Vsn, Types) ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
dec_zip_items([{opt, K, T}|Is], [{K, VEnc}|Fs], Vsn, Types) ->
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
dec_zip_items([{opt, K, T}|Is], [[KEnc,VEnc]|Fs], Vsn, Types) ->
case decode_(KEnc, Vsn, Types) of
K ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
OtherK ->
dec_zip_items(Is, [{OtherK, KEnc, VEnc}|Fs], Vsn, Types)
dec_zip_items(Is, [{OtherK, VEnc}|Fs], Vsn, Types)
end;
dec_zip_items([], [], _, _) ->
[].
encode_from_template(any, V, _E, Vsn, Types) ->
encode_(V, dyn(emit()), Vsn, Types);
encode_from_template(list, L, E, Vsn, Types) when is_list(L) ->
encode_from_template(any, V, Vsn, Types) ->
encode_(V, Vsn, Types);
encode_from_template(list, L, Vsn, Types) when is_list(L) ->
assert_type(is_list(L), list, L),
emit(E, list, Types,
emit(list, Types,
[encode_(V, Vsn, Types) || V <- L]);
encode_from_template(#{items := Items}, M, E, Vsn, Types) ->
encode_from_template(#{items := Items}, M, Vsn, Types) ->
assert_type(is_map(M), map, M),
Emit = noemit(E),
Encode = fun(K, Type, V) ->
[encode_from_template(any, K, Emit, Vsn, Types),
encode_from_template(Type, V, Emit, Vsn, Types)]
[encode_from_template(any, K, Vsn, Types),
encode_from_template(Type, V, Vsn, Types)]
end,
emit(E, map, Types,
emit(map, Types,
lists:foldr(
fun({K, Type}, Acc) ->
V = maps:get(K, M),
@ -497,129 +643,150 @@ encode_from_template(#{items := Items}, M, E, Vsn, Types) ->
Acc
end
end, [], Items));
encode_from_template(#{alt := Alts} = T, Term, E, Vsn, Types) when is_list(Alts) ->
encode_alt(Alts, Term, T, E, Vsn, Types);
encode_from_template(#{switch := Alts} = T, Term, E, Vsn, Types) when is_map(Alts),
is_map(Term) ->
encode_switch(Alts, Term, T, E, Vsn, Types);
encode_from_template(map, M, E, Vsn, Types) ->
encode_from_template(#{alt := Alts} = T, Term, Vsn, Types) when is_list(Alts) ->
encode_alt(Alts, Term, T, Vsn, Types);
encode_from_template(#{switch := Alts} = T, Term, Vsn, Types) when is_map(Alts),
is_map(Term) ->
encode_switch(Alts, Term, T, Vsn, Types);
encode_from_template(map, M, Vsn, Types) ->
assert_type(is_map(M), map, M),
Emit = emit(E),
emit(E, map, Types,
[[encode_from_template(any, K, Emit, Vsn, Types),
encode_from_template(any, V, Emit, Vsn, Types)]
emit(map, Types,
[[encode_from_template(any, K, Vsn, Types),
encode_from_template(any, V, Vsn, Types)]
|| {K, V} <- lists:sort(maps:to_list(M))]);
encode_from_template(tuple, T, E, Vsn, Types) ->
encode_from_template(tuple, T, Vsn, Types) ->
assert_type(is_tuple(T), tuple, T),
emit(E, tuple, Types,
[encode_(V, noemit(E), Vsn, Types) || V <- tuple_to_list(T)]);
encode_from_template(T, V, E, Vsn, Types) when is_tuple(T) ->
emit(tuple, Types,
[encode_(V, Vsn, Types) || V <- tuple_to_list(T)]);
encode_from_template(T, V, Vsn, Types) when is_tuple(T) ->
assert_type(is_tuple(V), T, V),
assert_type(tuple_size(T) =:= tuple_size(V), T, V),
Zipped = lists:zip(tuple_to_list(T), tuple_to_list(V)),
emit(E, tuple, Types,
[encode_from_template(T1, V1, noemit(E), Vsn, Types) || {T1, V1} <- Zipped]);
encode_from_template([Type] = T, List, E, Vsn, Types) ->
emit(tuple, Types,
[encode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped]);
encode_from_template(#{list := Type} = T, List, Vsn, Types) ->
assert_type(is_list(List), T, List),
emit(E, list, Types,
[encode_from_template(Type, V, noemit(E), Vsn, Types) || V <- List]);
encode_from_template(Type, List, E, Vsn, Types) when is_list(Type), is_list(List) ->
encode_fields(Type, List, E, Vsn, Types);
encode_from_template(label, V, E, _, Types) ->
emit(list, Types,
[encode_from_template(Type, V, Vsn, Types) || V <- List]);
encode_from_template(Type, List, Vsn, Types) when is_list(Type), is_list(List) ->
emit(list, Types, encode_fields(Type, List, Vsn, Types));
encode_from_template(label, V, _, Types) ->
assert_type(is_atom(V), label, V),
case find_cached_label(V, Types) of
error ->
encode_basic(label, V, E, Types);
encode_basic(label, V, Types);
{ok, Code} when is_integer(Code) ->
emit(E, label, Types,
emit(label, Types,
[encode_basic(int, Code)])
end;
encode_from_template(Type, V, E, _, Types) when Type == id
; Type == binary
; Type == bool
; Type == int
; Type == negint
; Type == label ->
encode_basic(Type, V, E, Types);
encode_from_template(Type, V, E, Vsn, Types) ->
encode_typed_(Type, V, E, Vsn, Types).
encode_from_template(Type, V, _, Types) when Type == id
; Type == binary
; Type == bool
; Type == int
; Type == negint
; Type == label ->
encode_basic(Type, V, Types);
encode_from_template(Type, V, Vsn, Types) ->
encode_typed_(Type, V, Vsn, Types).
assert_type(true, _, _) -> ok;
assert_type(_, Type, V) -> error({illegal, Type, V}).
decode_alt([A|Alts], Code, Fld, T, Vsn, Types) ->
try decode_from_template(A, Code, Fld, true, Vsn, Types)
try decode_typed_(A, Code, Fld, Vsn, Types)
catch error:_ ->
decode_alt(Alts, Code, Fld, T, Vsn, Types)
end;
decode_alt([], _Code, Fld, T, _, _) ->
error({illegal, T, Fld}).
encode_alt(Alts, Term, T, E, Vsn, Types) ->
encode_alt(Alts, Term, T, Vsn, Types) ->
%% Since we don't know which type may match, treat as dynamic.
encode_alt_(Alts, Term, T, dyn(E), Vsn, Types).
encode_alt_(Alts, Term, T, Vsn, Types).
encode_alt_([A|Alts], Term, T, E, Vsn, Types) ->
try encode_from_template(A, Term, E, Vsn, Types)
encode_alt_([A|Alts], Term, T, Vsn, Types) ->
try encode_from_template(A, Term, Vsn, Types)
catch error:_ ->
encode_alt_(Alts, Term, T, E, Vsn, Types)
encode_alt_(Alts, Term, T, Vsn, Types)
end;
encode_alt_([], Term, T, _, _, _) ->
encode_alt_([], Term, T, _, _) ->
error({illegal, T, Term}).
decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types) ->
[[KFld, VFld]] = Fld,
Key = decode_(KFld, Vsn, Types),
case maps:find(Key, Alts) of
{ok, SubType} ->
SubTerm = decode_from_template(SubType, Code, VFld, Dyn, Vsn, Types),
#{Key => SubTerm};
error ->
decode_switch(Alts, Code, Fld, T, Vsn, Types) ->
case is_map_type(Code, Vsn, Types) of
true ->
case Fld of
[[KFld, VFld]] ->
Key = decode_(KFld, Vsn, Types),
case maps:find(Key, Alts) of
{ok, SubType} ->
SubTerm = decode_typed_(SubType, VFld, Vsn, Types),
#{Key => SubTerm};
error ->
error({illegal, T, Fld})
end;
_ ->
error({illegal, T, Fld})
end;
false ->
error({illegal, T, Fld})
end.
encode_switch(Alts, Term, T, E, Vsn, Types) ->
is_map_type(Code, Vsn, Types) ->
case template(Code, Vsn, Types) of
map -> true;
#{items := _} -> true;
#{switch := _} -> true;
T ->
case maps:get(T, maps:get(templates, Types)) of
map -> true;
T -> false;
Other when is_atom(Other) ->
is_map_type(T, Vsn, Types);
_ -> false
end
end.
encode_switch(Alts, Term, T, Vsn, Types) ->
assert_type(map_size(Term) == 1, singleton_map, Term),
[{Key, Subterm}] = maps:to_list(Term),
case maps:find(Key, Alts) of
{ok, SubType} ->
Enc = encode_from_template(SubType, Subterm, E, Vsn, Types),
emit(E, map, Types,
[[encode_from_template(any, Key, E, Vsn, Types),
Enc]]);
Enc = encode_from_template(SubType, Subterm, Vsn, Types),
emit(map, Types,
[[encode_from_template(any, Key, Vsn, Types), Enc]]);
error ->
error({illegal, T, Term})
end.
%% Basically, dynamically encoding a statically defined object
encode_fields([{Field, Type}|TypesLeft],
[{Field, Val}|FieldsLeft], E, Vsn, Types) ->
[ encode_from_template(Type, Val, E, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)];
[{Field, Val}|FieldsLeft], Vsn, Types) ->
KType = auto_template(Field),
[ encode_from_template({KType, Type}, {Field, Val}, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
encode_fields([{_Field, _Type} = FT|_TypesLeft],
[Val |_FieldsLeft], _E, _Vsn, _Types) ->
[Val |_FieldsLeft], _Vsn, _Types) ->
error({illegal_field, FT, Val});
encode_fields([Type|TypesLeft],
[Val |FieldsLeft], E, Vsn, Types) when is_atom(Type) ->
[Val |FieldsLeft], Vsn, Types) when is_atom(Type) ->
%% Not sure about this ...
[ encode_from_template(Type, Val, E, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)];
encode_fields([], [], _, _, _) ->
[ encode_from_template(Type, Val, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
encode_fields([], [], _, _) ->
[].
decode_fields([{Tag, Type}|TypesLeft],
[Field |FieldsLeft], Dyn, Vsn, Types) ->
[Field |FieldsLeft], Vsn, Types) ->
[ {Tag, decode_from_template(Type, 0, Field, Dyn, Vsn, Types)}
| decode_fields(TypesLeft, FieldsLeft, Dyn, Vsn, Types)];
decode_fields([], [], _, _, _) ->
[ {Tag, decode_from_template(Type, 0, Field, Vsn, Types)}
| decode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
decode_fields([], [], _, _) ->
[].
emit(E, Tag, Types, Enc) when E > 0 ->
[emit_code(Tag, Types), Enc];
emit(0, _, _, Enc) ->
Enc.
emit(Tag, Types, Enc) ->
[emit_code(Tag, Types), Enc].
emit_code(Tag, #{rev := Tags}) ->
encode_basic(int, maps:get(Tag, Tags)).
@ -656,12 +823,10 @@ decode_basic(negint, Fld) ->
decode_basic(Type, Fld) ->
gmserialization:decode_field(Type, Fld).
encode_basic(negint, I, _, Types) when is_integer(I), I < 0 ->
encode_basic(negint, I, Types) when is_integer(I), I < 0 ->
[emit_code(negint, Types), gmserialization:encode_field(int, -I)];
encode_basic(Tag, V, E, Types) when E > 0 ->
[emit_code(Tag, Types), encode_basic(Tag, V)];
encode_basic(Tag, V, 0, _) ->
encode_basic(Tag, V).
encode_basic(Tag, V, Types) ->
[emit_code(Tag, Types), encode_basic(Tag, V)].
encode_basic(label, A) when is_atom(A) ->
encode_basic(binary, atom_to_binary(A, utf8));
@ -681,28 +846,34 @@ register_types(Types) when is_map(Types) ->
register_types(latest_vsn(), Types).
register_types(Vsn, Types) ->
Result = add_types(Types),
Result = add_types(Vsn, Types),
put_types(Vsn, Result).
add_types(Types) ->
add_types(Types, dynamic_types()).
add_types(?VSN, Types).
add_types(Types, PrevTypes) ->
add_types(Vsn, Types) ->
add_types(Vsn, Types, dynamic_types()).
add_types(Vsn, Types, PrevTypes) ->
Codes = maps:get(codes, Types, #{}),
Rev = rev_codes(Codes),
Templates = maps:get(templates, Types, #{}),
Labels = maps:get(labels, Types, #{}),
RevLabels = rev_codes(Labels),
Options = maps:get(options, Types, #{}),
#{codes := Codes0, rev := Rev0, labels := Labels0,
#{codes := Codes0, rev := Rev0, labels := Labels0, rev_labels := RevLabels0,
templates := Templates0, options := Options0} = PrevTypes,
Merged = #{ codes => maps:merge(Codes0, Codes)
, rev => maps:merge(Rev0, Rev)
, templates => maps:merge(Templates0, Templates)
, options => maps:merge(Options0, Options)
, labels => maps:merge(Labels0, Labels) },
, labels => maps:merge(Labels0, Labels)
, rev_labels => maps:merge(RevLabels0, RevLabels) },
assert_sizes(Merged),
assert_mappings(Merged),
assert_label_cache(Merged).
assert_label_cache(Merged),
assert_resolved_templates(Vsn, Merged).
latest_vsn() ->
case persistent_term:get(pt_key(), undefined) of
@ -762,6 +933,18 @@ set_opts(Opts) ->
set_opts(Opts, Types) ->
Types#{options => Opts}.
option(O, #{options := Opts}) ->
case Opts of
#{O := V} -> V;
_ -> default_option(O)
end.
default_option(O) ->
case dynamic_types() of
#{options := #{O := V}} -> V;
_ -> undefined
end.
get_opts(#{options := Opts}) ->
Opts.
@ -821,6 +1004,10 @@ assert_mappings(#{rev := Rev, templates := Ts} = Types) ->
error({missing_templates, Missing, Types})
end.
assert_resolved_templates(Vsn, #{templates := Ts} = Types) ->
_ = [template(T, Vsn, Types) || T <- maps:keys(Ts)],
Types.
assert_label_cache(#{labels := Labels} = Types) ->
Ls = maps:keys(Labels),
case [L || L <- Ls, not is_atom(L)] of
@ -901,6 +1088,7 @@ user_types_test_() ->
dynamic_types_test_() ->
[ ?_test(revert_to_default_types())
, ?_test(t_typed_map())
, ?_test(t_typed_list())
, ?_test(t_alts())
, ?_test(t_switch())
, ?_test(t_anyints())
@ -985,7 +1173,7 @@ t_bad_typed_encode(Type, Term, Error) ->
end.
t_reg_chain_objects_array() ->
Template = [{foo, {int, binary}}, {bar, [{int, int}]}, {baz, {int}}],
Template = [{foo, {int, binary}}, {bar, #{list => {int, int}}}, {baz, {int}}],
?debugFmt("Template = ~p", [Template]),
MyTypes = #{ codes => #{ 1002 => coa }
, templates => #{ coa => Template } },
@ -1042,7 +1230,7 @@ t_reg_label_cache2() ->
register_types(TFromL),
Tup = {'1', '1'},
Enc = gmser_dyn:encode_typed(lbl_tup2, Tup),
[<<0>>,<<1>>,[<<3,235>>,[[<<49>>],[<<49>>]]]] = Enc,
[<<0>>,<<1>>,[[<<3,235>>,<<?TUPLE>>],[[<<?LABEL>>,[<<49>>]],[<<?LABEL>>,[<<49>>]]]]] = Enc,
_Tup = gmser_dyn:decode(Enc).
t_reg_map() ->
@ -1053,11 +1241,7 @@ t_reg_map() ->
},
register_types(Types),
Enc0 = gmser_dyn:encode_typed(my_map, #{a => foo, b => 17}),
dbg:tracer(),
dbg:tpl(?MODULE,x),
dbg:p(all,[c]),
#{a := foo, b := 17} = try gmser_dyn:decode(Enc0)
after dbg:stop() end,
#{a := foo, b := 17} = gmser_dyn:decode(Enc0),
ok.
t_reg_options() ->
@ -1080,6 +1264,11 @@ t_typed_map() ->
?assertEqual(Term1, decode(Enc1)),
?assertEqual(Enc, encode_typed(#{items => Items}, Term1)).
t_typed_list() ->
Term = [1,2,3,4],
encode_typed(#{list => int}, Term),
ok.
t_alts() ->
t_round_trip_typed(#{alt => [negint, int]}, -4),
t_round_trip_typed(#{alt => [negint, int]}, 4),

11
src/gmser_dyn_types.erl
View File

@ -1,6 +1,7 @@
-module(gmser_dyn_types).
-export([ add_type/4
-export([ add_type/3 %% (Tag, Code, Template) -> Types1
, add_type/4 %% (Tag, Code, Template, Types) -> Types1
, from_list/2
, expand/1 ]).
-export([ next_code/1 ]).
@ -8,6 +9,14 @@
next_code(#{codes := Codes}) ->
lists:max(maps:keys(Codes)) + 1.
-spec add_type(Tag, Code, Template) -> Types
when Tag :: gmser_dyn:tag()
, Code :: gmser_dyn:code()
, Template :: gmser_dyn:template()
, Types :: gmser_dyn:types().
add_type(Tag, Code, Template) ->
add_type(Tag, Code, Template, gmser_dyn:registered_types()).
add_type(Tag, Code, Template, Types) ->
elem_to_type({Tag, Code, Template}, Types).