Fix OTP 28 dialyzer warnings, rewrite gmser_dyn decoder
Gajumaru Serialization Tests / tests (push) Successful in -3m56s

gmser_dyn no longer tries to compress output by omitting type tags.
Decoding streams using custom template codes can either use 'strict'
decoding, in which case matching templates must be registered on the
decoding end; in `strict => false` mode, the stream can still be decoded
without valudation if the custom template is missing.
This commit is contained in:
Ulf Wiger
2026-02-18 21:10:43 +01:00
parent 8734e67c66
commit 00699b08b7
3 changed files with 491 additions and 262 deletions
+426 -237
View File
@@ -29,6 +29,7 @@
, registered_types/1 %% -"-
, add_types/1 %% Returns updated types
, add_types/2 %% -"-
, add_types/3 %% -"-
, latest_vsn/0 %% -"-
, get_opts/1
, set_opts/1
@@ -44,12 +45,65 @@
-import(gmserialization, [ decode_field/2 ]).
-type tag() :: atom().
-type code() :: pos_integer().
-type basic_type() :: anyint
| negint
| int
| binary
| bool
| list
| map
| tuple
| id
| label.
-type key() :: term().
-type alt_type() :: #{alt := template()}.
-type list_type() :: #{list := template()}.
-type switch_type() :: #{switch := [{any(), template()}]}.
-type tuple_type() :: tuple().
-type items_type() :: #{items := [{key(), template()} | {opt, key(), template()}]}.
-type fields_type() :: [{key(), template()}].
-type template() :: basic_type()
| alt_type()
| list_type()
| switch_type()
| tuple_type()
| items_type()
| fields_type().
-type types() :: #{ vsn := pos_integer()
, codes := #{code() => tag()}
, rev := #{tag() => code()}
, templates := #{tag() => template()}
, labels := #{atom() => code()}
, rev_labels := #{code() => atom()}
, options := #{atom() => any()}
}.
-export_type([ tag/0
, code/0
, basic_type/0
, key/0
, alt_type/0
, switch_type/0
, list_type/0
, tuple_type/0
, items_type/0
, fields_type/0
, template/0
, types/0
]).
-define(VSN, 1).
-include_lib("kernel/include/logger.hrl").
-ifdef(TEST).
-compile([export_all, nowarn_export_all]).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-endif.
@@ -169,30 +223,80 @@ decode_typed(Type, Fields0, Vsn, Types0) ->
error({illegal_encoding, Other})
end.
decode_typed_(Type, [CodeBin, Fields], Vsn, Types) when is_map(Type) ->
decode_([[TemplateCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) ->
TemplateCode = decode_basic(int, TemplateCodeBin),
Code = decode_basic(int, CodeBin),
{Tag, Template} = template(Code, Vsn, Types),
Dyn = is_dyn_template(Tag),
case is_map_key(Code, Codes) of
true ->
Template = template(Code, Vsn, Types),
decode_from_template(Template, Code, Fld, Vsn, Types);
false ->
case option(strict, Types) of
true -> error({unknown_template, TemplateCode});
false ->
Template = template(Code, Vsn, Types),
decode_from_template(Template, Code, Fld, Vsn, Types)
end
end;
decode_([CodeBin, Flds], Vsn, Types) when is_binary(CodeBin) ->
Code = decode_basic(int, CodeBin),
Template = template(Code, Vsn, Types),
decode_from_template(Template, Code, Flds, Vsn, Types).
decode_typed_(Type, [[TCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) ->
TCode = decode_basic(int, TCodeBin),
Code = decode_basic(int, CodeBin),
case maps:find(TCode, Codes) of
{ok, Type} ->
TTemplate = template(TCode, Vsn, Types),
decode_from_template(TTemplate, Code, Fld, Vsn, Types);
{ok, TType} ->
Template = type_to_template(Type, Vsn, Types),
case is_subtype(TType, Template) of
true ->
decode_from_template(Template, Code, Fld, Vsn, Types);
false ->
TTemplate = template(TCode, Vsn, Types),
T1 = decode_from_template(TTemplate, Code, Fld, Vsn, Types),
T2 = decode_from_template(Template, Code, Fld, Vsn, Types),
if T1 =:= T2 -> T1;
true ->
error(badarg)
end
end;
error ->
case option(strict, Types) of
true ->
error(missing_template);
false ->
decode_typed_(Type, Code, Fld, Vsn, Types)
end
end;
decode_typed_(Type, [CodeBin, Fld], Vsn, Types) when is_binary(CodeBin) ->
Code = decode_basic(int, CodeBin),
decode_typed_(Type, Code, Fld, Vsn, Types).
decode_typed_(Type, Code, Fld, Vsn, Types) when is_map(Type) ->
Template = template(Code, Vsn, Types),
case {Type, Template} of
{#{items := _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{items := _}, items} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{alt := _}, _} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{switch:= _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types);
{#{items := _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{items := _}, items} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{alt := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{switch:= _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types);
{#{list := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types);
_ ->
error(badarg)
end;
decode_typed_(Type, [CodeBin, Fields], Vsn, Types) ->
Code = decode_basic(int, CodeBin),
{_, Template} = template(Type, Vsn, Types),
decode_typed_(Type, Code, Fld, Vsn, Types) when is_tuple(Type); is_list(Type) ->
decode_from_template(Type, Code, Fld, Vsn, Types);
decode_typed_(Type, Code, Fld, Vsn, Types) ->
Template = template(Type, Vsn, Types),
case template(Code, Vsn, Types) of
{Tag, Template} ->
Dyn = is_dyn_template(Tag),
decode_from_template(Template, Code, Fields, Dyn, Vsn, Types);
Template ->
decode_from_template(Template, Code, Fld, Vsn, Types);
_ ->
error(badarg)
end;
decode_typed_(_, _, _, _) ->
error(illegal_encoding).
end.
decode_tag_and_vsn([TagBin, VsnBin, Fields]) ->
{decode_basic(int, TagBin),
@@ -227,52 +331,54 @@ assert_vsn(V, #{} = Types ) -> Types#{vsn => V}.
-define(ID, 254).
-define(LABEL, 255).
-define(CODES, #{ 246 => anyint
, 247 => negint
, 248 => int
, 249 => binary
, 250 => bool
, 251 => list
, 252 => map
, 253 => tuple
, 254 => id
, 255 => label}).
-define(REV, #{ anyint => ?ANYINT
, negint => ?NEGINT
, int => ?INT
, binary => ?BINARY
, bool => ?BOOL
, list => ?LIST
, map => ?MAP
, tuple => ?TUPLE
, id => ?ID
, label => ?LABEL }).
-define(TEMPLATES, #{ anyint => #{alt => [negint, int]}
, negint => negint
, int => int
, binary => binary
, bool => bool
, list => list
, map => map
, tuple => tuple
, id => id
, label => label
}).
-define(OPTIONS, #{ missing_labels => fail
, strict => true }).
dynamic_types() ->
#{ vsn => ?VSN
, codes =>
#{ 246 => anyint
, 247 => negint
, 248 => int
, 249 => binary
, 250 => bool
, 251 => list
, 252 => map
, 253 => tuple
, 254 => id
, 255 => label}
, rev =>
#{ anyint => ?ANYINT
, negint => ?NEGINT
, int => ?INT
, binary => ?BINARY
, bool => ?BOOL
, list => ?LIST
, map => ?MAP
, tuple => ?TUPLE
, id => ?ID
, label => ?LABEL }
, codes => ?CODES
, rev => ?REV
, labels => #{}
, rev_labels => #{}
, templates =>
#{ anyint => #{alt => [negint, int]}
, negint => negint
, int => int
, binary => binary
, bool => bool
, list => list
, map => map
, tuple => tuple
, id => id
, label => label
}
, options => #{}
, templates => ?TEMPLATES
, options => ?OPTIONS
}.
is_dyn_template(T) ->
is_dyn_template_(T, dynamic_types()).
is_custom_template(T) ->
not is_core_template(T).
is_dyn_template_(T, #{templates := Ts}) ->
is_map_key(T, Ts).
is_core_template(T) -> is_map_key(T, ?TEMPLATES).
registered_types() ->
registered_types(latest_vsn()).
@@ -287,87 +393,123 @@ registered_types(Vsn) ->
dynamic_types()
end.
type_to_template(T, Vsn, Types) when is_atom(T); is_integer(T) ->
template(T, Vsn, Types);
type_to_template(Type, _Vsn, _Types) ->
Type.
is_subtype(T, T) -> true;
is_subtype(map, #{items := _}) -> true;
is_subtype(map, #{switch := _}) -> true;
is_subtype(tuple, T) when is_tuple(T) -> true;
is_subtype(list, L) when is_list(L) -> true;
is_subtype(_, #{alt := _}) -> true;
is_subtype(anyint, int) -> true;
is_subtype(_, _) ->
false.
template(any, _, _) -> any;
template(TagOrCode, Vsn, Types) ->
{Tag, Template} = get_template(TagOrCode, Types),
{Tag, dyn_template_(Template, Vsn)}.
Template = get_template(TagOrCode, Types),
dyn_template_(Template, Vsn).
get_template(Code, #{codes := Codes, templates := Ts}) when is_integer(Code) ->
Tag = maps:get(Code, Codes),
{Tag, maps:get(Tag, Ts)};
maps:get(Tag, Ts);
get_template(Tag, #{templates := Ts}) when is_atom(Tag) ->
{Tag, maps:get(Tag, Ts)}.
maps:get(Tag, Ts).
dyn_template_(F, Vsn) ->
if is_function(F, 0) -> F();
is_function(F, 1) -> F(Vsn);
true -> F
end.
dyn_template_(T, Vsn) ->
T1 = if is_function(T, 0) -> T();
is_function(T, 1) -> T(Vsn);
true -> T
end,
resolved_template(T1).
resolved_template(T) ->
if
is_map(T) -> T;
is_list(T) -> T;
is_tuple(T) -> T;
is_atom(T) ->
case is_core_template(T) of
true -> T;
false ->
error(unresolved_template)
end
end.
find_cached_label(Lbl, #{labels := Lbls}) ->
maps:find(Lbl, Lbls).
decode_([CodeBin, Flds], Vsn, Types) ->
Code = decode_basic(int, CodeBin),
{Tag, Template} = template(Code, Vsn, Types),
Dyn = is_dyn_template(Tag),
decode_from_template(Template, Code, Flds, Dyn, Vsn, Types).
encode_(Term, Vsn, Types) ->
encode_(Term, dyn(emit()), Vsn, Types).
encode_(Term, E, Vsn, Types) ->
{_Tag, Template} = auto_template(Term),
encode_from_template(Template, Term, E, Vsn, Types).
Template = auto_template(Term),
encode_from_template(Template, Term, Vsn, Types).
%% To control when to emit type codes:
%% If the template is predefined, it's 'not dynamic' (nodyn(E)).
%% If we are encoding against a type that's part of a predefined template,
%% we typically don't emit the type code, except at the very top.
%% So: emit type codes if the 'emit' bit is set, or if the 'dyn' bit is set.
emit() -> 2#01.
dyn() -> 2#10.
emit(E) -> E bor 2#01.
noemit(E) -> E band 2#10.
dyn(E) -> E bor 2#10.
nodyn(E) -> E band 2#01.
%% emit() -> 2#01.
%% dyn() -> 2#10.
%% emit(E) -> E bor 2#01.
%% noemit(E) -> E band 2#10.
%% dyn(E) -> E bor 2#10.
%% nodyn(E) -> E band 2#01.
encode_typed_(Type, Term, Vsn, #{codes := Codes, rev := Rev} = Types) ->
case (is_map_key(Type, Codes) orelse is_map_key(Type, Rev)) of
true ->
encode_typed_(Type, Term, nodyn(emit()), Vsn, Types);
encode_typed_1(Type, Term, Vsn, Types);
false ->
encode_maybe_template(Type, Term, Vsn, Types)
end.
encode_typed_(any, Term, _, Vsn, Types) ->
encode_(Term, dyn(emit()), Vsn, Types);
encode_typed_(Code, Term, E, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) ->
{_Tag, Template} = template(Code, Vsn, Types),
[encode_basic(int,Code),
encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)];
encode_typed_(Tag, Term, E, Vsn, #{templates := Ts, rev := Rev} = Types)
encode_typed_1(any, Term, Vsn, Types) ->
encode_(Term, Vsn, Types);
encode_typed_1(Code, Term, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) ->
Tag = maps:get(Code, Codes),
Template = template(Code, Vsn, Types),
Fld = encode_from_template(Template, Term, Vsn, Types),
case is_custom_template(Tag) of
true ->
[CodeI, FldI] = Fld,
[[encode_basic(int, Code), CodeI], FldI];
false ->
encode_from_template(Template, Term, Vsn, Types)
end;
encode_typed_1(Tag, Term, Vsn, #{templates := Ts, rev := Rev} = Types)
when is_map_key(Tag, Ts) ->
Template = dyn_template_(maps:get(Tag, Ts), Vsn),
Code = maps:get(Tag, Rev),
[encode_basic(int,Code),
encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)];
encode_typed_(MaybeTemplate, Term, _, Vsn, Types) ->
Fld = encode_from_template(Template, Term, Vsn, Types),
case is_custom_template(Tag) of
true ->
[CodeI, FldI] = Fld,
[[encode_basic(int,Code), CodeI], FldI];
false ->
Fld
end;
encode_typed_1(MaybeTemplate, Term, Vsn, Types) ->
encode_maybe_template(MaybeTemplate, Term, Vsn, Types).
encode_maybe_template(#{items := _} = Type, Term, Vsn, Types) ->
case is_map(Term) of
true ->
encode_from_template(Type, Term, emit(dyn()), Vsn, Types);
encode_from_template(Type, Term, Vsn, Types);
false ->
error({invalid, Type, Term})
end;
encode_maybe_template(#{alt := _} = Type, Term, Vsn, Types) ->
encode_from_template(Type, Term, Vsn, emit(dyn()), Types);
encode_from_template(Type, Term, Vsn, Types);
encode_maybe_template(#{switch := _} = Type, Term, Vsn, Types) ->
encode_from_template(Type, Term, Vsn, emit(dyn()), Types);
encode_from_template(Type, Term, Vsn, Types);
encode_maybe_template(#{list := _} = Type, Term, Vsn, Types) ->
encode_from_template(Type, Term, Vsn, Types);
encode_maybe_template(Pat, Term, Vsn, Types) when is_list(Pat);
is_tuple(Pat) ->
encode_from_template(Pat, Term, emit(dyn()), Vsn, Types);
encode_from_template(Pat, Term, Vsn, Types);
encode_maybe_template(Other, Term, _Vsn, _Types) ->
error({illegal_template, Other, Term}).
@@ -378,67 +520,72 @@ auto_template({id,Tag,V}) when Tag == account
; Tag == channel
; Tag == associate_chain
; Tag == entry ->
if is_binary(V) -> {id, id};
if is_binary(V) -> id;
true ->
%% close, but no cigar
{tuple, tuple}
tuple
end;
auto_template(T) ->
if is_map(T) -> {map, map};
is_list(T) -> {list, list};
is_tuple(T) -> {tuple, tuple};
is_binary(T) -> {binary, binary};
is_boolean(T) -> {bool, bool};
is_atom(T) -> {label, label}; % binary_to_existing_atom()
if is_map(T) -> map;
is_list(T) -> list;
is_tuple(T) -> tuple;
is_binary(T) -> binary;
is_boolean(T) -> bool;
is_atom(T) -> label; % binary_to_existing_atom()
is_integer(T),
T >= 0 -> {int, int};
T >= 0 -> int;
is_integer(T),
T < 0 -> {negint, negint};
T < 0 -> negint;
true ->
error({invalid_type, T})
end.
decode_from_template(any, _, Fld, _, Vsn, Types) ->
decode_from_template(any, _Code, Fld, Vsn, Types) ->
decode_(Fld, Vsn, Types);
decode_from_template(#{items := Items}, _, Fld, _Dyn, Vsn, Types) when is_list(Fld) ->
decode_from_template(#{items := Items}, _, Fld, Vsn, Types) when is_list(Fld) ->
Zipped = dec_zip_items(Items, Fld, Vsn, Types),
lists:foldl(
fun({K, Type, Code, V}, Map) ->
maps:is_key(K, Map) andalso error(badarg, duplicate_key),
Map#{K => decode_from_template(Type, Code, V, true, Vsn, Types)}
fun({K, Type, V}, Map) ->
case maps:is_key(K, Map) of
true -> error(duplicate_key);
false ->
Map#{K => decode_typed_(Type, V, Vsn, Types)}
end
end, #{}, Zipped);
decode_from_template(#{alt := Alts} = T, Code, Fld, _, Vsn, Types) when is_list(Alts) ->
decode_from_template(#{alt := Alts} = T, Code, Fld, Vsn, Types) when is_list(Alts) ->
decode_alt(Alts, Code, Fld, T, Vsn, Types);
decode_from_template(#{switch := Alts} = T, Code, Fld, Dyn, Vsn, Types) when is_map(Alts) ->
decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types);
decode_from_template(list, _, Flds, _, Vsn, Types) ->
decode_from_template(#{switch := Alts} = T, Code, Fld, Vsn, Types) when is_map(Alts) ->
decode_switch(Alts, Code, Fld, T, Vsn, Types);
decode_from_template(#{list := Type}, ?LIST, Fld, Vsn, Types) ->
[decode_typed_(Type, F, Vsn, Types) || F <- Fld];
decode_from_template(list, _, Flds, Vsn, Types) ->
[decode_(F, Vsn, Types) || F <- Flds];
decode_from_template(map, _, Fld, _, Vsn, Types) ->
TupleFields = [F || F <- Fld],
Items = [decode_from_template({any,any}, ?TUPLE, T, true, Vsn, Types)
|| T <- TupleFields],
decode_from_template(map, ?MAP, TupleFields, Vsn, Types) ->
Items = lists:map(fun([Ke, Ve]) ->
{decode_(Ke, Vsn, Types), decode_(Ve, Vsn, Types)}
end, TupleFields),
maps:from_list(Items);
decode_from_template(tuple, _, Fld, _, Vsn, Types) ->
decode_from_template(tuple, _, Fld, Vsn, Types) ->
Items = [decode_(F, Vsn, Types) || F <- Fld],
list_to_tuple(Items);
decode_from_template([Type], _, Fields, _Dyn, Vsn, Types) ->
[decode_typed_(Type, F, Vsn, Types)
|| F <- Fields];
decode_from_template(Type, _, V, Dyn, Vsn, Types) when is_list(Type), is_list(V) ->
decode_fields(Type, V, Dyn, Vsn, Types);
decode_from_template(Type, _, V, _Dyn, Vsn, Types) when is_tuple(Type), is_list(V) ->
decode_from_template(Type, _, Fields, Vsn, Types) when is_list(Type) ->
decode_fields(Type, Fields, Vsn, Types);
%% Zipped = lists:zip(Type, Fields, fail),
%% [decode_typed_(T, F, Vsn, Types)
%% || {T, F} <- Zipped];
decode_from_template(Type, _, V, Vsn, Types) when is_tuple(Type) ->
Zipped = lists:zip(tuple_to_list(Type), V),
Items = [decode_typed_(T1, V1, Vsn, Types) || {T1, V1} <- Zipped],
list_to_tuple(Items);
decode_from_template(label, _, [C], _, _, #{rev_labels := RLbls}) ->
decode_from_template(label, ?LABEL, [C], _, #{rev_labels := RLbls}) ->
Code = decode_basic(int, C),
maps:get(Code, RLbls);
decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int
; Type == negint
; Type == binary
; Type == bool
; Type == id
; Type == label ->
decode_from_template(Type, Code, Fld, Vsn, Types) when Type == int
; Type == negint
; Type == binary
; Type == bool
; Type == id
; Type == label ->
case template(Code, Vsn, Types) of
Type ->
decode_basic(Type, Fld, Types);
@@ -446,45 +593,44 @@ decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int
error(badarg)
end.
dec_zip_items([{K, T}|Is], [{K1, KEnc, VEnc}|Fs], Vsn, Types) ->
dec_zip_items([{K, T}|Is], [{K1, VEnc}|Fs], Vsn, Types) ->
if K =:= K1 ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
true ->
error(illegal_map)
end;
dec_zip_items([{K, T}|Is], [[KEnc, VEnc]|Fs], Vsn, Types) ->
case decode_(KEnc, Vsn, Types) of
K ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
_ ->
error(illegal_map)
end;
dec_zip_items([{opt, K, T}|Is], [{K, KEnc, VEnc}|Fs], Vsn, Types) ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
dec_zip_items([{opt, K, T}|Is], [{K, VEnc}|Fs], Vsn, Types) ->
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
dec_zip_items([{opt, K, T}|Is], [[KEnc,VEnc]|Fs], Vsn, Types) ->
case decode_(KEnc, Vsn, Types) of
K ->
[{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
[{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)];
OtherK ->
dec_zip_items(Is, [{OtherK, KEnc, VEnc}|Fs], Vsn, Types)
dec_zip_items(Is, [{OtherK, VEnc}|Fs], Vsn, Types)
end;
dec_zip_items([], [], _, _) ->
[].
encode_from_template(any, V, _E, Vsn, Types) ->
encode_(V, dyn(emit()), Vsn, Types);
encode_from_template(list, L, E, Vsn, Types) when is_list(L) ->
encode_from_template(any, V, Vsn, Types) ->
encode_(V, Vsn, Types);
encode_from_template(list, L, Vsn, Types) when is_list(L) ->
assert_type(is_list(L), list, L),
emit(E, list, Types,
emit(list, Types,
[encode_(V, Vsn, Types) || V <- L]);
encode_from_template(#{items := Items}, M, E, Vsn, Types) ->
encode_from_template(#{items := Items}, M, Vsn, Types) ->
assert_type(is_map(M), map, M),
Emit = noemit(E),
Encode = fun(K, Type, V) ->
[encode_from_template(any, K, Emit, Vsn, Types),
encode_from_template(Type, V, Emit, Vsn, Types)]
[encode_from_template(any, K, Vsn, Types),
encode_from_template(Type, V, Vsn, Types)]
end,
emit(E, map, Types,
emit(map, Types,
lists:foldr(
fun({K, Type}, Acc) ->
V = maps:get(K, M),
@@ -497,129 +643,150 @@ encode_from_template(#{items := Items}, M, E, Vsn, Types) ->
Acc
end
end, [], Items));
encode_from_template(#{alt := Alts} = T, Term, E, Vsn, Types) when is_list(Alts) ->
encode_alt(Alts, Term, T, E, Vsn, Types);
encode_from_template(#{switch := Alts} = T, Term, E, Vsn, Types) when is_map(Alts),
is_map(Term) ->
encode_switch(Alts, Term, T, E, Vsn, Types);
encode_from_template(map, M, E, Vsn, Types) ->
encode_from_template(#{alt := Alts} = T, Term, Vsn, Types) when is_list(Alts) ->
encode_alt(Alts, Term, T, Vsn, Types);
encode_from_template(#{switch := Alts} = T, Term, Vsn, Types) when is_map(Alts),
is_map(Term) ->
encode_switch(Alts, Term, T, Vsn, Types);
encode_from_template(map, M, Vsn, Types) ->
assert_type(is_map(M), map, M),
Emit = emit(E),
emit(E, map, Types,
[[encode_from_template(any, K, Emit, Vsn, Types),
encode_from_template(any, V, Emit, Vsn, Types)]
emit(map, Types,
[[encode_from_template(any, K, Vsn, Types),
encode_from_template(any, V, Vsn, Types)]
|| {K, V} <- lists:sort(maps:to_list(M))]);
encode_from_template(tuple, T, E, Vsn, Types) ->
encode_from_template(tuple, T, Vsn, Types) ->
assert_type(is_tuple(T), tuple, T),
emit(E, tuple, Types,
[encode_(V, noemit(E), Vsn, Types) || V <- tuple_to_list(T)]);
encode_from_template(T, V, E, Vsn, Types) when is_tuple(T) ->
emit(tuple, Types,
[encode_(V, Vsn, Types) || V <- tuple_to_list(T)]);
encode_from_template(T, V, Vsn, Types) when is_tuple(T) ->
assert_type(is_tuple(V), T, V),
assert_type(tuple_size(T) =:= tuple_size(V), T, V),
Zipped = lists:zip(tuple_to_list(T), tuple_to_list(V)),
emit(E, tuple, Types,
[encode_from_template(T1, V1, noemit(E), Vsn, Types) || {T1, V1} <- Zipped]);
encode_from_template([Type] = T, List, E, Vsn, Types) ->
emit(tuple, Types,
[encode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped]);
encode_from_template(#{list := Type} = T, List, Vsn, Types) ->
assert_type(is_list(List), T, List),
emit(E, list, Types,
[encode_from_template(Type, V, noemit(E), Vsn, Types) || V <- List]);
encode_from_template(Type, List, E, Vsn, Types) when is_list(Type), is_list(List) ->
encode_fields(Type, List, E, Vsn, Types);
encode_from_template(label, V, E, _, Types) ->
emit(list, Types,
[encode_from_template(Type, V, Vsn, Types) || V <- List]);
encode_from_template(Type, List, Vsn, Types) when is_list(Type), is_list(List) ->
emit(list, Types, encode_fields(Type, List, Vsn, Types));
encode_from_template(label, V, _, Types) ->
assert_type(is_atom(V), label, V),
case find_cached_label(V, Types) of
error ->
encode_basic(label, V, E, Types);
encode_basic(label, V, Types);
{ok, Code} when is_integer(Code) ->
emit(E, label, Types,
emit(label, Types,
[encode_basic(int, Code)])
end;
encode_from_template(Type, V, E, _, Types) when Type == id
; Type == binary
; Type == bool
; Type == int
; Type == negint
; Type == label ->
encode_basic(Type, V, E, Types);
encode_from_template(Type, V, E, Vsn, Types) ->
encode_typed_(Type, V, E, Vsn, Types).
encode_from_template(Type, V, _, Types) when Type == id
; Type == binary
; Type == bool
; Type == int
; Type == negint
; Type == label ->
encode_basic(Type, V, Types);
encode_from_template(Type, V, Vsn, Types) ->
encode_typed_(Type, V, Vsn, Types).
assert_type(true, _, _) -> ok;
assert_type(_, Type, V) -> error({illegal, Type, V}).
decode_alt([A|Alts], Code, Fld, T, Vsn, Types) ->
try decode_from_template(A, Code, Fld, true, Vsn, Types)
try decode_typed_(A, Code, Fld, Vsn, Types)
catch error:_ ->
decode_alt(Alts, Code, Fld, T, Vsn, Types)
end;
decode_alt([], _Code, Fld, T, _, _) ->
error({illegal, T, Fld}).
encode_alt(Alts, Term, T, E, Vsn, Types) ->
encode_alt(Alts, Term, T, Vsn, Types) ->
%% Since we don't know which type may match, treat as dynamic.
encode_alt_(Alts, Term, T, dyn(E), Vsn, Types).
encode_alt_(Alts, Term, T, Vsn, Types).
encode_alt_([A|Alts], Term, T, E, Vsn, Types) ->
try encode_from_template(A, Term, E, Vsn, Types)
encode_alt_([A|Alts], Term, T, Vsn, Types) ->
try encode_from_template(A, Term, Vsn, Types)
catch error:_ ->
encode_alt_(Alts, Term, T, E, Vsn, Types)
encode_alt_(Alts, Term, T, Vsn, Types)
end;
encode_alt_([], Term, T, _, _, _) ->
encode_alt_([], Term, T, _, _) ->
error({illegal, T, Term}).
decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types) ->
[[KFld, VFld]] = Fld,
Key = decode_(KFld, Vsn, Types),
case maps:find(Key, Alts) of
{ok, SubType} ->
SubTerm = decode_from_template(SubType, Code, VFld, Dyn, Vsn, Types),
#{Key => SubTerm};
error ->
decode_switch(Alts, Code, Fld, T, Vsn, Types) ->
case is_map_type(Code, Vsn, Types) of
true ->
case Fld of
[[KFld, VFld]] ->
Key = decode_(KFld, Vsn, Types),
case maps:find(Key, Alts) of
{ok, SubType} ->
SubTerm = decode_typed_(SubType, VFld, Vsn, Types),
#{Key => SubTerm};
error ->
error({illegal, T, Fld})
end;
_ ->
error({illegal, T, Fld})
end;
false ->
error({illegal, T, Fld})
end.
encode_switch(Alts, Term, T, E, Vsn, Types) ->
is_map_type(Code, Vsn, Types) ->
case template(Code, Vsn, Types) of
map -> true;
#{items := _} -> true;
#{switch := _} -> true;
T ->
case maps:get(T, maps:get(templates, Types)) of
map -> true;
T -> false;
Other when is_atom(Other) ->
is_map_type(T, Vsn, Types);
_ -> false
end
end.
encode_switch(Alts, Term, T, Vsn, Types) ->
assert_type(map_size(Term) == 1, singleton_map, Term),
[{Key, Subterm}] = maps:to_list(Term),
case maps:find(Key, Alts) of
{ok, SubType} ->
Enc = encode_from_template(SubType, Subterm, E, Vsn, Types),
emit(E, map, Types,
[[encode_from_template(any, Key, E, Vsn, Types),
Enc]]);
Enc = encode_from_template(SubType, Subterm, Vsn, Types),
emit(map, Types,
[[encode_from_template(any, Key, Vsn, Types), Enc]]);
error ->
error({illegal, T, Term})
end.
%% Basically, dynamically encoding a statically defined object
encode_fields([{Field, Type}|TypesLeft],
[{Field, Val}|FieldsLeft], E, Vsn, Types) ->
[ encode_from_template(Type, Val, E, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)];
[{Field, Val}|FieldsLeft], Vsn, Types) ->
KType = auto_template(Field),
[ encode_from_template({KType, Type}, {Field, Val}, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
encode_fields([{_Field, _Type} = FT|_TypesLeft],
[Val |_FieldsLeft], _E, _Vsn, _Types) ->
[Val |_FieldsLeft], _Vsn, _Types) ->
error({illegal_field, FT, Val});
encode_fields([Type|TypesLeft],
[Val |FieldsLeft], E, Vsn, Types) when is_atom(Type) ->
[Val |FieldsLeft], Vsn, Types) when is_atom(Type) ->
%% Not sure about this ...
[ encode_from_template(Type, Val, E, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)];
encode_fields([], [], _, _, _) ->
[ encode_from_template(Type, Val, Vsn, Types)
| encode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
encode_fields([], [], _, _) ->
[].
decode_fields([{Tag, Type}|TypesLeft],
[Field |FieldsLeft], Dyn, Vsn, Types) ->
[Field |FieldsLeft], Vsn, Types) ->
[ {Tag, decode_from_template(Type, 0, Field, Dyn, Vsn, Types)}
| decode_fields(TypesLeft, FieldsLeft, Dyn, Vsn, Types)];
decode_fields([], [], _, _, _) ->
[ {Tag, decode_from_template(Type, 0, Field, Vsn, Types)}
| decode_fields(TypesLeft, FieldsLeft, Vsn, Types)];
decode_fields([], [], _, _) ->
[].
emit(E, Tag, Types, Enc) when E > 0 ->
[emit_code(Tag, Types), Enc];
emit(0, _, _, Enc) ->
Enc.
emit(Tag, Types, Enc) ->
[emit_code(Tag, Types), Enc].
emit_code(Tag, #{rev := Tags}) ->
encode_basic(int, maps:get(Tag, Tags)).
@@ -656,12 +823,10 @@ decode_basic(negint, Fld) ->
decode_basic(Type, Fld) ->
gmserialization:decode_field(Type, Fld).
encode_basic(negint, I, _, Types) when is_integer(I), I < 0 ->
encode_basic(negint, I, Types) when is_integer(I), I < 0 ->
[emit_code(negint, Types), gmserialization:encode_field(int, -I)];
encode_basic(Tag, V, E, Types) when E > 0 ->
[emit_code(Tag, Types), encode_basic(Tag, V)];
encode_basic(Tag, V, 0, _) ->
encode_basic(Tag, V).
encode_basic(Tag, V, Types) ->
[emit_code(Tag, Types), encode_basic(Tag, V)].
encode_basic(label, A) when is_atom(A) ->
encode_basic(binary, atom_to_binary(A, utf8));
@@ -681,28 +846,34 @@ register_types(Types) when is_map(Types) ->
register_types(latest_vsn(), Types).
register_types(Vsn, Types) ->
Result = add_types(Types),
Result = add_types(Vsn, Types),
put_types(Vsn, Result).
add_types(Types) ->
add_types(Types, dynamic_types()).
add_types(?VSN, Types).
add_types(Types, PrevTypes) ->
add_types(Vsn, Types) ->
add_types(Vsn, Types, dynamic_types()).
add_types(Vsn, Types, PrevTypes) ->
Codes = maps:get(codes, Types, #{}),
Rev = rev_codes(Codes),
Templates = maps:get(templates, Types, #{}),
Labels = maps:get(labels, Types, #{}),
RevLabels = rev_codes(Labels),
Options = maps:get(options, Types, #{}),
#{codes := Codes0, rev := Rev0, labels := Labels0,
#{codes := Codes0, rev := Rev0, labels := Labels0, rev_labels := RevLabels0,
templates := Templates0, options := Options0} = PrevTypes,
Merged = #{ codes => maps:merge(Codes0, Codes)
, rev => maps:merge(Rev0, Rev)
, templates => maps:merge(Templates0, Templates)
, options => maps:merge(Options0, Options)
, labels => maps:merge(Labels0, Labels) },
, labels => maps:merge(Labels0, Labels)
, rev_labels => maps:merge(RevLabels0, RevLabels) },
assert_sizes(Merged),
assert_mappings(Merged),
assert_label_cache(Merged).
assert_label_cache(Merged),
assert_resolved_templates(Vsn, Merged).
latest_vsn() ->
case persistent_term:get(pt_key(), undefined) of
@@ -762,6 +933,18 @@ set_opts(Opts) ->
set_opts(Opts, Types) ->
Types#{options => Opts}.
option(O, #{options := Opts}) ->
case Opts of
#{O := V} -> V;
_ -> default_option(O)
end.
default_option(O) ->
case dynamic_types() of
#{options := #{O := V}} -> V;
_ -> undefined
end.
get_opts(#{options := Opts}) ->
Opts.
@@ -821,6 +1004,10 @@ assert_mappings(#{rev := Rev, templates := Ts} = Types) ->
error({missing_templates, Missing, Types})
end.
assert_resolved_templates(Vsn, #{templates := Ts} = Types) ->
_ = [template(T, Vsn, Types) || T <- maps:keys(Ts)],
Types.
assert_label_cache(#{labels := Labels} = Types) ->
Ls = maps:keys(Labels),
case [L || L <- Ls, not is_atom(L)] of
@@ -901,6 +1088,7 @@ user_types_test_() ->
dynamic_types_test_() ->
[ ?_test(revert_to_default_types())
, ?_test(t_typed_map())
, ?_test(t_typed_list())
, ?_test(t_alts())
, ?_test(t_switch())
, ?_test(t_anyints())
@@ -985,7 +1173,7 @@ t_bad_typed_encode(Type, Term, Error) ->
end.
t_reg_chain_objects_array() ->
Template = [{foo, {int, binary}}, {bar, [{int, int}]}, {baz, {int}}],
Template = [{foo, {int, binary}}, {bar, #{list => {int, int}}}, {baz, {int}}],
?debugFmt("Template = ~p", [Template]),
MyTypes = #{ codes => #{ 1002 => coa }
, templates => #{ coa => Template } },
@@ -1042,7 +1230,7 @@ t_reg_label_cache2() ->
register_types(TFromL),
Tup = {'1', '1'},
Enc = gmser_dyn:encode_typed(lbl_tup2, Tup),
[<<0>>,<<1>>,[<<3,235>>,[[<<49>>],[<<49>>]]]] = Enc,
[<<0>>,<<1>>,[[<<3,235>>,<<?TUPLE>>],[[<<?LABEL>>,[<<49>>]],[<<?LABEL>>,[<<49>>]]]]] = Enc,
_Tup = gmser_dyn:decode(Enc).
t_reg_map() ->
@@ -1053,11 +1241,7 @@ t_reg_map() ->
},
register_types(Types),
Enc0 = gmser_dyn:encode_typed(my_map, #{a => foo, b => 17}),
dbg:tracer(),
dbg:tpl(?MODULE,x),
dbg:p(all,[c]),
#{a := foo, b := 17} = try gmser_dyn:decode(Enc0)
after dbg:stop() end,
#{a := foo, b := 17} = gmser_dyn:decode(Enc0),
ok.
t_reg_options() ->
@@ -1080,6 +1264,11 @@ t_typed_map() ->
?assertEqual(Term1, decode(Enc1)),
?assertEqual(Enc, encode_typed(#{items => Items}, Term1)).
t_typed_list() ->
Term = [1,2,3,4],
encode_typed(#{list => int}, Term),
ok.
t_alts() ->
t_round_trip_typed(#{alt => [negint, int]}, -4),
t_round_trip_typed(#{alt => [negint, int]}, 4),
+10 -1
View File
@@ -1,6 +1,7 @@
-module(gmser_dyn_types).
-export([ add_type/4
-export([ add_type/3 %% (Tag, Code, Template) -> Types1
, add_type/4 %% (Tag, Code, Template, Types) -> Types1
, from_list/2
, expand/1 ]).
-export([ next_code/1 ]).
@@ -8,6 +9,14 @@
next_code(#{codes := Codes}) ->
lists:max(maps:keys(Codes)) + 1.
-spec add_type(Tag, Code, Template) -> Types
when Tag :: gmser_dyn:tag()
, Code :: gmser_dyn:code()
, Template :: gmser_dyn:template()
, Types :: gmser_dyn:types().
add_type(Tag, Code, Template) ->
add_type(Tag, Code, Template, gmser_dyn:registered_types()).
add_type(Tag, Code, Template, Types) ->
elem_to_type({Tag, Code, Template}, Types).