From 8734e67c66cdebf1d30d1bfb6d865ca15f871a2d Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Sun, 15 Feb 2026 12:12:04 +0100 Subject: [PATCH 1/2] WIP refactor gmser_dyn --- README.md | 78 +++++++++++++- rebar.config | 9 +- rebar.lock | 10 +- src/gmser_dyn.erl | 260 ++++++++++++++++++++++++++++++++++------------ src/gmser_id.erl | 21 +++- 5 files changed, 306 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index d24432f..eddc9bc 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,25 @@ range from 10 to 200, and also to stay within 1 byte.) When encoding `map` types, the map elements are first sorted. When specifying a map type for template-driven encoding, use -the `#{items => [{Key, Value}]}` construct. +the `#{items => [{Key, ValueType} | {opt, Key, ValueType}]}` construct. +The key names are included in the encoding, and are match against the item +specs during decoding. If the key names don't match, the decoding fails, unless +for an `{opt, K, V}` item, in which case that item spec is skipped. +```erlang +T = #{items => [{a,int},{opt,b,int},{c,int}]} +E1 = gmser_dyn:encode_typed(T, #{a => 1, b => 2, c => 3}) -> + [<<0>>,<<1>>,[<<252>>, + [[[<<255>>,<<97>>],[<<248>>,<<1>>]], + [[<<255>>,<<98>>],[<<248>>,<<2>>]], + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] +E2 = gmser_dyn:encode_typed(T, #{a => 1, c => 3}) -> + [<<0>>,<<1>>,[<<252>>, + [[[<<255>>,<<97>>],[<<248>>,<<1>>]], + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] +gmser_dyn:decode_typed(T,E2) -> + #{c => 3,a => 1} +``` ## Labels @@ -119,6 +136,10 @@ inserting the top-level tag), leading to some compression of the output. This also means that the serialized term cannot be decoded without the same schema information on the decoder side. +In some cases, the type tags will still be emitted. These are when alternative types +appear, and for enumerated map types (`#{items => ...}`). In the latter case, it is +due to the support for optional items. + In the case of a directly provided template, all type information is inserted, such that the serialized term can be decoded without any added type information. The template types are still enforced during encoding. @@ -135,10 +156,14 @@ ET(lt2i, [{1,2}]) -> [<<0>>,<<1>>,[<<3,232>>,[[<<1>>,<<2>>]]]] ### Alternative types The dynamic encoder supports two additions to the `gmserialization` template -language: `any` and `#{alt => [AltTypes]}`. +language: `any`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`. + +#### `any` The `any` type doesn't have an associated code, but enforces dynamic encoding. +#### `alt` + The `#{alt => [Type]}` construct also enforces dynamic encoding, and will try to encode as each type in the list, in the specified order, until one matches. @@ -150,6 +175,55 @@ gmser_dyn:encode_typed(anyint,-5) -> [<<0>>,<<1>>,[<<246>>,[<<247>>,<<5>>]]] gmser_dyn:encode_typed(anyint,5) -> [<<0>>,<<1>>,[<<246>>,[<<248>>,<<5>>]]] ``` +#### `switch` + +The `switch` type allows for encoding a 'tagged' object, where the tag determines +the type. + +```erlang +E1 = gmser_dyn:encode_typed(#{switch => #{name => binary, age => int}}, #{age => 29}) -> + [<<0>>,<<1>>,[<<252>>,[[[<<255>>,<<97,103,101>>],[<<248>>,<<29>>]]]]] +gmser_dyn:decode_typed(#{switch => #{name => binary, age => int}}, E1) -> + #{age => 29} +E2 = gmser_dyn:encode_typed(#{switch => #{name => binary, age => int}}, #{name => <<"Ulf">>}) -> + [<<0>>,<<1>>,[<<252>>,[[[<<255>>,<<110,97,109,101>>],[<<249>>,<<85,108,102>>]]]]] +gmser_dyn:decode_typed(#{switch => #{name => binary, age => int}}, E1) -> + #{name => <<"Ulf">>} +``` + +A practical use of `switch` would be in a protocol schema: + +```erlang +t_msg(_) -> + #{switch => #{ call => t_call + , reply => t_reply + , notification => t_notification }}. + +t_call(_) -> + #{items => [ {id, anyint} + , {req, t_req} ]}. + +t_reply(_) -> + #{alt => [#{items => [ {id, anyint} + , {result, t_result} ]}, + #{items => [ {id, anyint} + , {code, anyint} + , {message, binary} ]} + ]}. +``` + +In this scenario, messages are 'taggged' as 1-element maps, e.g.: + +```erlang +async_request(Msg) -> + Id = erlang:unique_integer(), + gmmp_cp:to_server( + whereis(gmmp_core_connector), + #{call => #{ id => Id + , req => Msg }}), + Id. +``` + ### Notes Note that `anyint` is a standard type. The static serializer supports only diff --git a/rebar.config b/rebar.config index 21e8e38..b4bcb62 100644 --- a/rebar.config +++ b/rebar.config @@ -7,4 +7,11 @@ {enacl, {git, "https://git.qpq.swiss/QPQ-AG/enacl.git", - {ref, "4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}}]}. + {ref, "4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}}, + {eblake2, "1.0.0"} + ]}. + +{dialyzer, + [ {plt_apps, all_deps}, + {base_plt_apps, [erts, kernel, stdlib, enacl, base58, eblake2]} + ]}. \ No newline at end of file diff --git a/rebar.lock b/rebar.lock index cdf3d34..d0655e8 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,8 +1,16 @@ +{"1.2.0", [{<<"base58">>, {git,"https://git.qpq.swiss/QPQ-AG/erl-base58.git", {ref,"e6aa62eeae3d4388311401f06e4b939bf4e94b9c"}}, 0}, + {<<"eblake2">>,{pkg,<<"eblake2">>,<<"1.0.0">>},0}, {<<"enacl">>, {git,"https://git.qpq.swiss/QPQ-AG/enacl.git", {ref,"4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}, - 0}]. + 0}]}. +[ +{pkg_hash,[ + {<<"eblake2">>, <<"EC8AD20E438AAB3F2E8D5D118C366A0754219195F8A0F536587440F8F9BCF2EF">>}]}, +{pkg_hash_ext,[ + {<<"eblake2">>, <<"3C4D300A91845B25D501929A26AC2E6F7157480846FAB2347A4C11AE52E08A99">>}]} +]. diff --git a/src/gmser_dyn.erl b/src/gmser_dyn.erl index 08511c4..ded3219 100644 --- a/src/gmser_dyn.erl +++ b/src/gmser_dyn.erl @@ -24,10 +24,12 @@ , deserialize/3 ]). %% (Bin, Vsn, Types) -> Term %% register a type schema, inspect existing schema --export([ register_types/1 - , registered_types/0 - , registered_types/1 - , latest_vsn/0 +-export([ register_types/1 %% updates stored types + , registered_types/0 %% -"- + , registered_types/1 %% -"- + , add_types/1 %% Returns updated types + , add_types/2 %% -"- + , latest_vsn/0 %% -"- , get_opts/1 , set_opts/1 , set_opts/2 @@ -162,11 +164,36 @@ decode_typed(Type, Fields0, Vsn, Types0) -> Types = proper_types(Types0, Vsn), case decode_tag_and_vsn(Fields0) of {0, Vsn, Fields} -> - decode_from_template(Type, Fields, Vsn, Types); + decode_typed_(Type, Fields, Vsn, Types); Other -> error({illegal_encoding, Other}) end. +decode_typed_(Type, [CodeBin, Fields], Vsn, Types) when is_map(Type) -> + Code = decode_basic(int, CodeBin), + {Tag, Template} = template(Code, Vsn, Types), + Dyn = is_dyn_template(Tag), + case {Type, Template} of + {#{items := _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{items := _}, items} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{alt := _}, _} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{switch:= _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + _ -> + error(badarg) + end; +decode_typed_(Type, [CodeBin, Fields], Vsn, Types) -> + Code = decode_basic(int, CodeBin), + {_, Template} = template(Type, Vsn, Types), + case template(Code, Vsn, Types) of + {Tag, Template} -> + Dyn = is_dyn_template(Tag), + decode_from_template(Template, Code, Fields, Dyn, Vsn, Types); + _ -> + error(badarg) + end; +decode_typed_(_, _, _, _) -> + error(illegal_encoding). + decode_tag_and_vsn([TagBin, VsnBin, Fields]) -> {decode_basic(int, TagBin), decode_basic(int, VsnBin), @@ -189,6 +216,17 @@ assert_vsn(V, #{vsn := V} = Types) -> Types; assert_vsn(V, #{vsn := Other} ) -> error({version_mismatch, V, Other}); assert_vsn(V, #{} = Types ) -> Types#{vsn => V}. +-define(ANYINT, 246). +-define(NEGINT, 247). +-define(INT, 248). +-define(BINARY, 249). +-define(BOOL, 250). +-define(LIST, 251). +-define(MAP, 252). +-define(TUPLE, 253). +-define(ID, 254). +-define(LABEL, 255). + dynamic_types() -> #{ vsn => ?VSN , codes => @@ -203,16 +241,16 @@ dynamic_types() -> , 254 => id , 255 => label} , rev => - #{ anyint => 246 - , negint => 247 - , int => 248 - , binary => 249 - , bool => 250 - , list => 251 - , map => 252 - , tuple => 253 - , id => 254 - , label => 255} + #{ anyint => ?ANYINT + , negint => ?NEGINT + , int => ?INT + , binary => ?BINARY + , bool => ?BOOL + , list => ?LIST + , map => ?MAP + , tuple => ?TUPLE + , id => ?ID + , label => ?LABEL } , labels => #{} , rev_labels => #{} , templates => @@ -230,6 +268,12 @@ dynamic_types() -> , options => #{} }. +is_dyn_template(T) -> + is_dyn_template_(T, dynamic_types()). + +is_dyn_template_(T, #{templates := Ts}) -> + is_map_key(T, Ts). + registered_types() -> registered_types(latest_vsn()). @@ -264,8 +308,9 @@ find_cached_label(Lbl, #{labels := Lbls}) -> decode_([CodeBin, Flds], Vsn, Types) -> Code = decode_basic(int, CodeBin), - {_Tag, Template} = template(Code, Vsn, Types), - decode_from_template(Template, Flds, Vsn, Types). + {Tag, Template} = template(Code, Vsn, Types), + Dyn = is_dyn_template(Tag), + decode_from_template(Template, Code, Flds, Dyn, Vsn, Types). encode_(Term, Vsn, Types) -> encode_(Term, dyn(emit()), Vsn, Types). @@ -351,53 +396,80 @@ auto_template(T) -> T < 0 -> {negint, negint}; true -> error({invalid_type, T}) - end. + end. -decode_from_template(any, Fld, Vsn, Types) -> +decode_from_template(any, _, Fld, _, Vsn, Types) -> decode_(Fld, Vsn, Types); -decode_from_template(#{items := Items}, Fld, Vsn, Types) when is_list(Fld) -> - Zipped = lists:zipwith( - fun({{K, T}, V}) -> {K, T, V}; - ({{opt,K,T}, V}) -> {K, T, V} - end, Items, Fld), +decode_from_template(#{items := Items}, _, Fld, _Dyn, Vsn, Types) when is_list(Fld) -> + Zipped = dec_zip_items(Items, Fld, Vsn, Types), lists:foldl( - fun({K, Type, V}, Map) -> - maps:is_key(K, Map) andalso error(badarg, duplicate_field), - Map#{K => decode_from_template({any,Type}, V, Vsn, Types)} + fun({K, Type, Code, V}, Map) -> + maps:is_key(K, Map) andalso error(badarg, duplicate_key), + Map#{K => decode_from_template(Type, Code, V, true, Vsn, Types)} end, #{}, Zipped); -decode_from_template(#{alt := Alts} = T, Fld, Vsn, Types) when is_list(Alts) -> - decode_alt(Alts, Fld, T, Vsn, Types); -decode_from_template(#{switch := Alts} = T, Fld, Vsn, Types) when is_map(Alts) -> - decode_switch(Alts, Fld, T, Vsn, Types); -decode_from_template(list, Flds, Vsn, Types) -> +decode_from_template(#{alt := Alts} = T, Code, Fld, _, Vsn, Types) when is_list(Alts) -> + decode_alt(Alts, Code, Fld, T, Vsn, Types); +decode_from_template(#{switch := Alts} = T, Code, Fld, Dyn, Vsn, Types) when is_map(Alts) -> + decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types); +decode_from_template(list, _, Flds, _, Vsn, Types) -> [decode_(F, Vsn, Types) || F <- Flds]; -decode_from_template(map, Fld, Vsn, Types) -> +decode_from_template(map, _, Fld, _, Vsn, Types) -> TupleFields = [F || F <- Fld], - Items = [decode_from_template({any,any}, T, Vsn, Types) + Items = [decode_from_template({any,any}, ?TUPLE, T, true, Vsn, Types) || T <- TupleFields], maps:from_list(Items); -decode_from_template(tuple, Fld, Vsn, Types) -> +decode_from_template(tuple, _, Fld, _, Vsn, Types) -> Items = [decode_(F, Vsn, Types) || F <- Fld], list_to_tuple(Items); -decode_from_template([Type], Fields, Vsn, Types) -> - [decode_from_template(Type, F, Vsn, Types) +decode_from_template([Type], _, Fields, _Dyn, Vsn, Types) -> + [decode_typed_(Type, F, Vsn, Types) || F <- Fields]; -decode_from_template(Type, V, Vsn, Types) when is_list(Type), is_list(V) -> - decode_fields(Type, V, Vsn, Types); -decode_from_template(Type, V, Vsn, Types) when is_tuple(Type), is_list(V) -> +decode_from_template(Type, _, V, Dyn, Vsn, Types) when is_list(Type), is_list(V) -> + decode_fields(Type, V, Dyn, Vsn, Types); +decode_from_template(Type, _, V, _Dyn, Vsn, Types) when is_tuple(Type), is_list(V) -> Zipped = lists:zip(tuple_to_list(Type), V), - Items = [decode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped], + Items = [decode_typed_(T1, V1, Vsn, Types) || {T1, V1} <- Zipped], list_to_tuple(Items); -decode_from_template(label, [C], _, #{rev_labels := RLbls}) -> +decode_from_template(label, _, [C], _, _, #{rev_labels := RLbls}) -> Code = decode_basic(int, C), maps:get(Code, RLbls); -decode_from_template(Type, Fld, _, Types) when Type == int - ; Type == negint - ; Type == binary - ; Type == bool - ; Type == id - ; Type == label -> - decode_basic(Type, Fld, Types). +decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int + ; Type == negint + ; Type == binary + ; Type == bool + ; Type == id + ; Type == label -> + case template(Code, Vsn, Types) of + Type -> + decode_basic(Type, Fld, Types); + _ -> + error(badarg) + end. + +dec_zip_items([{K, T}|Is], [{K1, KEnc, VEnc}|Fs], Vsn, Types) -> + if K =:= K1 -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + true -> + error(illegal_map) + end; +dec_zip_items([{K, T}|Is], [[KEnc, VEnc]|Fs], Vsn, Types) -> + case decode_(KEnc, Vsn, Types) of + K -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + _ -> + error(illegal_map) + end; +dec_zip_items([{opt, K, T}|Is], [{K, KEnc, VEnc}|Fs], Vsn, Types) -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; +dec_zip_items([{opt, K, T}|Is], [[KEnc,VEnc]|Fs], Vsn, Types) -> + case decode_(KEnc, Vsn, Types) of + K -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + OtherK -> + dec_zip_items(Is, [{OtherK, KEnc, VEnc}|Fs], Vsn, Types) + end; +dec_zip_items([], [], _, _) -> + []. encode_from_template(any, V, _E, Vsn, Types) -> encode_(V, dyn(emit()), Vsn, Types); @@ -475,12 +547,12 @@ encode_from_template(Type, V, E, Vsn, Types) -> assert_type(true, _, _) -> ok; assert_type(_, Type, V) -> error({illegal, Type, V}). -decode_alt([A|Alts], Fld, T, Vsn, Types) -> - try decode_from_template(A, Fld, Vsn, Types) +decode_alt([A|Alts], Code, Fld, T, Vsn, Types) -> + try decode_from_template(A, Code, Fld, true, Vsn, Types) catch error:_ -> - decode_alt(Alts, Fld, T, Vsn, Types) + decode_alt(Alts, Code, Fld, T, Vsn, Types) end; -decode_alt([], Fld, T, _, _) -> +decode_alt([], _Code, Fld, T, _, _) -> error({illegal, T, Fld}). encode_alt(Alts, Term, T, E, Vsn, Types) -> @@ -495,12 +567,12 @@ encode_alt_([A|Alts], Term, T, E, Vsn, Types) -> encode_alt_([], Term, T, _, _, _) -> error({illegal, T, Term}). -decode_switch(Alts, Fld, T, Vsn, Types) -> - [KFld, VFld] = Fld, +decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types) -> + [[KFld, VFld]] = Fld, Key = decode_(KFld, Vsn, Types), case maps:find(Key, Alts) of {ok, SubType} -> - SubTerm = decode_from_template(SubType, VFld, Vsn, Types), + SubTerm = decode_from_template(SubType, Code, VFld, Dyn, Vsn, Types), #{Key => SubTerm}; error -> error({illegal, T, Fld}) @@ -536,11 +608,11 @@ encode_fields([], [], _, _, _) -> []. decode_fields([{Tag, Type}|TypesLeft], - [Field |FieldsLeft], Vsn, Types) -> + [Field |FieldsLeft], Dyn, Vsn, Types) -> - [ {Tag, decode_from_template(Type, Field, Vsn, Types)} - | decode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; -decode_fields([], [], _, _) -> + [ {Tag, decode_from_template(Type, 0, Field, Dyn, Vsn, Types)} + | decode_fields(TypesLeft, FieldsLeft, Dyn, Vsn, Types)]; +decode_fields([], [], _, _, _) -> []. @@ -609,14 +681,20 @@ register_types(Types) when is_map(Types) -> register_types(latest_vsn(), Types). register_types(Vsn, Types) -> + Result = add_types(Types), + put_types(Vsn, Result). + +add_types(Types) -> + add_types(Types, dynamic_types()). + +add_types(Types, PrevTypes) -> Codes = maps:get(codes, Types, #{}), Rev = rev_codes(Codes), Templates = maps:get(templates, Types, #{}), Labels = maps:get(labels, Types, #{}), Options = maps:get(options, Types, #{}), #{codes := Codes0, rev := Rev0, labels := Labels0, - templates := Templates0, options := Options0} = - dynamic_types(), + templates := Templates0, options := Options0} = PrevTypes, Merged = #{ codes => maps:merge(Codes0, Codes) , rev => maps:merge(Rev0, Rev) , templates => maps:merge(Templates0, Templates) @@ -624,8 +702,7 @@ register_types(Vsn, Types) -> , labels => maps:merge(Labels0, Labels) }, assert_sizes(Merged), assert_mappings(Merged), - Merged1 = assert_label_cache(Merged), - put_types(Vsn, Merged1). + assert_label_cache(Merged). latest_vsn() -> case persistent_term:get(pt_key(), undefined) of @@ -817,6 +894,7 @@ user_types_test_() -> , ?_test(t_reg_template_vsnd_fun()) , ?_test(t_reg_label_cache()) , ?_test(t_reg_label_cache2()) + , ?_test(t_reg_map()) , ?_test(t_reg_options()) ]}. @@ -833,6 +911,45 @@ versioned_types_test_() -> [ ?_test(t_new_version()) ]. +consistency_test_() -> + [?_test(t_full_round_trip(Type, Term)) + || {Type, Term} <- + lists:flatmap( + fun({Type, Terms}) -> + [{Type,T} || T <- Terms] + end, full_round_trip_terms())]. + +full_round_trip_terms() -> + [ + { #{items => [{a, binary}]}, [#{a => <<"foo">>}] } + , { #{items => [ {a, int} + , {opt, b, int} + , {c, int} ]}, [ #{a => 1, b => 2, c => 3} + , #{a => 1, c => 3} ] } + , { #{alt => [int, label]}, [ 1, a ] } + , { #{switch => #{ a => int, b => binary }}, [ #{a => 17} + , #{b => <<"foo">>} ]} + ]. + +t_full_round_trip(Type, Term) -> + ?debugFmt("Type = ~p, Term = ~p", [Type, Term]), + Types = dynamic_types(), + ETyped = encode_typed(Type, Term, Types), + Types1 = gmser_dyn_types:add_type(mytype, 1030, Type, Types), + EReg = encode_typed(mytype, Term, Types1), + DTyped = decode_typed(Type, ETyped, Types), + ?debugFmt("DTyped = ~p", [DTyped]), + DReg = decode_typed(mytype, EReg, Types1), + ?assertEqual(Term, DTyped), + ?assertEqual(Term, DReg), + DDynT = decode(ETyped), + ?assertEqual(Term, DDynT), + try decode(EReg) + catch + error:_ -> + ok + end. + t_round_trip(T) -> ?debugVal(T), ?assertMatch({T, T}, {T, decode(encode(T))}). @@ -928,6 +1045,21 @@ t_reg_label_cache2() -> [<<0>>,<<1>>,[<<3,235>>,[[<<49>>],[<<49>>]]]] = Enc, _Tup = gmser_dyn:decode(Enc). +t_reg_map() -> + Types = + #{codes => #{1013 => my_map}, + templates => #{my_map => #{items => [{a, label}, + {b, int}]}} + }, + register_types(Types), + Enc0 = gmser_dyn:encode_typed(my_map, #{a => foo, b => 17}), + dbg:tracer(), + dbg:tpl(?MODULE,x), + dbg:p(all,[c]), + #{a := foo, b := 17} = try gmser_dyn:decode(Enc0) + after dbg:stop() end, + ok. + t_reg_options() -> register_types(set_opts(#{missing_labels => convert})), [Dyn,Vsn,[Am,<<"random">>]] = gmser_dyn:encode(random), diff --git a/src/gmser_id.erl b/src/gmser_id.erl index 04d0c46..2734027 100644 --- a/src/gmser_id.erl +++ b/src/gmser_id.erl @@ -17,6 +17,9 @@ , is_id/1 ]). +-export([ t_id/1 + ]). + %% For aec_serialization -export([ encode/1 , decode/1 @@ -26,11 +29,18 @@ , val }). --type tag() :: 'account' | 'name' - | 'commitment' | 'contract' | 'channel' - | 'associate_chain' | 'entry' . +-type tag() :: 'account' + | 'associate_chain' + | 'channel' + | 'commitment' + | 'contract' + | 'contract_source' + | 'name' + | 'native_token' + | 'entry'. + -type val() :: <<_:256>>. --opaque(id() :: #id{}). +-type id() :: #id{}. -export_type([ id/0 , tag/0 @@ -94,6 +104,9 @@ decode(<>) -> #id{ tag = decode_tag(Tag) , val = Val}. +-spec t_id(any()) -> id(). +t_id(#id{} = Id) -> Id. + %%%=================================================================== %%% Internal functions %%%=================================================================== -- 2.30.2 From 00699b08b7056e2bf62ecac777bb4360027df3ed Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Wed, 18 Feb 2026 21:10:43 +0100 Subject: [PATCH 2/2] Fix OTP 28 dialyzer warnings, rewrite gmser_dyn decoder gmser_dyn no longer tries to compress output by omitting type tags. Decoding streams using custom template codes can either use 'strict' decoding, in which case matching templates must be registered on the decoding end; in `strict => false` mode, the stream can still be decoded without valudation if the custom template is missing. --- README.md | 79 +++-- src/gmser_dyn.erl | 663 ++++++++++++++++++++++++++-------------- src/gmser_dyn_types.erl | 11 +- 3 files changed, 491 insertions(+), 262 deletions(-) diff --git a/README.md b/README.md index eddc9bc..76b52b1 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,12 @@ T = #{items => [{a,int},{opt,b,int},{c,int}]} E1 = gmser_dyn:encode_typed(T, #{a => 1, b => 2, c => 3}) -> [<<0>>,<<1>>,[<<252>>, [[[<<255>>,<<97>>],[<<248>>,<<1>>]], - [[<<255>>,<<98>>],[<<248>>,<<2>>]], - [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] + [[<<255>>,<<98>>],[<<248>>,<<2>>]], + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] E2 = gmser_dyn:encode_typed(T, #{a => 1, c => 3}) -> [<<0>>,<<1>>,[<<252>>, [[[<<255>>,<<97>>],[<<248>>,<<1>>]], - [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] gmser_dyn:decode_typed(T,E2) -> #{c => 3,a => 1} ``` @@ -81,12 +81,12 @@ converted to binaries, and `create` means that the atom is created dynamically. The option can be passed e.g.: ```erlang -gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert})) +gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert})) ``` or ```erlang -gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert}, Types)) +gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert}, Types)) ``` By calling `gmser_dyn:register_types/1`, after having added options to the type map, @@ -131,37 +131,68 @@ Templates can be provided to the encoder by either naming an already registered type, or by passing a template directly. In both cases, the encoder will enforce the type information in the template. -If the template has been registered, the encoder omits inner type tags (still -inserting the top-level tag), leading to some compression of the output. -This also means that the serialized term cannot be decoded without the same -schema information on the decoder side. - -In some cases, the type tags will still be emitted. These are when alternative types -appear, and for enumerated map types (`#{items => ...}`). In the latter case, it is -due to the support for optional items. - -In the case of a directly provided template, all type information is inserted, -such that the serialized term can be decoded without any added type information. -The template types are still enforced during encoding. +If the template has been registered, the encoder uses the registered type specification +to drive the encoding. The code of the registered template is embedded in the encoded +output: ```erlang -ET = fun(Type,Term) -> io:fwrite("~w~n", [gmser_dyn:encode_typed(Type,Term)]) end. +gmser_dyn:encode_typed({int,int,int}, {1,2,3}) -> + [<<0>>,<<1>>,[<<253>>, + [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]] -ET([{int,int}], [{1,2}]) -> [<<0>>,<<1>>,[<<251>>,[[[<<248>>,<<1>>],[<<248>>,<<2>>]]]]] - -gmser_dyn:register_type(1000,lt2i,[{int,int}]). -ET(lt2i, [{1,2}]) -> [<<0>>,<<1>>,[<<3,232>>,[[<<1>>,<<2>>]]]] +Types = gmser_dyn_types:add_type(t3,1013,{int,int,int}). +gmser_dyn:encode_typed(t3, {1,2,3}, Types) -> + [<<0>>,<<1>>,[[<<3,245>>,<<253>>], + [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]] ``` +Note that the original `<<253>>` type code is wrapped as `[<<3,245>>,<<253>>]`, +where `<<3,245>>` corresponds to the custom code `1013`. + +Using the default option `#{strict => true}`, the decoder will extract the custom +type spec, and validate the encoded data against it. If the custom code is missing, +the decoder aborts. Using `#{strict => false}`, the custom code is used if it exists, +but otherwise, it's ignored, and the encoded data is decoded using the dynamic type +info. + ### Alternative types -The dynamic encoder supports two additions to the `gmserialization` template -language: `any`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`. +The dynamic encoder supports a few additions to the `gmserialization` template +language: `any`, `#{list => Type}`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`. #### `any` The `any` type doesn't have an associated code, but enforces dynamic encoding. +#### `list` + +The original list type notation expects a key-value list, e.g. + +`[{name, binary}, {age, int}]` + +```erlang +EL = gmser_dyn:encode_typed([{name,binary},{age,int}], [{name,<<"Ulf">>},{age,29}]) -> + [<<0>>,<<1>>,[<<251>>, + [[<<253>>,[[<<255>>,<<110,97,109,101>>],[<<249>>,<<85,108,102>>]]], + [<<253>>,[[<<255>>,<<97,103,101>>],[<<248>>,<<29>>]]]]]] +``` +Note that the encoding explicitly lays out a `[{Key, Value}]` structure, all +dynamically typed. This means it can be dynamically decoded without templates. + +```erlang +gmser_dyn:decode(EL). +[{name,<<"Ulf">>},{age,29}] +``` + +In order to specify something like Erlang's `[integer()]` type, we can use +the following: + +```erlang +gmser_dyn:encode_typed(#{list => int}, [1,2,3,4]) -> + [<<0>>,<<1>>,[<<251>>, + [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>],[<<248>>,<<4>>]]]] +``` + #### `alt` The `#{alt => [Type]}` construct also enforces dynamic encoding, and will try diff --git a/src/gmser_dyn.erl b/src/gmser_dyn.erl index ded3219..9ca5ffc 100644 --- a/src/gmser_dyn.erl +++ b/src/gmser_dyn.erl @@ -29,6 +29,7 @@ , registered_types/1 %% -"- , add_types/1 %% Returns updated types , add_types/2 %% -"- + , add_types/3 %% -"- , latest_vsn/0 %% -"- , get_opts/1 , set_opts/1 @@ -44,12 +45,65 @@ -import(gmserialization, [ decode_field/2 ]). +-type tag() :: atom(). +-type code() :: pos_integer(). +-type basic_type() :: anyint + | negint + | int + | binary + | bool + | list + | map + | tuple + | id + | label. + +-type key() :: term(). +-type alt_type() :: #{alt := template()}. +-type list_type() :: #{list := template()}. +-type switch_type() :: #{switch := [{any(), template()}]}. +-type tuple_type() :: tuple(). +-type items_type() :: #{items := [{key(), template()} | {opt, key(), template()}]}. +-type fields_type() :: [{key(), template()}]. + +-type template() :: basic_type() + | alt_type() + | list_type() + | switch_type() + | tuple_type() + | items_type() + | fields_type(). + +-type types() :: #{ vsn := pos_integer() + , codes := #{code() => tag()} + , rev := #{tag() => code()} + , templates := #{tag() => template()} + , labels := #{atom() => code()} + , rev_labels := #{code() => atom()} + , options := #{atom() => any()} + }. + +-export_type([ tag/0 + , code/0 + , basic_type/0 + , key/0 + , alt_type/0 + , switch_type/0 + , list_type/0 + , tuple_type/0 + , items_type/0 + , fields_type/0 + , template/0 + , types/0 + ]). + -define(VSN, 1). -include_lib("kernel/include/logger.hrl"). -ifdef(TEST). --compile([export_all, nowarn_export_all]). +-compile(nowarn_export_all). +-compile(export_all). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -169,30 +223,80 @@ decode_typed(Type, Fields0, Vsn, Types0) -> error({illegal_encoding, Other}) end. -decode_typed_(Type, [CodeBin, Fields], Vsn, Types) when is_map(Type) -> +decode_([[TemplateCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) -> + TemplateCode = decode_basic(int, TemplateCodeBin), Code = decode_basic(int, CodeBin), - {Tag, Template} = template(Code, Vsn, Types), - Dyn = is_dyn_template(Tag), + case is_map_key(Code, Codes) of + true -> + Template = template(Code, Vsn, Types), + decode_from_template(Template, Code, Fld, Vsn, Types); + false -> + case option(strict, Types) of + true -> error({unknown_template, TemplateCode}); + false -> + Template = template(Code, Vsn, Types), + decode_from_template(Template, Code, Fld, Vsn, Types) + end + end; +decode_([CodeBin, Flds], Vsn, Types) when is_binary(CodeBin) -> + Code = decode_basic(int, CodeBin), + Template = template(Code, Vsn, Types), + decode_from_template(Template, Code, Flds, Vsn, Types). + +decode_typed_(Type, [[TCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) -> + TCode = decode_basic(int, TCodeBin), + Code = decode_basic(int, CodeBin), + case maps:find(TCode, Codes) of + {ok, Type} -> + TTemplate = template(TCode, Vsn, Types), + decode_from_template(TTemplate, Code, Fld, Vsn, Types); + {ok, TType} -> + Template = type_to_template(Type, Vsn, Types), + case is_subtype(TType, Template) of + true -> + decode_from_template(Template, Code, Fld, Vsn, Types); + false -> + TTemplate = template(TCode, Vsn, Types), + T1 = decode_from_template(TTemplate, Code, Fld, Vsn, Types), + T2 = decode_from_template(Template, Code, Fld, Vsn, Types), + if T1 =:= T2 -> T1; + true -> + error(badarg) + end + end; + error -> + case option(strict, Types) of + true -> + error(missing_template); + false -> + decode_typed_(Type, Code, Fld, Vsn, Types) + end + end; +decode_typed_(Type, [CodeBin, Fld], Vsn, Types) when is_binary(CodeBin) -> + Code = decode_basic(int, CodeBin), + decode_typed_(Type, Code, Fld, Vsn, Types). + +decode_typed_(Type, Code, Fld, Vsn, Types) when is_map(Type) -> + Template = template(Code, Vsn, Types), case {Type, Template} of - {#{items := _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); - {#{items := _}, items} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); - {#{alt := _}, _} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); - {#{switch:= _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{items := _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{items := _}, items} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{alt := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{switch:= _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{list := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types); _ -> error(badarg) end; -decode_typed_(Type, [CodeBin, Fields], Vsn, Types) -> - Code = decode_basic(int, CodeBin), - {_, Template} = template(Type, Vsn, Types), +decode_typed_(Type, Code, Fld, Vsn, Types) when is_tuple(Type); is_list(Type) -> + decode_from_template(Type, Code, Fld, Vsn, Types); +decode_typed_(Type, Code, Fld, Vsn, Types) -> + Template = template(Type, Vsn, Types), case template(Code, Vsn, Types) of - {Tag, Template} -> - Dyn = is_dyn_template(Tag), - decode_from_template(Template, Code, Fields, Dyn, Vsn, Types); + Template -> + decode_from_template(Template, Code, Fld, Vsn, Types); _ -> error(badarg) - end; -decode_typed_(_, _, _, _) -> - error(illegal_encoding). + end. decode_tag_and_vsn([TagBin, VsnBin, Fields]) -> {decode_basic(int, TagBin), @@ -227,52 +331,54 @@ assert_vsn(V, #{} = Types ) -> Types#{vsn => V}. -define(ID, 254). -define(LABEL, 255). +-define(CODES, #{ 246 => anyint + , 247 => negint + , 248 => int + , 249 => binary + , 250 => bool + , 251 => list + , 252 => map + , 253 => tuple + , 254 => id + , 255 => label}). +-define(REV, #{ anyint => ?ANYINT + , negint => ?NEGINT + , int => ?INT + , binary => ?BINARY + , bool => ?BOOL + , list => ?LIST + , map => ?MAP + , tuple => ?TUPLE + , id => ?ID + , label => ?LABEL }). +-define(TEMPLATES, #{ anyint => #{alt => [negint, int]} + , negint => negint + , int => int + , binary => binary + , bool => bool + , list => list + , map => map + , tuple => tuple + , id => id + , label => label + }). +-define(OPTIONS, #{ missing_labels => fail + , strict => true }). + dynamic_types() -> #{ vsn => ?VSN - , codes => - #{ 246 => anyint - , 247 => negint - , 248 => int - , 249 => binary - , 250 => bool - , 251 => list - , 252 => map - , 253 => tuple - , 254 => id - , 255 => label} - , rev => - #{ anyint => ?ANYINT - , negint => ?NEGINT - , int => ?INT - , binary => ?BINARY - , bool => ?BOOL - , list => ?LIST - , map => ?MAP - , tuple => ?TUPLE - , id => ?ID - , label => ?LABEL } + , codes => ?CODES + , rev => ?REV , labels => #{} , rev_labels => #{} - , templates => - #{ anyint => #{alt => [negint, int]} - , negint => negint - , int => int - , binary => binary - , bool => bool - , list => list - , map => map - , tuple => tuple - , id => id - , label => label - } - , options => #{} + , templates => ?TEMPLATES + , options => ?OPTIONS }. -is_dyn_template(T) -> - is_dyn_template_(T, dynamic_types()). +is_custom_template(T) -> + not is_core_template(T). -is_dyn_template_(T, #{templates := Ts}) -> - is_map_key(T, Ts). +is_core_template(T) -> is_map_key(T, ?TEMPLATES). registered_types() -> registered_types(latest_vsn()). @@ -287,87 +393,123 @@ registered_types(Vsn) -> dynamic_types() end. +type_to_template(T, Vsn, Types) when is_atom(T); is_integer(T) -> + template(T, Vsn, Types); +type_to_template(Type, _Vsn, _Types) -> + Type. + +is_subtype(T, T) -> true; +is_subtype(map, #{items := _}) -> true; +is_subtype(map, #{switch := _}) -> true; +is_subtype(tuple, T) when is_tuple(T) -> true; +is_subtype(list, L) when is_list(L) -> true; +is_subtype(_, #{alt := _}) -> true; +is_subtype(anyint, int) -> true; +is_subtype(_, _) -> + false. + +template(any, _, _) -> any; template(TagOrCode, Vsn, Types) -> - {Tag, Template} = get_template(TagOrCode, Types), - {Tag, dyn_template_(Template, Vsn)}. + Template = get_template(TagOrCode, Types), + dyn_template_(Template, Vsn). get_template(Code, #{codes := Codes, templates := Ts}) when is_integer(Code) -> Tag = maps:get(Code, Codes), - {Tag, maps:get(Tag, Ts)}; + maps:get(Tag, Ts); get_template(Tag, #{templates := Ts}) when is_atom(Tag) -> - {Tag, maps:get(Tag, Ts)}. + maps:get(Tag, Ts). -dyn_template_(F, Vsn) -> - if is_function(F, 0) -> F(); - is_function(F, 1) -> F(Vsn); - true -> F - end. +dyn_template_(T, Vsn) -> + T1 = if is_function(T, 0) -> T(); + is_function(T, 1) -> T(Vsn); + true -> T + end, + resolved_template(T1). + +resolved_template(T) -> + if + is_map(T) -> T; + is_list(T) -> T; + is_tuple(T) -> T; + is_atom(T) -> + case is_core_template(T) of + true -> T; + false -> + error(unresolved_template) + end + end. find_cached_label(Lbl, #{labels := Lbls}) -> maps:find(Lbl, Lbls). -decode_([CodeBin, Flds], Vsn, Types) -> - Code = decode_basic(int, CodeBin), - {Tag, Template} = template(Code, Vsn, Types), - Dyn = is_dyn_template(Tag), - decode_from_template(Template, Code, Flds, Dyn, Vsn, Types). - encode_(Term, Vsn, Types) -> - encode_(Term, dyn(emit()), Vsn, Types). - -encode_(Term, E, Vsn, Types) -> - {_Tag, Template} = auto_template(Term), - encode_from_template(Template, Term, E, Vsn, Types). + Template = auto_template(Term), + encode_from_template(Template, Term, Vsn, Types). %% To control when to emit type codes: %% If the template is predefined, it's 'not dynamic' (nodyn(E)). %% If we are encoding against a type that's part of a predefined template, %% we typically don't emit the type code, except at the very top. %% So: emit type codes if the 'emit' bit is set, or if the 'dyn' bit is set. -emit() -> 2#01. -dyn() -> 2#10. -emit(E) -> E bor 2#01. -noemit(E) -> E band 2#10. -dyn(E) -> E bor 2#10. -nodyn(E) -> E band 2#01. +%% emit() -> 2#01. +%% dyn() -> 2#10. +%% emit(E) -> E bor 2#01. +%% noemit(E) -> E band 2#10. +%% dyn(E) -> E bor 2#10. +%% nodyn(E) -> E band 2#01. encode_typed_(Type, Term, Vsn, #{codes := Codes, rev := Rev} = Types) -> case (is_map_key(Type, Codes) orelse is_map_key(Type, Rev)) of true -> - encode_typed_(Type, Term, nodyn(emit()), Vsn, Types); + encode_typed_1(Type, Term, Vsn, Types); false -> encode_maybe_template(Type, Term, Vsn, Types) end. -encode_typed_(any, Term, _, Vsn, Types) -> - encode_(Term, dyn(emit()), Vsn, Types); -encode_typed_(Code, Term, E, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) -> - {_Tag, Template} = template(Code, Vsn, Types), - [encode_basic(int,Code), - encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)]; -encode_typed_(Tag, Term, E, Vsn, #{templates := Ts, rev := Rev} = Types) +encode_typed_1(any, Term, Vsn, Types) -> + encode_(Term, Vsn, Types); +encode_typed_1(Code, Term, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) -> + Tag = maps:get(Code, Codes), + Template = template(Code, Vsn, Types), + Fld = encode_from_template(Template, Term, Vsn, Types), + case is_custom_template(Tag) of + true -> + [CodeI, FldI] = Fld, + [[encode_basic(int, Code), CodeI], FldI]; + false -> + encode_from_template(Template, Term, Vsn, Types) + end; +encode_typed_1(Tag, Term, Vsn, #{templates := Ts, rev := Rev} = Types) when is_map_key(Tag, Ts) -> Template = dyn_template_(maps:get(Tag, Ts), Vsn), Code = maps:get(Tag, Rev), - [encode_basic(int,Code), - encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)]; -encode_typed_(MaybeTemplate, Term, _, Vsn, Types) -> + Fld = encode_from_template(Template, Term, Vsn, Types), + case is_custom_template(Tag) of + true -> + [CodeI, FldI] = Fld, + [[encode_basic(int,Code), CodeI], FldI]; + false -> + Fld + end; +encode_typed_1(MaybeTemplate, Term, Vsn, Types) -> encode_maybe_template(MaybeTemplate, Term, Vsn, Types). encode_maybe_template(#{items := _} = Type, Term, Vsn, Types) -> case is_map(Term) of true -> - encode_from_template(Type, Term, emit(dyn()), Vsn, Types); + encode_from_template(Type, Term, Vsn, Types); false -> error({invalid, Type, Term}) end; encode_maybe_template(#{alt := _} = Type, Term, Vsn, Types) -> - encode_from_template(Type, Term, Vsn, emit(dyn()), Types); + encode_from_template(Type, Term, Vsn, Types); encode_maybe_template(#{switch := _} = Type, Term, Vsn, Types) -> - encode_from_template(Type, Term, Vsn, emit(dyn()), Types); + encode_from_template(Type, Term, Vsn, Types); +encode_maybe_template(#{list := _} = Type, Term, Vsn, Types) -> + encode_from_template(Type, Term, Vsn, Types); encode_maybe_template(Pat, Term, Vsn, Types) when is_list(Pat); is_tuple(Pat) -> - encode_from_template(Pat, Term, emit(dyn()), Vsn, Types); + encode_from_template(Pat, Term, Vsn, Types); encode_maybe_template(Other, Term, _Vsn, _Types) -> error({illegal_template, Other, Term}). @@ -378,67 +520,72 @@ auto_template({id,Tag,V}) when Tag == account ; Tag == channel ; Tag == associate_chain ; Tag == entry -> - if is_binary(V) -> {id, id}; + if is_binary(V) -> id; true -> %% close, but no cigar - {tuple, tuple} + tuple end; auto_template(T) -> - if is_map(T) -> {map, map}; - is_list(T) -> {list, list}; - is_tuple(T) -> {tuple, tuple}; - is_binary(T) -> {binary, binary}; - is_boolean(T) -> {bool, bool}; - is_atom(T) -> {label, label}; % binary_to_existing_atom() + if is_map(T) -> map; + is_list(T) -> list; + is_tuple(T) -> tuple; + is_binary(T) -> binary; + is_boolean(T) -> bool; + is_atom(T) -> label; % binary_to_existing_atom() is_integer(T), - T >= 0 -> {int, int}; + T >= 0 -> int; is_integer(T), - T < 0 -> {negint, negint}; + T < 0 -> negint; true -> error({invalid_type, T}) end. -decode_from_template(any, _, Fld, _, Vsn, Types) -> +decode_from_template(any, _Code, Fld, Vsn, Types) -> decode_(Fld, Vsn, Types); -decode_from_template(#{items := Items}, _, Fld, _Dyn, Vsn, Types) when is_list(Fld) -> +decode_from_template(#{items := Items}, _, Fld, Vsn, Types) when is_list(Fld) -> Zipped = dec_zip_items(Items, Fld, Vsn, Types), lists:foldl( - fun({K, Type, Code, V}, Map) -> - maps:is_key(K, Map) andalso error(badarg, duplicate_key), - Map#{K => decode_from_template(Type, Code, V, true, Vsn, Types)} + fun({K, Type, V}, Map) -> + case maps:is_key(K, Map) of + true -> error(duplicate_key); + false -> + Map#{K => decode_typed_(Type, V, Vsn, Types)} + end end, #{}, Zipped); -decode_from_template(#{alt := Alts} = T, Code, Fld, _, Vsn, Types) when is_list(Alts) -> +decode_from_template(#{alt := Alts} = T, Code, Fld, Vsn, Types) when is_list(Alts) -> decode_alt(Alts, Code, Fld, T, Vsn, Types); -decode_from_template(#{switch := Alts} = T, Code, Fld, Dyn, Vsn, Types) when is_map(Alts) -> - decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types); -decode_from_template(list, _, Flds, _, Vsn, Types) -> +decode_from_template(#{switch := Alts} = T, Code, Fld, Vsn, Types) when is_map(Alts) -> + decode_switch(Alts, Code, Fld, T, Vsn, Types); +decode_from_template(#{list := Type}, ?LIST, Fld, Vsn, Types) -> + [decode_typed_(Type, F, Vsn, Types) || F <- Fld]; +decode_from_template(list, _, Flds, Vsn, Types) -> [decode_(F, Vsn, Types) || F <- Flds]; -decode_from_template(map, _, Fld, _, Vsn, Types) -> - TupleFields = [F || F <- Fld], - Items = [decode_from_template({any,any}, ?TUPLE, T, true, Vsn, Types) - || T <- TupleFields], +decode_from_template(map, ?MAP, TupleFields, Vsn, Types) -> + Items = lists:map(fun([Ke, Ve]) -> + {decode_(Ke, Vsn, Types), decode_(Ve, Vsn, Types)} + end, TupleFields), maps:from_list(Items); -decode_from_template(tuple, _, Fld, _, Vsn, Types) -> +decode_from_template(tuple, _, Fld, Vsn, Types) -> Items = [decode_(F, Vsn, Types) || F <- Fld], list_to_tuple(Items); -decode_from_template([Type], _, Fields, _Dyn, Vsn, Types) -> - [decode_typed_(Type, F, Vsn, Types) - || F <- Fields]; -decode_from_template(Type, _, V, Dyn, Vsn, Types) when is_list(Type), is_list(V) -> - decode_fields(Type, V, Dyn, Vsn, Types); -decode_from_template(Type, _, V, _Dyn, Vsn, Types) when is_tuple(Type), is_list(V) -> +decode_from_template(Type, _, Fields, Vsn, Types) when is_list(Type) -> + decode_fields(Type, Fields, Vsn, Types); + %% Zipped = lists:zip(Type, Fields, fail), + %% [decode_typed_(T, F, Vsn, Types) + %% || {T, F} <- Zipped]; +decode_from_template(Type, _, V, Vsn, Types) when is_tuple(Type) -> Zipped = lists:zip(tuple_to_list(Type), V), Items = [decode_typed_(T1, V1, Vsn, Types) || {T1, V1} <- Zipped], list_to_tuple(Items); -decode_from_template(label, _, [C], _, _, #{rev_labels := RLbls}) -> +decode_from_template(label, ?LABEL, [C], _, #{rev_labels := RLbls}) -> Code = decode_basic(int, C), maps:get(Code, RLbls); -decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int - ; Type == negint - ; Type == binary - ; Type == bool - ; Type == id - ; Type == label -> +decode_from_template(Type, Code, Fld, Vsn, Types) when Type == int + ; Type == negint + ; Type == binary + ; Type == bool + ; Type == id + ; Type == label -> case template(Code, Vsn, Types) of Type -> decode_basic(Type, Fld, Types); @@ -446,45 +593,44 @@ decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int error(badarg) end. -dec_zip_items([{K, T}|Is], [{K1, KEnc, VEnc}|Fs], Vsn, Types) -> +dec_zip_items([{K, T}|Is], [{K1, VEnc}|Fs], Vsn, Types) -> if K =:= K1 -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; true -> error(illegal_map) end; dec_zip_items([{K, T}|Is], [[KEnc, VEnc]|Fs], Vsn, Types) -> case decode_(KEnc, Vsn, Types) of K -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; _ -> error(illegal_map) end; -dec_zip_items([{opt, K, T}|Is], [{K, KEnc, VEnc}|Fs], Vsn, Types) -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; +dec_zip_items([{opt, K, T}|Is], [{K, VEnc}|Fs], Vsn, Types) -> + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; dec_zip_items([{opt, K, T}|Is], [[KEnc,VEnc]|Fs], Vsn, Types) -> case decode_(KEnc, Vsn, Types) of K -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; OtherK -> - dec_zip_items(Is, [{OtherK, KEnc, VEnc}|Fs], Vsn, Types) + dec_zip_items(Is, [{OtherK, VEnc}|Fs], Vsn, Types) end; dec_zip_items([], [], _, _) -> []. -encode_from_template(any, V, _E, Vsn, Types) -> - encode_(V, dyn(emit()), Vsn, Types); -encode_from_template(list, L, E, Vsn, Types) when is_list(L) -> +encode_from_template(any, V, Vsn, Types) -> + encode_(V, Vsn, Types); +encode_from_template(list, L, Vsn, Types) when is_list(L) -> assert_type(is_list(L), list, L), - emit(E, list, Types, + emit(list, Types, [encode_(V, Vsn, Types) || V <- L]); -encode_from_template(#{items := Items}, M, E, Vsn, Types) -> +encode_from_template(#{items := Items}, M, Vsn, Types) -> assert_type(is_map(M), map, M), - Emit = noemit(E), Encode = fun(K, Type, V) -> - [encode_from_template(any, K, Emit, Vsn, Types), - encode_from_template(Type, V, Emit, Vsn, Types)] + [encode_from_template(any, K, Vsn, Types), + encode_from_template(Type, V, Vsn, Types)] end, - emit(E, map, Types, + emit(map, Types, lists:foldr( fun({K, Type}, Acc) -> V = maps:get(K, M), @@ -497,129 +643,150 @@ encode_from_template(#{items := Items}, M, E, Vsn, Types) -> Acc end end, [], Items)); -encode_from_template(#{alt := Alts} = T, Term, E, Vsn, Types) when is_list(Alts) -> - encode_alt(Alts, Term, T, E, Vsn, Types); -encode_from_template(#{switch := Alts} = T, Term, E, Vsn, Types) when is_map(Alts), - is_map(Term) -> - encode_switch(Alts, Term, T, E, Vsn, Types); -encode_from_template(map, M, E, Vsn, Types) -> +encode_from_template(#{alt := Alts} = T, Term, Vsn, Types) when is_list(Alts) -> + encode_alt(Alts, Term, T, Vsn, Types); +encode_from_template(#{switch := Alts} = T, Term, Vsn, Types) when is_map(Alts), + is_map(Term) -> + encode_switch(Alts, Term, T, Vsn, Types); +encode_from_template(map, M, Vsn, Types) -> assert_type(is_map(M), map, M), - Emit = emit(E), - emit(E, map, Types, - [[encode_from_template(any, K, Emit, Vsn, Types), - encode_from_template(any, V, Emit, Vsn, Types)] + emit(map, Types, + [[encode_from_template(any, K, Vsn, Types), + encode_from_template(any, V, Vsn, Types)] || {K, V} <- lists:sort(maps:to_list(M))]); -encode_from_template(tuple, T, E, Vsn, Types) -> +encode_from_template(tuple, T, Vsn, Types) -> assert_type(is_tuple(T), tuple, T), - emit(E, tuple, Types, - [encode_(V, noemit(E), Vsn, Types) || V <- tuple_to_list(T)]); -encode_from_template(T, V, E, Vsn, Types) when is_tuple(T) -> + emit(tuple, Types, + [encode_(V, Vsn, Types) || V <- tuple_to_list(T)]); +encode_from_template(T, V, Vsn, Types) when is_tuple(T) -> assert_type(is_tuple(V), T, V), assert_type(tuple_size(T) =:= tuple_size(V), T, V), Zipped = lists:zip(tuple_to_list(T), tuple_to_list(V)), - emit(E, tuple, Types, - [encode_from_template(T1, V1, noemit(E), Vsn, Types) || {T1, V1} <- Zipped]); -encode_from_template([Type] = T, List, E, Vsn, Types) -> + emit(tuple, Types, + [encode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped]); +encode_from_template(#{list := Type} = T, List, Vsn, Types) -> assert_type(is_list(List), T, List), - emit(E, list, Types, - [encode_from_template(Type, V, noemit(E), Vsn, Types) || V <- List]); -encode_from_template(Type, List, E, Vsn, Types) when is_list(Type), is_list(List) -> - encode_fields(Type, List, E, Vsn, Types); -encode_from_template(label, V, E, _, Types) -> + emit(list, Types, + [encode_from_template(Type, V, Vsn, Types) || V <- List]); +encode_from_template(Type, List, Vsn, Types) when is_list(Type), is_list(List) -> + emit(list, Types, encode_fields(Type, List, Vsn, Types)); +encode_from_template(label, V, _, Types) -> assert_type(is_atom(V), label, V), case find_cached_label(V, Types) of error -> - encode_basic(label, V, E, Types); + encode_basic(label, V, Types); {ok, Code} when is_integer(Code) -> - emit(E, label, Types, + emit(label, Types, [encode_basic(int, Code)]) end; -encode_from_template(Type, V, E, _, Types) when Type == id - ; Type == binary - ; Type == bool - ; Type == int - ; Type == negint - ; Type == label -> - encode_basic(Type, V, E, Types); -encode_from_template(Type, V, E, Vsn, Types) -> - encode_typed_(Type, V, E, Vsn, Types). +encode_from_template(Type, V, _, Types) when Type == id + ; Type == binary + ; Type == bool + ; Type == int + ; Type == negint + ; Type == label -> + encode_basic(Type, V, Types); +encode_from_template(Type, V, Vsn, Types) -> + encode_typed_(Type, V, Vsn, Types). assert_type(true, _, _) -> ok; assert_type(_, Type, V) -> error({illegal, Type, V}). decode_alt([A|Alts], Code, Fld, T, Vsn, Types) -> - try decode_from_template(A, Code, Fld, true, Vsn, Types) + try decode_typed_(A, Code, Fld, Vsn, Types) catch error:_ -> decode_alt(Alts, Code, Fld, T, Vsn, Types) end; decode_alt([], _Code, Fld, T, _, _) -> error({illegal, T, Fld}). -encode_alt(Alts, Term, T, E, Vsn, Types) -> +encode_alt(Alts, Term, T, Vsn, Types) -> %% Since we don't know which type may match, treat as dynamic. - encode_alt_(Alts, Term, T, dyn(E), Vsn, Types). + encode_alt_(Alts, Term, T, Vsn, Types). -encode_alt_([A|Alts], Term, T, E, Vsn, Types) -> - try encode_from_template(A, Term, E, Vsn, Types) +encode_alt_([A|Alts], Term, T, Vsn, Types) -> + try encode_from_template(A, Term, Vsn, Types) catch error:_ -> - encode_alt_(Alts, Term, T, E, Vsn, Types) + encode_alt_(Alts, Term, T, Vsn, Types) end; -encode_alt_([], Term, T, _, _, _) -> +encode_alt_([], Term, T, _, _) -> error({illegal, T, Term}). -decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types) -> - [[KFld, VFld]] = Fld, - Key = decode_(KFld, Vsn, Types), - case maps:find(Key, Alts) of - {ok, SubType} -> - SubTerm = decode_from_template(SubType, Code, VFld, Dyn, Vsn, Types), - #{Key => SubTerm}; - error -> +decode_switch(Alts, Code, Fld, T, Vsn, Types) -> + case is_map_type(Code, Vsn, Types) of + true -> + case Fld of + [[KFld, VFld]] -> + Key = decode_(KFld, Vsn, Types), + case maps:find(Key, Alts) of + {ok, SubType} -> + SubTerm = decode_typed_(SubType, VFld, Vsn, Types), + #{Key => SubTerm}; + error -> + error({illegal, T, Fld}) + end; + _ -> + error({illegal, T, Fld}) + end; + false -> error({illegal, T, Fld}) end. -encode_switch(Alts, Term, T, E, Vsn, Types) -> +is_map_type(Code, Vsn, Types) -> + case template(Code, Vsn, Types) of + map -> true; + #{items := _} -> true; + #{switch := _} -> true; + T -> + case maps:get(T, maps:get(templates, Types)) of + map -> true; + T -> false; + Other when is_atom(Other) -> + is_map_type(T, Vsn, Types); + _ -> false + end + end. + +encode_switch(Alts, Term, T, Vsn, Types) -> assert_type(map_size(Term) == 1, singleton_map, Term), [{Key, Subterm}] = maps:to_list(Term), case maps:find(Key, Alts) of {ok, SubType} -> - Enc = encode_from_template(SubType, Subterm, E, Vsn, Types), - emit(E, map, Types, - [[encode_from_template(any, Key, E, Vsn, Types), - Enc]]); + Enc = encode_from_template(SubType, Subterm, Vsn, Types), + emit(map, Types, + [[encode_from_template(any, Key, Vsn, Types), Enc]]); error -> error({illegal, T, Term}) end. %% Basically, dynamically encoding a statically defined object encode_fields([{Field, Type}|TypesLeft], - [{Field, Val}|FieldsLeft], E, Vsn, Types) -> - [ encode_from_template(Type, Val, E, Vsn, Types) - | encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)]; + [{Field, Val}|FieldsLeft], Vsn, Types) -> + KType = auto_template(Field), + [ encode_from_template({KType, Type}, {Field, Val}, Vsn, Types) + | encode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; encode_fields([{_Field, _Type} = FT|_TypesLeft], - [Val |_FieldsLeft], _E, _Vsn, _Types) -> + [Val |_FieldsLeft], _Vsn, _Types) -> error({illegal_field, FT, Val}); encode_fields([Type|TypesLeft], - [Val |FieldsLeft], E, Vsn, Types) when is_atom(Type) -> + [Val |FieldsLeft], Vsn, Types) when is_atom(Type) -> %% Not sure about this ... - [ encode_from_template(Type, Val, E, Vsn, Types) - | encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)]; -encode_fields([], [], _, _, _) -> + [ encode_from_template(Type, Val, Vsn, Types) + | encode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; +encode_fields([], [], _, _) -> []. decode_fields([{Tag, Type}|TypesLeft], - [Field |FieldsLeft], Dyn, Vsn, Types) -> + [Field |FieldsLeft], Vsn, Types) -> - [ {Tag, decode_from_template(Type, 0, Field, Dyn, Vsn, Types)} - | decode_fields(TypesLeft, FieldsLeft, Dyn, Vsn, Types)]; -decode_fields([], [], _, _, _) -> + [ {Tag, decode_from_template(Type, 0, Field, Vsn, Types)} + | decode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; +decode_fields([], [], _, _) -> []. -emit(E, Tag, Types, Enc) when E > 0 -> - [emit_code(Tag, Types), Enc]; -emit(0, _, _, Enc) -> - Enc. +emit(Tag, Types, Enc) -> + [emit_code(Tag, Types), Enc]. emit_code(Tag, #{rev := Tags}) -> encode_basic(int, maps:get(Tag, Tags)). @@ -656,12 +823,10 @@ decode_basic(negint, Fld) -> decode_basic(Type, Fld) -> gmserialization:decode_field(Type, Fld). -encode_basic(negint, I, _, Types) when is_integer(I), I < 0 -> +encode_basic(negint, I, Types) when is_integer(I), I < 0 -> [emit_code(negint, Types), gmserialization:encode_field(int, -I)]; -encode_basic(Tag, V, E, Types) when E > 0 -> - [emit_code(Tag, Types), encode_basic(Tag, V)]; -encode_basic(Tag, V, 0, _) -> - encode_basic(Tag, V). +encode_basic(Tag, V, Types) -> + [emit_code(Tag, Types), encode_basic(Tag, V)]. encode_basic(label, A) when is_atom(A) -> encode_basic(binary, atom_to_binary(A, utf8)); @@ -681,28 +846,34 @@ register_types(Types) when is_map(Types) -> register_types(latest_vsn(), Types). register_types(Vsn, Types) -> - Result = add_types(Types), + Result = add_types(Vsn, Types), put_types(Vsn, Result). add_types(Types) -> - add_types(Types, dynamic_types()). + add_types(?VSN, Types). -add_types(Types, PrevTypes) -> +add_types(Vsn, Types) -> + add_types(Vsn, Types, dynamic_types()). + +add_types(Vsn, Types, PrevTypes) -> Codes = maps:get(codes, Types, #{}), Rev = rev_codes(Codes), Templates = maps:get(templates, Types, #{}), Labels = maps:get(labels, Types, #{}), + RevLabels = rev_codes(Labels), Options = maps:get(options, Types, #{}), - #{codes := Codes0, rev := Rev0, labels := Labels0, + #{codes := Codes0, rev := Rev0, labels := Labels0, rev_labels := RevLabels0, templates := Templates0, options := Options0} = PrevTypes, Merged = #{ codes => maps:merge(Codes0, Codes) , rev => maps:merge(Rev0, Rev) , templates => maps:merge(Templates0, Templates) , options => maps:merge(Options0, Options) - , labels => maps:merge(Labels0, Labels) }, + , labels => maps:merge(Labels0, Labels) + , rev_labels => maps:merge(RevLabels0, RevLabels) }, assert_sizes(Merged), assert_mappings(Merged), - assert_label_cache(Merged). + assert_label_cache(Merged), + assert_resolved_templates(Vsn, Merged). latest_vsn() -> case persistent_term:get(pt_key(), undefined) of @@ -762,6 +933,18 @@ set_opts(Opts) -> set_opts(Opts, Types) -> Types#{options => Opts}. +option(O, #{options := Opts}) -> + case Opts of + #{O := V} -> V; + _ -> default_option(O) + end. + +default_option(O) -> + case dynamic_types() of + #{options := #{O := V}} -> V; + _ -> undefined + end. + get_opts(#{options := Opts}) -> Opts. @@ -821,6 +1004,10 @@ assert_mappings(#{rev := Rev, templates := Ts} = Types) -> error({missing_templates, Missing, Types}) end. +assert_resolved_templates(Vsn, #{templates := Ts} = Types) -> + _ = [template(T, Vsn, Types) || T <- maps:keys(Ts)], + Types. + assert_label_cache(#{labels := Labels} = Types) -> Ls = maps:keys(Labels), case [L || L <- Ls, not is_atom(L)] of @@ -901,6 +1088,7 @@ user_types_test_() -> dynamic_types_test_() -> [ ?_test(revert_to_default_types()) , ?_test(t_typed_map()) + , ?_test(t_typed_list()) , ?_test(t_alts()) , ?_test(t_switch()) , ?_test(t_anyints()) @@ -985,7 +1173,7 @@ t_bad_typed_encode(Type, Term, Error) -> end. t_reg_chain_objects_array() -> - Template = [{foo, {int, binary}}, {bar, [{int, int}]}, {baz, {int}}], + Template = [{foo, {int, binary}}, {bar, #{list => {int, int}}}, {baz, {int}}], ?debugFmt("Template = ~p", [Template]), MyTypes = #{ codes => #{ 1002 => coa } , templates => #{ coa => Template } }, @@ -1042,7 +1230,7 @@ t_reg_label_cache2() -> register_types(TFromL), Tup = {'1', '1'}, Enc = gmser_dyn:encode_typed(lbl_tup2, Tup), - [<<0>>,<<1>>,[<<3,235>>,[[<<49>>],[<<49>>]]]] = Enc, + [<<0>>,<<1>>,[[<<3,235>>,<>],[[<>,[<<49>>]],[<>,[<<49>>]]]]] = Enc, _Tup = gmser_dyn:decode(Enc). t_reg_map() -> @@ -1053,11 +1241,7 @@ t_reg_map() -> }, register_types(Types), Enc0 = gmser_dyn:encode_typed(my_map, #{a => foo, b => 17}), - dbg:tracer(), - dbg:tpl(?MODULE,x), - dbg:p(all,[c]), - #{a := foo, b := 17} = try gmser_dyn:decode(Enc0) - after dbg:stop() end, + #{a := foo, b := 17} = gmser_dyn:decode(Enc0), ok. t_reg_options() -> @@ -1080,6 +1264,11 @@ t_typed_map() -> ?assertEqual(Term1, decode(Enc1)), ?assertEqual(Enc, encode_typed(#{items => Items}, Term1)). +t_typed_list() -> + Term = [1,2,3,4], + encode_typed(#{list => int}, Term), + ok. + t_alts() -> t_round_trip_typed(#{alt => [negint, int]}, -4), t_round_trip_typed(#{alt => [negint, int]}, 4), diff --git a/src/gmser_dyn_types.erl b/src/gmser_dyn_types.erl index 6117d6d..1a41f1c 100644 --- a/src/gmser_dyn_types.erl +++ b/src/gmser_dyn_types.erl @@ -1,6 +1,7 @@ -module(gmser_dyn_types). --export([ add_type/4 +-export([ add_type/3 %% (Tag, Code, Template) -> Types1 + , add_type/4 %% (Tag, Code, Template, Types) -> Types1 , from_list/2 , expand/1 ]). -export([ next_code/1 ]). @@ -8,6 +9,14 @@ next_code(#{codes := Codes}) -> lists:max(maps:keys(Codes)) + 1. +-spec add_type(Tag, Code, Template) -> Types + when Tag :: gmser_dyn:tag() + , Code :: gmser_dyn:code() + , Template :: gmser_dyn:template() + , Types :: gmser_dyn:types(). +add_type(Tag, Code, Template) -> + add_type(Tag, Code, Template, gmser_dyn:registered_types()). + add_type(Tag, Code, Template, Types) -> elem_to_type({Tag, Code, Template}, Types). -- 2.30.2