From 8734e67c66cdebf1d30d1bfb6d865ca15f871a2d Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Sun, 15 Feb 2026 12:12:04 +0100 Subject: [PATCH] WIP refactor gmser_dyn --- README.md | 78 +++++++++++++- rebar.config | 9 +- rebar.lock | 10 +- src/gmser_dyn.erl | 260 ++++++++++++++++++++++++++++++++++------------ src/gmser_id.erl | 21 +++- 5 files changed, 306 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index d24432f..eddc9bc 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,25 @@ range from 10 to 200, and also to stay within 1 byte.) When encoding `map` types, the map elements are first sorted. When specifying a map type for template-driven encoding, use -the `#{items => [{Key, Value}]}` construct. +the `#{items => [{Key, ValueType} | {opt, Key, ValueType}]}` construct. +The key names are included in the encoding, and are match against the item +specs during decoding. If the key names don't match, the decoding fails, unless +for an `{opt, K, V}` item, in which case that item spec is skipped. +```erlang +T = #{items => [{a,int},{opt,b,int},{c,int}]} +E1 = gmser_dyn:encode_typed(T, #{a => 1, b => 2, c => 3}) -> + [<<0>>,<<1>>,[<<252>>, + [[[<<255>>,<<97>>],[<<248>>,<<1>>]], + [[<<255>>,<<98>>],[<<248>>,<<2>>]], + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] +E2 = gmser_dyn:encode_typed(T, #{a => 1, c => 3}) -> + [<<0>>,<<1>>,[<<252>>, + [[[<<255>>,<<97>>],[<<248>>,<<1>>]], + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] +gmser_dyn:decode_typed(T,E2) -> + #{c => 3,a => 1} +``` ## Labels @@ -119,6 +136,10 @@ inserting the top-level tag), leading to some compression of the output. This also means that the serialized term cannot be decoded without the same schema information on the decoder side. +In some cases, the type tags will still be emitted. These are when alternative types +appear, and for enumerated map types (`#{items => ...}`). In the latter case, it is +due to the support for optional items. + In the case of a directly provided template, all type information is inserted, such that the serialized term can be decoded without any added type information. The template types are still enforced during encoding. @@ -135,10 +156,14 @@ ET(lt2i, [{1,2}]) -> [<<0>>,<<1>>,[<<3,232>>,[[<<1>>,<<2>>]]]] ### Alternative types The dynamic encoder supports two additions to the `gmserialization` template -language: `any` and `#{alt => [AltTypes]}`. +language: `any`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`. + +#### `any` The `any` type doesn't have an associated code, but enforces dynamic encoding. +#### `alt` + The `#{alt => [Type]}` construct also enforces dynamic encoding, and will try to encode as each type in the list, in the specified order, until one matches. @@ -150,6 +175,55 @@ gmser_dyn:encode_typed(anyint,-5) -> [<<0>>,<<1>>,[<<246>>,[<<247>>,<<5>>]]] gmser_dyn:encode_typed(anyint,5) -> [<<0>>,<<1>>,[<<246>>,[<<248>>,<<5>>]]] ``` +#### `switch` + +The `switch` type allows for encoding a 'tagged' object, where the tag determines +the type. + +```erlang +E1 = gmser_dyn:encode_typed(#{switch => #{name => binary, age => int}}, #{age => 29}) -> + [<<0>>,<<1>>,[<<252>>,[[[<<255>>,<<97,103,101>>],[<<248>>,<<29>>]]]]] +gmser_dyn:decode_typed(#{switch => #{name => binary, age => int}}, E1) -> + #{age => 29} +E2 = gmser_dyn:encode_typed(#{switch => #{name => binary, age => int}}, #{name => <<"Ulf">>}) -> + [<<0>>,<<1>>,[<<252>>,[[[<<255>>,<<110,97,109,101>>],[<<249>>,<<85,108,102>>]]]]] +gmser_dyn:decode_typed(#{switch => #{name => binary, age => int}}, E1) -> + #{name => <<"Ulf">>} +``` + +A practical use of `switch` would be in a protocol schema: + +```erlang +t_msg(_) -> + #{switch => #{ call => t_call + , reply => t_reply + , notification => t_notification }}. + +t_call(_) -> + #{items => [ {id, anyint} + , {req, t_req} ]}. + +t_reply(_) -> + #{alt => [#{items => [ {id, anyint} + , {result, t_result} ]}, + #{items => [ {id, anyint} + , {code, anyint} + , {message, binary} ]} + ]}. +``` + +In this scenario, messages are 'taggged' as 1-element maps, e.g.: + +```erlang +async_request(Msg) -> + Id = erlang:unique_integer(), + gmmp_cp:to_server( + whereis(gmmp_core_connector), + #{call => #{ id => Id + , req => Msg }}), + Id. +``` + ### Notes Note that `anyint` is a standard type. The static serializer supports only diff --git a/rebar.config b/rebar.config index 21e8e38..b4bcb62 100644 --- a/rebar.config +++ b/rebar.config @@ -7,4 +7,11 @@ {enacl, {git, "https://git.qpq.swiss/QPQ-AG/enacl.git", - {ref, "4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}}]}. + {ref, "4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}}, + {eblake2, "1.0.0"} + ]}. + +{dialyzer, + [ {plt_apps, all_deps}, + {base_plt_apps, [erts, kernel, stdlib, enacl, base58, eblake2]} + ]}. \ No newline at end of file diff --git a/rebar.lock b/rebar.lock index cdf3d34..d0655e8 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,8 +1,16 @@ +{"1.2.0", [{<<"base58">>, {git,"https://git.qpq.swiss/QPQ-AG/erl-base58.git", {ref,"e6aa62eeae3d4388311401f06e4b939bf4e94b9c"}}, 0}, + {<<"eblake2">>,{pkg,<<"eblake2">>,<<"1.0.0">>},0}, {<<"enacl">>, {git,"https://git.qpq.swiss/QPQ-AG/enacl.git", {ref,"4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}}, - 0}]. + 0}]}. +[ +{pkg_hash,[ + {<<"eblake2">>, <<"EC8AD20E438AAB3F2E8D5D118C366A0754219195F8A0F536587440F8F9BCF2EF">>}]}, +{pkg_hash_ext,[ + {<<"eblake2">>, <<"3C4D300A91845B25D501929A26AC2E6F7157480846FAB2347A4C11AE52E08A99">>}]} +]. diff --git a/src/gmser_dyn.erl b/src/gmser_dyn.erl index 08511c4..ded3219 100644 --- a/src/gmser_dyn.erl +++ b/src/gmser_dyn.erl @@ -24,10 +24,12 @@ , deserialize/3 ]). %% (Bin, Vsn, Types) -> Term %% register a type schema, inspect existing schema --export([ register_types/1 - , registered_types/0 - , registered_types/1 - , latest_vsn/0 +-export([ register_types/1 %% updates stored types + , registered_types/0 %% -"- + , registered_types/1 %% -"- + , add_types/1 %% Returns updated types + , add_types/2 %% -"- + , latest_vsn/0 %% -"- , get_opts/1 , set_opts/1 , set_opts/2 @@ -162,11 +164,36 @@ decode_typed(Type, Fields0, Vsn, Types0) -> Types = proper_types(Types0, Vsn), case decode_tag_and_vsn(Fields0) of {0, Vsn, Fields} -> - decode_from_template(Type, Fields, Vsn, Types); + decode_typed_(Type, Fields, Vsn, Types); Other -> error({illegal_encoding, Other}) end. +decode_typed_(Type, [CodeBin, Fields], Vsn, Types) when is_map(Type) -> + Code = decode_basic(int, CodeBin), + {Tag, Template} = template(Code, Vsn, Types), + Dyn = is_dyn_template(Tag), + case {Type, Template} of + {#{items := _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{items := _}, items} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{alt := _}, _} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{switch:= _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + _ -> + error(badarg) + end; +decode_typed_(Type, [CodeBin, Fields], Vsn, Types) -> + Code = decode_basic(int, CodeBin), + {_, Template} = template(Type, Vsn, Types), + case template(Code, Vsn, Types) of + {Tag, Template} -> + Dyn = is_dyn_template(Tag), + decode_from_template(Template, Code, Fields, Dyn, Vsn, Types); + _ -> + error(badarg) + end; +decode_typed_(_, _, _, _) -> + error(illegal_encoding). + decode_tag_and_vsn([TagBin, VsnBin, Fields]) -> {decode_basic(int, TagBin), decode_basic(int, VsnBin), @@ -189,6 +216,17 @@ assert_vsn(V, #{vsn := V} = Types) -> Types; assert_vsn(V, #{vsn := Other} ) -> error({version_mismatch, V, Other}); assert_vsn(V, #{} = Types ) -> Types#{vsn => V}. +-define(ANYINT, 246). +-define(NEGINT, 247). +-define(INT, 248). +-define(BINARY, 249). +-define(BOOL, 250). +-define(LIST, 251). +-define(MAP, 252). +-define(TUPLE, 253). +-define(ID, 254). +-define(LABEL, 255). + dynamic_types() -> #{ vsn => ?VSN , codes => @@ -203,16 +241,16 @@ dynamic_types() -> , 254 => id , 255 => label} , rev => - #{ anyint => 246 - , negint => 247 - , int => 248 - , binary => 249 - , bool => 250 - , list => 251 - , map => 252 - , tuple => 253 - , id => 254 - , label => 255} + #{ anyint => ?ANYINT + , negint => ?NEGINT + , int => ?INT + , binary => ?BINARY + , bool => ?BOOL + , list => ?LIST + , map => ?MAP + , tuple => ?TUPLE + , id => ?ID + , label => ?LABEL } , labels => #{} , rev_labels => #{} , templates => @@ -230,6 +268,12 @@ dynamic_types() -> , options => #{} }. +is_dyn_template(T) -> + is_dyn_template_(T, dynamic_types()). + +is_dyn_template_(T, #{templates := Ts}) -> + is_map_key(T, Ts). + registered_types() -> registered_types(latest_vsn()). @@ -264,8 +308,9 @@ find_cached_label(Lbl, #{labels := Lbls}) -> decode_([CodeBin, Flds], Vsn, Types) -> Code = decode_basic(int, CodeBin), - {_Tag, Template} = template(Code, Vsn, Types), - decode_from_template(Template, Flds, Vsn, Types). + {Tag, Template} = template(Code, Vsn, Types), + Dyn = is_dyn_template(Tag), + decode_from_template(Template, Code, Flds, Dyn, Vsn, Types). encode_(Term, Vsn, Types) -> encode_(Term, dyn(emit()), Vsn, Types). @@ -351,53 +396,80 @@ auto_template(T) -> T < 0 -> {negint, negint}; true -> error({invalid_type, T}) - end. + end. -decode_from_template(any, Fld, Vsn, Types) -> +decode_from_template(any, _, Fld, _, Vsn, Types) -> decode_(Fld, Vsn, Types); -decode_from_template(#{items := Items}, Fld, Vsn, Types) when is_list(Fld) -> - Zipped = lists:zipwith( - fun({{K, T}, V}) -> {K, T, V}; - ({{opt,K,T}, V}) -> {K, T, V} - end, Items, Fld), +decode_from_template(#{items := Items}, _, Fld, _Dyn, Vsn, Types) when is_list(Fld) -> + Zipped = dec_zip_items(Items, Fld, Vsn, Types), lists:foldl( - fun({K, Type, V}, Map) -> - maps:is_key(K, Map) andalso error(badarg, duplicate_field), - Map#{K => decode_from_template({any,Type}, V, Vsn, Types)} + fun({K, Type, Code, V}, Map) -> + maps:is_key(K, Map) andalso error(badarg, duplicate_key), + Map#{K => decode_from_template(Type, Code, V, true, Vsn, Types)} end, #{}, Zipped); -decode_from_template(#{alt := Alts} = T, Fld, Vsn, Types) when is_list(Alts) -> - decode_alt(Alts, Fld, T, Vsn, Types); -decode_from_template(#{switch := Alts} = T, Fld, Vsn, Types) when is_map(Alts) -> - decode_switch(Alts, Fld, T, Vsn, Types); -decode_from_template(list, Flds, Vsn, Types) -> +decode_from_template(#{alt := Alts} = T, Code, Fld, _, Vsn, Types) when is_list(Alts) -> + decode_alt(Alts, Code, Fld, T, Vsn, Types); +decode_from_template(#{switch := Alts} = T, Code, Fld, Dyn, Vsn, Types) when is_map(Alts) -> + decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types); +decode_from_template(list, _, Flds, _, Vsn, Types) -> [decode_(F, Vsn, Types) || F <- Flds]; -decode_from_template(map, Fld, Vsn, Types) -> +decode_from_template(map, _, Fld, _, Vsn, Types) -> TupleFields = [F || F <- Fld], - Items = [decode_from_template({any,any}, T, Vsn, Types) + Items = [decode_from_template({any,any}, ?TUPLE, T, true, Vsn, Types) || T <- TupleFields], maps:from_list(Items); -decode_from_template(tuple, Fld, Vsn, Types) -> +decode_from_template(tuple, _, Fld, _, Vsn, Types) -> Items = [decode_(F, Vsn, Types) || F <- Fld], list_to_tuple(Items); -decode_from_template([Type], Fields, Vsn, Types) -> - [decode_from_template(Type, F, Vsn, Types) +decode_from_template([Type], _, Fields, _Dyn, Vsn, Types) -> + [decode_typed_(Type, F, Vsn, Types) || F <- Fields]; -decode_from_template(Type, V, Vsn, Types) when is_list(Type), is_list(V) -> - decode_fields(Type, V, Vsn, Types); -decode_from_template(Type, V, Vsn, Types) when is_tuple(Type), is_list(V) -> +decode_from_template(Type, _, V, Dyn, Vsn, Types) when is_list(Type), is_list(V) -> + decode_fields(Type, V, Dyn, Vsn, Types); +decode_from_template(Type, _, V, _Dyn, Vsn, Types) when is_tuple(Type), is_list(V) -> Zipped = lists:zip(tuple_to_list(Type), V), - Items = [decode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped], + Items = [decode_typed_(T1, V1, Vsn, Types) || {T1, V1} <- Zipped], list_to_tuple(Items); -decode_from_template(label, [C], _, #{rev_labels := RLbls}) -> +decode_from_template(label, _, [C], _, _, #{rev_labels := RLbls}) -> Code = decode_basic(int, C), maps:get(Code, RLbls); -decode_from_template(Type, Fld, _, Types) when Type == int - ; Type == negint - ; Type == binary - ; Type == bool - ; Type == id - ; Type == label -> - decode_basic(Type, Fld, Types). +decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int + ; Type == negint + ; Type == binary + ; Type == bool + ; Type == id + ; Type == label -> + case template(Code, Vsn, Types) of + Type -> + decode_basic(Type, Fld, Types); + _ -> + error(badarg) + end. + +dec_zip_items([{K, T}|Is], [{K1, KEnc, VEnc}|Fs], Vsn, Types) -> + if K =:= K1 -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + true -> + error(illegal_map) + end; +dec_zip_items([{K, T}|Is], [[KEnc, VEnc]|Fs], Vsn, Types) -> + case decode_(KEnc, Vsn, Types) of + K -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + _ -> + error(illegal_map) + end; +dec_zip_items([{opt, K, T}|Is], [{K, KEnc, VEnc}|Fs], Vsn, Types) -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; +dec_zip_items([{opt, K, T}|Is], [[KEnc,VEnc]|Fs], Vsn, Types) -> + case decode_(KEnc, Vsn, Types) of + K -> + [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + OtherK -> + dec_zip_items(Is, [{OtherK, KEnc, VEnc}|Fs], Vsn, Types) + end; +dec_zip_items([], [], _, _) -> + []. encode_from_template(any, V, _E, Vsn, Types) -> encode_(V, dyn(emit()), Vsn, Types); @@ -475,12 +547,12 @@ encode_from_template(Type, V, E, Vsn, Types) -> assert_type(true, _, _) -> ok; assert_type(_, Type, V) -> error({illegal, Type, V}). -decode_alt([A|Alts], Fld, T, Vsn, Types) -> - try decode_from_template(A, Fld, Vsn, Types) +decode_alt([A|Alts], Code, Fld, T, Vsn, Types) -> + try decode_from_template(A, Code, Fld, true, Vsn, Types) catch error:_ -> - decode_alt(Alts, Fld, T, Vsn, Types) + decode_alt(Alts, Code, Fld, T, Vsn, Types) end; -decode_alt([], Fld, T, _, _) -> +decode_alt([], _Code, Fld, T, _, _) -> error({illegal, T, Fld}). encode_alt(Alts, Term, T, E, Vsn, Types) -> @@ -495,12 +567,12 @@ encode_alt_([A|Alts], Term, T, E, Vsn, Types) -> encode_alt_([], Term, T, _, _, _) -> error({illegal, T, Term}). -decode_switch(Alts, Fld, T, Vsn, Types) -> - [KFld, VFld] = Fld, +decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types) -> + [[KFld, VFld]] = Fld, Key = decode_(KFld, Vsn, Types), case maps:find(Key, Alts) of {ok, SubType} -> - SubTerm = decode_from_template(SubType, VFld, Vsn, Types), + SubTerm = decode_from_template(SubType, Code, VFld, Dyn, Vsn, Types), #{Key => SubTerm}; error -> error({illegal, T, Fld}) @@ -536,11 +608,11 @@ encode_fields([], [], _, _, _) -> []. decode_fields([{Tag, Type}|TypesLeft], - [Field |FieldsLeft], Vsn, Types) -> + [Field |FieldsLeft], Dyn, Vsn, Types) -> - [ {Tag, decode_from_template(Type, Field, Vsn, Types)} - | decode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; -decode_fields([], [], _, _) -> + [ {Tag, decode_from_template(Type, 0, Field, Dyn, Vsn, Types)} + | decode_fields(TypesLeft, FieldsLeft, Dyn, Vsn, Types)]; +decode_fields([], [], _, _, _) -> []. @@ -609,14 +681,20 @@ register_types(Types) when is_map(Types) -> register_types(latest_vsn(), Types). register_types(Vsn, Types) -> + Result = add_types(Types), + put_types(Vsn, Result). + +add_types(Types) -> + add_types(Types, dynamic_types()). + +add_types(Types, PrevTypes) -> Codes = maps:get(codes, Types, #{}), Rev = rev_codes(Codes), Templates = maps:get(templates, Types, #{}), Labels = maps:get(labels, Types, #{}), Options = maps:get(options, Types, #{}), #{codes := Codes0, rev := Rev0, labels := Labels0, - templates := Templates0, options := Options0} = - dynamic_types(), + templates := Templates0, options := Options0} = PrevTypes, Merged = #{ codes => maps:merge(Codes0, Codes) , rev => maps:merge(Rev0, Rev) , templates => maps:merge(Templates0, Templates) @@ -624,8 +702,7 @@ register_types(Vsn, Types) -> , labels => maps:merge(Labels0, Labels) }, assert_sizes(Merged), assert_mappings(Merged), - Merged1 = assert_label_cache(Merged), - put_types(Vsn, Merged1). + assert_label_cache(Merged). latest_vsn() -> case persistent_term:get(pt_key(), undefined) of @@ -817,6 +894,7 @@ user_types_test_() -> , ?_test(t_reg_template_vsnd_fun()) , ?_test(t_reg_label_cache()) , ?_test(t_reg_label_cache2()) + , ?_test(t_reg_map()) , ?_test(t_reg_options()) ]}. @@ -833,6 +911,45 @@ versioned_types_test_() -> [ ?_test(t_new_version()) ]. +consistency_test_() -> + [?_test(t_full_round_trip(Type, Term)) + || {Type, Term} <- + lists:flatmap( + fun({Type, Terms}) -> + [{Type,T} || T <- Terms] + end, full_round_trip_terms())]. + +full_round_trip_terms() -> + [ + { #{items => [{a, binary}]}, [#{a => <<"foo">>}] } + , { #{items => [ {a, int} + , {opt, b, int} + , {c, int} ]}, [ #{a => 1, b => 2, c => 3} + , #{a => 1, c => 3} ] } + , { #{alt => [int, label]}, [ 1, a ] } + , { #{switch => #{ a => int, b => binary }}, [ #{a => 17} + , #{b => <<"foo">>} ]} + ]. + +t_full_round_trip(Type, Term) -> + ?debugFmt("Type = ~p, Term = ~p", [Type, Term]), + Types = dynamic_types(), + ETyped = encode_typed(Type, Term, Types), + Types1 = gmser_dyn_types:add_type(mytype, 1030, Type, Types), + EReg = encode_typed(mytype, Term, Types1), + DTyped = decode_typed(Type, ETyped, Types), + ?debugFmt("DTyped = ~p", [DTyped]), + DReg = decode_typed(mytype, EReg, Types1), + ?assertEqual(Term, DTyped), + ?assertEqual(Term, DReg), + DDynT = decode(ETyped), + ?assertEqual(Term, DDynT), + try decode(EReg) + catch + error:_ -> + ok + end. + t_round_trip(T) -> ?debugVal(T), ?assertMatch({T, T}, {T, decode(encode(T))}). @@ -928,6 +1045,21 @@ t_reg_label_cache2() -> [<<0>>,<<1>>,[<<3,235>>,[[<<49>>],[<<49>>]]]] = Enc, _Tup = gmser_dyn:decode(Enc). +t_reg_map() -> + Types = + #{codes => #{1013 => my_map}, + templates => #{my_map => #{items => [{a, label}, + {b, int}]}} + }, + register_types(Types), + Enc0 = gmser_dyn:encode_typed(my_map, #{a => foo, b => 17}), + dbg:tracer(), + dbg:tpl(?MODULE,x), + dbg:p(all,[c]), + #{a := foo, b := 17} = try gmser_dyn:decode(Enc0) + after dbg:stop() end, + ok. + t_reg_options() -> register_types(set_opts(#{missing_labels => convert})), [Dyn,Vsn,[Am,<<"random">>]] = gmser_dyn:encode(random), diff --git a/src/gmser_id.erl b/src/gmser_id.erl index 04d0c46..2734027 100644 --- a/src/gmser_id.erl +++ b/src/gmser_id.erl @@ -17,6 +17,9 @@ , is_id/1 ]). +-export([ t_id/1 + ]). + %% For aec_serialization -export([ encode/1 , decode/1 @@ -26,11 +29,18 @@ , val }). --type tag() :: 'account' | 'name' - | 'commitment' | 'contract' | 'channel' - | 'associate_chain' | 'entry' . +-type tag() :: 'account' + | 'associate_chain' + | 'channel' + | 'commitment' + | 'contract' + | 'contract_source' + | 'name' + | 'native_token' + | 'entry'. + -type val() :: <<_:256>>. --opaque(id() :: #id{}). +-type id() :: #id{}. -export_type([ id/0 , tag/0 @@ -94,6 +104,9 @@ decode(<>) -> #id{ tag = decode_tag(Tag) , val = Val}. +-spec t_id(any()) -> id(). +t_id(#id{} = Id) -> Id. + %%%=================================================================== %%% Internal functions %%%===================================================================