diff --git a/README.md b/README.md index eddc9bc..76b52b1 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,12 @@ T = #{items => [{a,int},{opt,b,int},{c,int}]} E1 = gmser_dyn:encode_typed(T, #{a => 1, b => 2, c => 3}) -> [<<0>>,<<1>>,[<<252>>, [[[<<255>>,<<97>>],[<<248>>,<<1>>]], - [[<<255>>,<<98>>],[<<248>>,<<2>>]], - [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] + [[<<255>>,<<98>>],[<<248>>,<<2>>]], + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] E2 = gmser_dyn:encode_typed(T, #{a => 1, c => 3}) -> [<<0>>,<<1>>,[<<252>>, [[[<<255>>,<<97>>],[<<248>>,<<1>>]], - [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] + [[<<255>>,<<99>>],[<<248>>,<<3>>]]]]] gmser_dyn:decode_typed(T,E2) -> #{c => 3,a => 1} ``` @@ -81,12 +81,12 @@ converted to binaries, and `create` means that the atom is created dynamically. The option can be passed e.g.: ```erlang -gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert})) +gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert})) ``` or ```erlang -gmser_dyn:deserialize(Binary, set_opts(#{missing_labels => convert}, Types)) +gmser_dyn:deserialize(Binary, gmser_dyn:set_opts(#{missing_labels => convert}, Types)) ``` By calling `gmser_dyn:register_types/1`, after having added options to the type map, @@ -131,37 +131,68 @@ Templates can be provided to the encoder by either naming an already registered type, or by passing a template directly. In both cases, the encoder will enforce the type information in the template. -If the template has been registered, the encoder omits inner type tags (still -inserting the top-level tag), leading to some compression of the output. -This also means that the serialized term cannot be decoded without the same -schema information on the decoder side. - -In some cases, the type tags will still be emitted. These are when alternative types -appear, and for enumerated map types (`#{items => ...}`). In the latter case, it is -due to the support for optional items. - -In the case of a directly provided template, all type information is inserted, -such that the serialized term can be decoded without any added type information. -The template types are still enforced during encoding. +If the template has been registered, the encoder uses the registered type specification +to drive the encoding. The code of the registered template is embedded in the encoded +output: ```erlang -ET = fun(Type,Term) -> io:fwrite("~w~n", [gmser_dyn:encode_typed(Type,Term)]) end. +gmser_dyn:encode_typed({int,int,int}, {1,2,3}) -> + [<<0>>,<<1>>,[<<253>>, + [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]] -ET([{int,int}], [{1,2}]) -> [<<0>>,<<1>>,[<<251>>,[[[<<248>>,<<1>>],[<<248>>,<<2>>]]]]] - -gmser_dyn:register_type(1000,lt2i,[{int,int}]). -ET(lt2i, [{1,2}]) -> [<<0>>,<<1>>,[<<3,232>>,[[<<1>>,<<2>>]]]] +Types = gmser_dyn_types:add_type(t3,1013,{int,int,int}). +gmser_dyn:encode_typed(t3, {1,2,3}, Types) -> + [<<0>>,<<1>>,[[<<3,245>>,<<253>>], + [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>]]]] ``` +Note that the original `<<253>>` type code is wrapped as `[<<3,245>>,<<253>>]`, +where `<<3,245>>` corresponds to the custom code `1013`. + +Using the default option `#{strict => true}`, the decoder will extract the custom +type spec, and validate the encoded data against it. If the custom code is missing, +the decoder aborts. Using `#{strict => false}`, the custom code is used if it exists, +but otherwise, it's ignored, and the encoded data is decoded using the dynamic type +info. + ### Alternative types -The dynamic encoder supports two additions to the `gmserialization` template -language: `any`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`. +The dynamic encoder supports a few additions to the `gmserialization` template +language: `any`, `#{list => Type}`, `#{alt => [AltTypes]}` and `#{switch => [AltTypes]}`. #### `any` The `any` type doesn't have an associated code, but enforces dynamic encoding. +#### `list` + +The original list type notation expects a key-value list, e.g. + +`[{name, binary}, {age, int}]` + +```erlang +EL = gmser_dyn:encode_typed([{name,binary},{age,int}], [{name,<<"Ulf">>},{age,29}]) -> + [<<0>>,<<1>>,[<<251>>, + [[<<253>>,[[<<255>>,<<110,97,109,101>>],[<<249>>,<<85,108,102>>]]], + [<<253>>,[[<<255>>,<<97,103,101>>],[<<248>>,<<29>>]]]]]] +``` +Note that the encoding explicitly lays out a `[{Key, Value}]` structure, all +dynamically typed. This means it can be dynamically decoded without templates. + +```erlang +gmser_dyn:decode(EL). +[{name,<<"Ulf">>},{age,29}] +``` + +In order to specify something like Erlang's `[integer()]` type, we can use +the following: + +```erlang +gmser_dyn:encode_typed(#{list => int}, [1,2,3,4]) -> + [<<0>>,<<1>>,[<<251>>, + [[<<248>>,<<1>>],[<<248>>,<<2>>],[<<248>>,<<3>>],[<<248>>,<<4>>]]]] +``` + #### `alt` The `#{alt => [Type]}` construct also enforces dynamic encoding, and will try diff --git a/src/gmser_dyn.erl b/src/gmser_dyn.erl index ded3219..9ca5ffc 100644 --- a/src/gmser_dyn.erl +++ b/src/gmser_dyn.erl @@ -29,6 +29,7 @@ , registered_types/1 %% -"- , add_types/1 %% Returns updated types , add_types/2 %% -"- + , add_types/3 %% -"- , latest_vsn/0 %% -"- , get_opts/1 , set_opts/1 @@ -44,12 +45,65 @@ -import(gmserialization, [ decode_field/2 ]). +-type tag() :: atom(). +-type code() :: pos_integer(). +-type basic_type() :: anyint + | negint + | int + | binary + | bool + | list + | map + | tuple + | id + | label. + +-type key() :: term(). +-type alt_type() :: #{alt := template()}. +-type list_type() :: #{list := template()}. +-type switch_type() :: #{switch := [{any(), template()}]}. +-type tuple_type() :: tuple(). +-type items_type() :: #{items := [{key(), template()} | {opt, key(), template()}]}. +-type fields_type() :: [{key(), template()}]. + +-type template() :: basic_type() + | alt_type() + | list_type() + | switch_type() + | tuple_type() + | items_type() + | fields_type(). + +-type types() :: #{ vsn := pos_integer() + , codes := #{code() => tag()} + , rev := #{tag() => code()} + , templates := #{tag() => template()} + , labels := #{atom() => code()} + , rev_labels := #{code() => atom()} + , options := #{atom() => any()} + }. + +-export_type([ tag/0 + , code/0 + , basic_type/0 + , key/0 + , alt_type/0 + , switch_type/0 + , list_type/0 + , tuple_type/0 + , items_type/0 + , fields_type/0 + , template/0 + , types/0 + ]). + -define(VSN, 1). -include_lib("kernel/include/logger.hrl"). -ifdef(TEST). --compile([export_all, nowarn_export_all]). +-compile(nowarn_export_all). +-compile(export_all). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -169,30 +223,80 @@ decode_typed(Type, Fields0, Vsn, Types0) -> error({illegal_encoding, Other}) end. -decode_typed_(Type, [CodeBin, Fields], Vsn, Types) when is_map(Type) -> +decode_([[TemplateCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) -> + TemplateCode = decode_basic(int, TemplateCodeBin), Code = decode_basic(int, CodeBin), - {Tag, Template} = template(Code, Vsn, Types), - Dyn = is_dyn_template(Tag), + case is_map_key(Code, Codes) of + true -> + Template = template(Code, Vsn, Types), + decode_from_template(Template, Code, Fld, Vsn, Types); + false -> + case option(strict, Types) of + true -> error({unknown_template, TemplateCode}); + false -> + Template = template(Code, Vsn, Types), + decode_from_template(Template, Code, Fld, Vsn, Types) + end + end; +decode_([CodeBin, Flds], Vsn, Types) when is_binary(CodeBin) -> + Code = decode_basic(int, CodeBin), + Template = template(Code, Vsn, Types), + decode_from_template(Template, Code, Flds, Vsn, Types). + +decode_typed_(Type, [[TCodeBin, CodeBin], Fld], Vsn, #{codes := Codes} = Types) -> + TCode = decode_basic(int, TCodeBin), + Code = decode_basic(int, CodeBin), + case maps:find(TCode, Codes) of + {ok, Type} -> + TTemplate = template(TCode, Vsn, Types), + decode_from_template(TTemplate, Code, Fld, Vsn, Types); + {ok, TType} -> + Template = type_to_template(Type, Vsn, Types), + case is_subtype(TType, Template) of + true -> + decode_from_template(Template, Code, Fld, Vsn, Types); + false -> + TTemplate = template(TCode, Vsn, Types), + T1 = decode_from_template(TTemplate, Code, Fld, Vsn, Types), + T2 = decode_from_template(Template, Code, Fld, Vsn, Types), + if T1 =:= T2 -> T1; + true -> + error(badarg) + end + end; + error -> + case option(strict, Types) of + true -> + error(missing_template); + false -> + decode_typed_(Type, Code, Fld, Vsn, Types) + end + end; +decode_typed_(Type, [CodeBin, Fld], Vsn, Types) when is_binary(CodeBin) -> + Code = decode_basic(int, CodeBin), + decode_typed_(Type, Code, Fld, Vsn, Types). + +decode_typed_(Type, Code, Fld, Vsn, Types) when is_map(Type) -> + Template = template(Code, Vsn, Types), case {Type, Template} of - {#{items := _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); - {#{items := _}, items} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); - {#{alt := _}, _} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); - {#{switch:= _}, map} -> decode_from_template(Type, Code, Fields, Dyn, Vsn, Types); + {#{items := _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{items := _}, items} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{alt := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{switch:= _}, map} -> decode_from_template(Type, Code, Fld, Vsn, Types); + {#{list := _}, _} -> decode_from_template(Type, Code, Fld, Vsn, Types); _ -> error(badarg) end; -decode_typed_(Type, [CodeBin, Fields], Vsn, Types) -> - Code = decode_basic(int, CodeBin), - {_, Template} = template(Type, Vsn, Types), +decode_typed_(Type, Code, Fld, Vsn, Types) when is_tuple(Type); is_list(Type) -> + decode_from_template(Type, Code, Fld, Vsn, Types); +decode_typed_(Type, Code, Fld, Vsn, Types) -> + Template = template(Type, Vsn, Types), case template(Code, Vsn, Types) of - {Tag, Template} -> - Dyn = is_dyn_template(Tag), - decode_from_template(Template, Code, Fields, Dyn, Vsn, Types); + Template -> + decode_from_template(Template, Code, Fld, Vsn, Types); _ -> error(badarg) - end; -decode_typed_(_, _, _, _) -> - error(illegal_encoding). + end. decode_tag_and_vsn([TagBin, VsnBin, Fields]) -> {decode_basic(int, TagBin), @@ -227,52 +331,54 @@ assert_vsn(V, #{} = Types ) -> Types#{vsn => V}. -define(ID, 254). -define(LABEL, 255). +-define(CODES, #{ 246 => anyint + , 247 => negint + , 248 => int + , 249 => binary + , 250 => bool + , 251 => list + , 252 => map + , 253 => tuple + , 254 => id + , 255 => label}). +-define(REV, #{ anyint => ?ANYINT + , negint => ?NEGINT + , int => ?INT + , binary => ?BINARY + , bool => ?BOOL + , list => ?LIST + , map => ?MAP + , tuple => ?TUPLE + , id => ?ID + , label => ?LABEL }). +-define(TEMPLATES, #{ anyint => #{alt => [negint, int]} + , negint => negint + , int => int + , binary => binary + , bool => bool + , list => list + , map => map + , tuple => tuple + , id => id + , label => label + }). +-define(OPTIONS, #{ missing_labels => fail + , strict => true }). + dynamic_types() -> #{ vsn => ?VSN - , codes => - #{ 246 => anyint - , 247 => negint - , 248 => int - , 249 => binary - , 250 => bool - , 251 => list - , 252 => map - , 253 => tuple - , 254 => id - , 255 => label} - , rev => - #{ anyint => ?ANYINT - , negint => ?NEGINT - , int => ?INT - , binary => ?BINARY - , bool => ?BOOL - , list => ?LIST - , map => ?MAP - , tuple => ?TUPLE - , id => ?ID - , label => ?LABEL } + , codes => ?CODES + , rev => ?REV , labels => #{} , rev_labels => #{} - , templates => - #{ anyint => #{alt => [negint, int]} - , negint => negint - , int => int - , binary => binary - , bool => bool - , list => list - , map => map - , tuple => tuple - , id => id - , label => label - } - , options => #{} + , templates => ?TEMPLATES + , options => ?OPTIONS }. -is_dyn_template(T) -> - is_dyn_template_(T, dynamic_types()). +is_custom_template(T) -> + not is_core_template(T). -is_dyn_template_(T, #{templates := Ts}) -> - is_map_key(T, Ts). +is_core_template(T) -> is_map_key(T, ?TEMPLATES). registered_types() -> registered_types(latest_vsn()). @@ -287,87 +393,123 @@ registered_types(Vsn) -> dynamic_types() end. +type_to_template(T, Vsn, Types) when is_atom(T); is_integer(T) -> + template(T, Vsn, Types); +type_to_template(Type, _Vsn, _Types) -> + Type. + +is_subtype(T, T) -> true; +is_subtype(map, #{items := _}) -> true; +is_subtype(map, #{switch := _}) -> true; +is_subtype(tuple, T) when is_tuple(T) -> true; +is_subtype(list, L) when is_list(L) -> true; +is_subtype(_, #{alt := _}) -> true; +is_subtype(anyint, int) -> true; +is_subtype(_, _) -> + false. + +template(any, _, _) -> any; template(TagOrCode, Vsn, Types) -> - {Tag, Template} = get_template(TagOrCode, Types), - {Tag, dyn_template_(Template, Vsn)}. + Template = get_template(TagOrCode, Types), + dyn_template_(Template, Vsn). get_template(Code, #{codes := Codes, templates := Ts}) when is_integer(Code) -> Tag = maps:get(Code, Codes), - {Tag, maps:get(Tag, Ts)}; + maps:get(Tag, Ts); get_template(Tag, #{templates := Ts}) when is_atom(Tag) -> - {Tag, maps:get(Tag, Ts)}. + maps:get(Tag, Ts). -dyn_template_(F, Vsn) -> - if is_function(F, 0) -> F(); - is_function(F, 1) -> F(Vsn); - true -> F - end. +dyn_template_(T, Vsn) -> + T1 = if is_function(T, 0) -> T(); + is_function(T, 1) -> T(Vsn); + true -> T + end, + resolved_template(T1). + +resolved_template(T) -> + if + is_map(T) -> T; + is_list(T) -> T; + is_tuple(T) -> T; + is_atom(T) -> + case is_core_template(T) of + true -> T; + false -> + error(unresolved_template) + end + end. find_cached_label(Lbl, #{labels := Lbls}) -> maps:find(Lbl, Lbls). -decode_([CodeBin, Flds], Vsn, Types) -> - Code = decode_basic(int, CodeBin), - {Tag, Template} = template(Code, Vsn, Types), - Dyn = is_dyn_template(Tag), - decode_from_template(Template, Code, Flds, Dyn, Vsn, Types). - encode_(Term, Vsn, Types) -> - encode_(Term, dyn(emit()), Vsn, Types). - -encode_(Term, E, Vsn, Types) -> - {_Tag, Template} = auto_template(Term), - encode_from_template(Template, Term, E, Vsn, Types). + Template = auto_template(Term), + encode_from_template(Template, Term, Vsn, Types). %% To control when to emit type codes: %% If the template is predefined, it's 'not dynamic' (nodyn(E)). %% If we are encoding against a type that's part of a predefined template, %% we typically don't emit the type code, except at the very top. %% So: emit type codes if the 'emit' bit is set, or if the 'dyn' bit is set. -emit() -> 2#01. -dyn() -> 2#10. -emit(E) -> E bor 2#01. -noemit(E) -> E band 2#10. -dyn(E) -> E bor 2#10. -nodyn(E) -> E band 2#01. +%% emit() -> 2#01. +%% dyn() -> 2#10. +%% emit(E) -> E bor 2#01. +%% noemit(E) -> E band 2#10. +%% dyn(E) -> E bor 2#10. +%% nodyn(E) -> E band 2#01. encode_typed_(Type, Term, Vsn, #{codes := Codes, rev := Rev} = Types) -> case (is_map_key(Type, Codes) orelse is_map_key(Type, Rev)) of true -> - encode_typed_(Type, Term, nodyn(emit()), Vsn, Types); + encode_typed_1(Type, Term, Vsn, Types); false -> encode_maybe_template(Type, Term, Vsn, Types) end. -encode_typed_(any, Term, _, Vsn, Types) -> - encode_(Term, dyn(emit()), Vsn, Types); -encode_typed_(Code, Term, E, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) -> - {_Tag, Template} = template(Code, Vsn, Types), - [encode_basic(int,Code), - encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)]; -encode_typed_(Tag, Term, E, Vsn, #{templates := Ts, rev := Rev} = Types) +encode_typed_1(any, Term, Vsn, Types) -> + encode_(Term, Vsn, Types); +encode_typed_1(Code, Term, Vsn, #{codes := Codes} = Types) when is_map_key(Code, Codes) -> + Tag = maps:get(Code, Codes), + Template = template(Code, Vsn, Types), + Fld = encode_from_template(Template, Term, Vsn, Types), + case is_custom_template(Tag) of + true -> + [CodeI, FldI] = Fld, + [[encode_basic(int, Code), CodeI], FldI]; + false -> + encode_from_template(Template, Term, Vsn, Types) + end; +encode_typed_1(Tag, Term, Vsn, #{templates := Ts, rev := Rev} = Types) when is_map_key(Tag, Ts) -> Template = dyn_template_(maps:get(Tag, Ts), Vsn), Code = maps:get(Tag, Rev), - [encode_basic(int,Code), - encode_from_template(Template, Term, noemit(nodyn(E)), Vsn, Types)]; -encode_typed_(MaybeTemplate, Term, _, Vsn, Types) -> + Fld = encode_from_template(Template, Term, Vsn, Types), + case is_custom_template(Tag) of + true -> + [CodeI, FldI] = Fld, + [[encode_basic(int,Code), CodeI], FldI]; + false -> + Fld + end; +encode_typed_1(MaybeTemplate, Term, Vsn, Types) -> encode_maybe_template(MaybeTemplate, Term, Vsn, Types). encode_maybe_template(#{items := _} = Type, Term, Vsn, Types) -> case is_map(Term) of true -> - encode_from_template(Type, Term, emit(dyn()), Vsn, Types); + encode_from_template(Type, Term, Vsn, Types); false -> error({invalid, Type, Term}) end; encode_maybe_template(#{alt := _} = Type, Term, Vsn, Types) -> - encode_from_template(Type, Term, Vsn, emit(dyn()), Types); + encode_from_template(Type, Term, Vsn, Types); encode_maybe_template(#{switch := _} = Type, Term, Vsn, Types) -> - encode_from_template(Type, Term, Vsn, emit(dyn()), Types); + encode_from_template(Type, Term, Vsn, Types); +encode_maybe_template(#{list := _} = Type, Term, Vsn, Types) -> + encode_from_template(Type, Term, Vsn, Types); encode_maybe_template(Pat, Term, Vsn, Types) when is_list(Pat); is_tuple(Pat) -> - encode_from_template(Pat, Term, emit(dyn()), Vsn, Types); + encode_from_template(Pat, Term, Vsn, Types); encode_maybe_template(Other, Term, _Vsn, _Types) -> error({illegal_template, Other, Term}). @@ -378,67 +520,72 @@ auto_template({id,Tag,V}) when Tag == account ; Tag == channel ; Tag == associate_chain ; Tag == entry -> - if is_binary(V) -> {id, id}; + if is_binary(V) -> id; true -> %% close, but no cigar - {tuple, tuple} + tuple end; auto_template(T) -> - if is_map(T) -> {map, map}; - is_list(T) -> {list, list}; - is_tuple(T) -> {tuple, tuple}; - is_binary(T) -> {binary, binary}; - is_boolean(T) -> {bool, bool}; - is_atom(T) -> {label, label}; % binary_to_existing_atom() + if is_map(T) -> map; + is_list(T) -> list; + is_tuple(T) -> tuple; + is_binary(T) -> binary; + is_boolean(T) -> bool; + is_atom(T) -> label; % binary_to_existing_atom() is_integer(T), - T >= 0 -> {int, int}; + T >= 0 -> int; is_integer(T), - T < 0 -> {negint, negint}; + T < 0 -> negint; true -> error({invalid_type, T}) end. -decode_from_template(any, _, Fld, _, Vsn, Types) -> +decode_from_template(any, _Code, Fld, Vsn, Types) -> decode_(Fld, Vsn, Types); -decode_from_template(#{items := Items}, _, Fld, _Dyn, Vsn, Types) when is_list(Fld) -> +decode_from_template(#{items := Items}, _, Fld, Vsn, Types) when is_list(Fld) -> Zipped = dec_zip_items(Items, Fld, Vsn, Types), lists:foldl( - fun({K, Type, Code, V}, Map) -> - maps:is_key(K, Map) andalso error(badarg, duplicate_key), - Map#{K => decode_from_template(Type, Code, V, true, Vsn, Types)} + fun({K, Type, V}, Map) -> + case maps:is_key(K, Map) of + true -> error(duplicate_key); + false -> + Map#{K => decode_typed_(Type, V, Vsn, Types)} + end end, #{}, Zipped); -decode_from_template(#{alt := Alts} = T, Code, Fld, _, Vsn, Types) when is_list(Alts) -> +decode_from_template(#{alt := Alts} = T, Code, Fld, Vsn, Types) when is_list(Alts) -> decode_alt(Alts, Code, Fld, T, Vsn, Types); -decode_from_template(#{switch := Alts} = T, Code, Fld, Dyn, Vsn, Types) when is_map(Alts) -> - decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types); -decode_from_template(list, _, Flds, _, Vsn, Types) -> +decode_from_template(#{switch := Alts} = T, Code, Fld, Vsn, Types) when is_map(Alts) -> + decode_switch(Alts, Code, Fld, T, Vsn, Types); +decode_from_template(#{list := Type}, ?LIST, Fld, Vsn, Types) -> + [decode_typed_(Type, F, Vsn, Types) || F <- Fld]; +decode_from_template(list, _, Flds, Vsn, Types) -> [decode_(F, Vsn, Types) || F <- Flds]; -decode_from_template(map, _, Fld, _, Vsn, Types) -> - TupleFields = [F || F <- Fld], - Items = [decode_from_template({any,any}, ?TUPLE, T, true, Vsn, Types) - || T <- TupleFields], +decode_from_template(map, ?MAP, TupleFields, Vsn, Types) -> + Items = lists:map(fun([Ke, Ve]) -> + {decode_(Ke, Vsn, Types), decode_(Ve, Vsn, Types)} + end, TupleFields), maps:from_list(Items); -decode_from_template(tuple, _, Fld, _, Vsn, Types) -> +decode_from_template(tuple, _, Fld, Vsn, Types) -> Items = [decode_(F, Vsn, Types) || F <- Fld], list_to_tuple(Items); -decode_from_template([Type], _, Fields, _Dyn, Vsn, Types) -> - [decode_typed_(Type, F, Vsn, Types) - || F <- Fields]; -decode_from_template(Type, _, V, Dyn, Vsn, Types) when is_list(Type), is_list(V) -> - decode_fields(Type, V, Dyn, Vsn, Types); -decode_from_template(Type, _, V, _Dyn, Vsn, Types) when is_tuple(Type), is_list(V) -> +decode_from_template(Type, _, Fields, Vsn, Types) when is_list(Type) -> + decode_fields(Type, Fields, Vsn, Types); + %% Zipped = lists:zip(Type, Fields, fail), + %% [decode_typed_(T, F, Vsn, Types) + %% || {T, F} <- Zipped]; +decode_from_template(Type, _, V, Vsn, Types) when is_tuple(Type) -> Zipped = lists:zip(tuple_to_list(Type), V), Items = [decode_typed_(T1, V1, Vsn, Types) || {T1, V1} <- Zipped], list_to_tuple(Items); -decode_from_template(label, _, [C], _, _, #{rev_labels := RLbls}) -> +decode_from_template(label, ?LABEL, [C], _, #{rev_labels := RLbls}) -> Code = decode_basic(int, C), maps:get(Code, RLbls); -decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int - ; Type == negint - ; Type == binary - ; Type == bool - ; Type == id - ; Type == label -> +decode_from_template(Type, Code, Fld, Vsn, Types) when Type == int + ; Type == negint + ; Type == binary + ; Type == bool + ; Type == id + ; Type == label -> case template(Code, Vsn, Types) of Type -> decode_basic(Type, Fld, Types); @@ -446,45 +593,44 @@ decode_from_template(Type, Code, Fld, _, Vsn, Types) when Type == int error(badarg) end. -dec_zip_items([{K, T}|Is], [{K1, KEnc, VEnc}|Fs], Vsn, Types) -> +dec_zip_items([{K, T}|Is], [{K1, VEnc}|Fs], Vsn, Types) -> if K =:= K1 -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; true -> error(illegal_map) end; dec_zip_items([{K, T}|Is], [[KEnc, VEnc]|Fs], Vsn, Types) -> case decode_(KEnc, Vsn, Types) of K -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; _ -> error(illegal_map) end; -dec_zip_items([{opt, K, T}|Is], [{K, KEnc, VEnc}|Fs], Vsn, Types) -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; +dec_zip_items([{opt, K, T}|Is], [{K, VEnc}|Fs], Vsn, Types) -> + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; dec_zip_items([{opt, K, T}|Is], [[KEnc,VEnc]|Fs], Vsn, Types) -> case decode_(KEnc, Vsn, Types) of K -> - [{K, T, KEnc, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; + [{K, T, VEnc} | dec_zip_items(Is, Fs, Vsn, Types)]; OtherK -> - dec_zip_items(Is, [{OtherK, KEnc, VEnc}|Fs], Vsn, Types) + dec_zip_items(Is, [{OtherK, VEnc}|Fs], Vsn, Types) end; dec_zip_items([], [], _, _) -> []. -encode_from_template(any, V, _E, Vsn, Types) -> - encode_(V, dyn(emit()), Vsn, Types); -encode_from_template(list, L, E, Vsn, Types) when is_list(L) -> +encode_from_template(any, V, Vsn, Types) -> + encode_(V, Vsn, Types); +encode_from_template(list, L, Vsn, Types) when is_list(L) -> assert_type(is_list(L), list, L), - emit(E, list, Types, + emit(list, Types, [encode_(V, Vsn, Types) || V <- L]); -encode_from_template(#{items := Items}, M, E, Vsn, Types) -> +encode_from_template(#{items := Items}, M, Vsn, Types) -> assert_type(is_map(M), map, M), - Emit = noemit(E), Encode = fun(K, Type, V) -> - [encode_from_template(any, K, Emit, Vsn, Types), - encode_from_template(Type, V, Emit, Vsn, Types)] + [encode_from_template(any, K, Vsn, Types), + encode_from_template(Type, V, Vsn, Types)] end, - emit(E, map, Types, + emit(map, Types, lists:foldr( fun({K, Type}, Acc) -> V = maps:get(K, M), @@ -497,129 +643,150 @@ encode_from_template(#{items := Items}, M, E, Vsn, Types) -> Acc end end, [], Items)); -encode_from_template(#{alt := Alts} = T, Term, E, Vsn, Types) when is_list(Alts) -> - encode_alt(Alts, Term, T, E, Vsn, Types); -encode_from_template(#{switch := Alts} = T, Term, E, Vsn, Types) when is_map(Alts), - is_map(Term) -> - encode_switch(Alts, Term, T, E, Vsn, Types); -encode_from_template(map, M, E, Vsn, Types) -> +encode_from_template(#{alt := Alts} = T, Term, Vsn, Types) when is_list(Alts) -> + encode_alt(Alts, Term, T, Vsn, Types); +encode_from_template(#{switch := Alts} = T, Term, Vsn, Types) when is_map(Alts), + is_map(Term) -> + encode_switch(Alts, Term, T, Vsn, Types); +encode_from_template(map, M, Vsn, Types) -> assert_type(is_map(M), map, M), - Emit = emit(E), - emit(E, map, Types, - [[encode_from_template(any, K, Emit, Vsn, Types), - encode_from_template(any, V, Emit, Vsn, Types)] + emit(map, Types, + [[encode_from_template(any, K, Vsn, Types), + encode_from_template(any, V, Vsn, Types)] || {K, V} <- lists:sort(maps:to_list(M))]); -encode_from_template(tuple, T, E, Vsn, Types) -> +encode_from_template(tuple, T, Vsn, Types) -> assert_type(is_tuple(T), tuple, T), - emit(E, tuple, Types, - [encode_(V, noemit(E), Vsn, Types) || V <- tuple_to_list(T)]); -encode_from_template(T, V, E, Vsn, Types) when is_tuple(T) -> + emit(tuple, Types, + [encode_(V, Vsn, Types) || V <- tuple_to_list(T)]); +encode_from_template(T, V, Vsn, Types) when is_tuple(T) -> assert_type(is_tuple(V), T, V), assert_type(tuple_size(T) =:= tuple_size(V), T, V), Zipped = lists:zip(tuple_to_list(T), tuple_to_list(V)), - emit(E, tuple, Types, - [encode_from_template(T1, V1, noemit(E), Vsn, Types) || {T1, V1} <- Zipped]); -encode_from_template([Type] = T, List, E, Vsn, Types) -> + emit(tuple, Types, + [encode_from_template(T1, V1, Vsn, Types) || {T1, V1} <- Zipped]); +encode_from_template(#{list := Type} = T, List, Vsn, Types) -> assert_type(is_list(List), T, List), - emit(E, list, Types, - [encode_from_template(Type, V, noemit(E), Vsn, Types) || V <- List]); -encode_from_template(Type, List, E, Vsn, Types) when is_list(Type), is_list(List) -> - encode_fields(Type, List, E, Vsn, Types); -encode_from_template(label, V, E, _, Types) -> + emit(list, Types, + [encode_from_template(Type, V, Vsn, Types) || V <- List]); +encode_from_template(Type, List, Vsn, Types) when is_list(Type), is_list(List) -> + emit(list, Types, encode_fields(Type, List, Vsn, Types)); +encode_from_template(label, V, _, Types) -> assert_type(is_atom(V), label, V), case find_cached_label(V, Types) of error -> - encode_basic(label, V, E, Types); + encode_basic(label, V, Types); {ok, Code} when is_integer(Code) -> - emit(E, label, Types, + emit(label, Types, [encode_basic(int, Code)]) end; -encode_from_template(Type, V, E, _, Types) when Type == id - ; Type == binary - ; Type == bool - ; Type == int - ; Type == negint - ; Type == label -> - encode_basic(Type, V, E, Types); -encode_from_template(Type, V, E, Vsn, Types) -> - encode_typed_(Type, V, E, Vsn, Types). +encode_from_template(Type, V, _, Types) when Type == id + ; Type == binary + ; Type == bool + ; Type == int + ; Type == negint + ; Type == label -> + encode_basic(Type, V, Types); +encode_from_template(Type, V, Vsn, Types) -> + encode_typed_(Type, V, Vsn, Types). assert_type(true, _, _) -> ok; assert_type(_, Type, V) -> error({illegal, Type, V}). decode_alt([A|Alts], Code, Fld, T, Vsn, Types) -> - try decode_from_template(A, Code, Fld, true, Vsn, Types) + try decode_typed_(A, Code, Fld, Vsn, Types) catch error:_ -> decode_alt(Alts, Code, Fld, T, Vsn, Types) end; decode_alt([], _Code, Fld, T, _, _) -> error({illegal, T, Fld}). -encode_alt(Alts, Term, T, E, Vsn, Types) -> +encode_alt(Alts, Term, T, Vsn, Types) -> %% Since we don't know which type may match, treat as dynamic. - encode_alt_(Alts, Term, T, dyn(E), Vsn, Types). + encode_alt_(Alts, Term, T, Vsn, Types). -encode_alt_([A|Alts], Term, T, E, Vsn, Types) -> - try encode_from_template(A, Term, E, Vsn, Types) +encode_alt_([A|Alts], Term, T, Vsn, Types) -> + try encode_from_template(A, Term, Vsn, Types) catch error:_ -> - encode_alt_(Alts, Term, T, E, Vsn, Types) + encode_alt_(Alts, Term, T, Vsn, Types) end; -encode_alt_([], Term, T, _, _, _) -> +encode_alt_([], Term, T, _, _) -> error({illegal, T, Term}). -decode_switch(Alts, Code, Fld, T, Dyn, Vsn, Types) -> - [[KFld, VFld]] = Fld, - Key = decode_(KFld, Vsn, Types), - case maps:find(Key, Alts) of - {ok, SubType} -> - SubTerm = decode_from_template(SubType, Code, VFld, Dyn, Vsn, Types), - #{Key => SubTerm}; - error -> +decode_switch(Alts, Code, Fld, T, Vsn, Types) -> + case is_map_type(Code, Vsn, Types) of + true -> + case Fld of + [[KFld, VFld]] -> + Key = decode_(KFld, Vsn, Types), + case maps:find(Key, Alts) of + {ok, SubType} -> + SubTerm = decode_typed_(SubType, VFld, Vsn, Types), + #{Key => SubTerm}; + error -> + error({illegal, T, Fld}) + end; + _ -> + error({illegal, T, Fld}) + end; + false -> error({illegal, T, Fld}) end. -encode_switch(Alts, Term, T, E, Vsn, Types) -> +is_map_type(Code, Vsn, Types) -> + case template(Code, Vsn, Types) of + map -> true; + #{items := _} -> true; + #{switch := _} -> true; + T -> + case maps:get(T, maps:get(templates, Types)) of + map -> true; + T -> false; + Other when is_atom(Other) -> + is_map_type(T, Vsn, Types); + _ -> false + end + end. + +encode_switch(Alts, Term, T, Vsn, Types) -> assert_type(map_size(Term) == 1, singleton_map, Term), [{Key, Subterm}] = maps:to_list(Term), case maps:find(Key, Alts) of {ok, SubType} -> - Enc = encode_from_template(SubType, Subterm, E, Vsn, Types), - emit(E, map, Types, - [[encode_from_template(any, Key, E, Vsn, Types), - Enc]]); + Enc = encode_from_template(SubType, Subterm, Vsn, Types), + emit(map, Types, + [[encode_from_template(any, Key, Vsn, Types), Enc]]); error -> error({illegal, T, Term}) end. %% Basically, dynamically encoding a statically defined object encode_fields([{Field, Type}|TypesLeft], - [{Field, Val}|FieldsLeft], E, Vsn, Types) -> - [ encode_from_template(Type, Val, E, Vsn, Types) - | encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)]; + [{Field, Val}|FieldsLeft], Vsn, Types) -> + KType = auto_template(Field), + [ encode_from_template({KType, Type}, {Field, Val}, Vsn, Types) + | encode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; encode_fields([{_Field, _Type} = FT|_TypesLeft], - [Val |_FieldsLeft], _E, _Vsn, _Types) -> + [Val |_FieldsLeft], _Vsn, _Types) -> error({illegal_field, FT, Val}); encode_fields([Type|TypesLeft], - [Val |FieldsLeft], E, Vsn, Types) when is_atom(Type) -> + [Val |FieldsLeft], Vsn, Types) when is_atom(Type) -> %% Not sure about this ... - [ encode_from_template(Type, Val, E, Vsn, Types) - | encode_fields(TypesLeft, FieldsLeft, E, Vsn, Types)]; -encode_fields([], [], _, _, _) -> + [ encode_from_template(Type, Val, Vsn, Types) + | encode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; +encode_fields([], [], _, _) -> []. decode_fields([{Tag, Type}|TypesLeft], - [Field |FieldsLeft], Dyn, Vsn, Types) -> + [Field |FieldsLeft], Vsn, Types) -> - [ {Tag, decode_from_template(Type, 0, Field, Dyn, Vsn, Types)} - | decode_fields(TypesLeft, FieldsLeft, Dyn, Vsn, Types)]; -decode_fields([], [], _, _, _) -> + [ {Tag, decode_from_template(Type, 0, Field, Vsn, Types)} + | decode_fields(TypesLeft, FieldsLeft, Vsn, Types)]; +decode_fields([], [], _, _) -> []. -emit(E, Tag, Types, Enc) when E > 0 -> - [emit_code(Tag, Types), Enc]; -emit(0, _, _, Enc) -> - Enc. +emit(Tag, Types, Enc) -> + [emit_code(Tag, Types), Enc]. emit_code(Tag, #{rev := Tags}) -> encode_basic(int, maps:get(Tag, Tags)). @@ -656,12 +823,10 @@ decode_basic(negint, Fld) -> decode_basic(Type, Fld) -> gmserialization:decode_field(Type, Fld). -encode_basic(negint, I, _, Types) when is_integer(I), I < 0 -> +encode_basic(negint, I, Types) when is_integer(I), I < 0 -> [emit_code(negint, Types), gmserialization:encode_field(int, -I)]; -encode_basic(Tag, V, E, Types) when E > 0 -> - [emit_code(Tag, Types), encode_basic(Tag, V)]; -encode_basic(Tag, V, 0, _) -> - encode_basic(Tag, V). +encode_basic(Tag, V, Types) -> + [emit_code(Tag, Types), encode_basic(Tag, V)]. encode_basic(label, A) when is_atom(A) -> encode_basic(binary, atom_to_binary(A, utf8)); @@ -681,28 +846,34 @@ register_types(Types) when is_map(Types) -> register_types(latest_vsn(), Types). register_types(Vsn, Types) -> - Result = add_types(Types), + Result = add_types(Vsn, Types), put_types(Vsn, Result). add_types(Types) -> - add_types(Types, dynamic_types()). + add_types(?VSN, Types). -add_types(Types, PrevTypes) -> +add_types(Vsn, Types) -> + add_types(Vsn, Types, dynamic_types()). + +add_types(Vsn, Types, PrevTypes) -> Codes = maps:get(codes, Types, #{}), Rev = rev_codes(Codes), Templates = maps:get(templates, Types, #{}), Labels = maps:get(labels, Types, #{}), + RevLabels = rev_codes(Labels), Options = maps:get(options, Types, #{}), - #{codes := Codes0, rev := Rev0, labels := Labels0, + #{codes := Codes0, rev := Rev0, labels := Labels0, rev_labels := RevLabels0, templates := Templates0, options := Options0} = PrevTypes, Merged = #{ codes => maps:merge(Codes0, Codes) , rev => maps:merge(Rev0, Rev) , templates => maps:merge(Templates0, Templates) , options => maps:merge(Options0, Options) - , labels => maps:merge(Labels0, Labels) }, + , labels => maps:merge(Labels0, Labels) + , rev_labels => maps:merge(RevLabels0, RevLabels) }, assert_sizes(Merged), assert_mappings(Merged), - assert_label_cache(Merged). + assert_label_cache(Merged), + assert_resolved_templates(Vsn, Merged). latest_vsn() -> case persistent_term:get(pt_key(), undefined) of @@ -762,6 +933,18 @@ set_opts(Opts) -> set_opts(Opts, Types) -> Types#{options => Opts}. +option(O, #{options := Opts}) -> + case Opts of + #{O := V} -> V; + _ -> default_option(O) + end. + +default_option(O) -> + case dynamic_types() of + #{options := #{O := V}} -> V; + _ -> undefined + end. + get_opts(#{options := Opts}) -> Opts. @@ -821,6 +1004,10 @@ assert_mappings(#{rev := Rev, templates := Ts} = Types) -> error({missing_templates, Missing, Types}) end. +assert_resolved_templates(Vsn, #{templates := Ts} = Types) -> + _ = [template(T, Vsn, Types) || T <- maps:keys(Ts)], + Types. + assert_label_cache(#{labels := Labels} = Types) -> Ls = maps:keys(Labels), case [L || L <- Ls, not is_atom(L)] of @@ -901,6 +1088,7 @@ user_types_test_() -> dynamic_types_test_() -> [ ?_test(revert_to_default_types()) , ?_test(t_typed_map()) + , ?_test(t_typed_list()) , ?_test(t_alts()) , ?_test(t_switch()) , ?_test(t_anyints()) @@ -985,7 +1173,7 @@ t_bad_typed_encode(Type, Term, Error) -> end. t_reg_chain_objects_array() -> - Template = [{foo, {int, binary}}, {bar, [{int, int}]}, {baz, {int}}], + Template = [{foo, {int, binary}}, {bar, #{list => {int, int}}}, {baz, {int}}], ?debugFmt("Template = ~p", [Template]), MyTypes = #{ codes => #{ 1002 => coa } , templates => #{ coa => Template } }, @@ -1042,7 +1230,7 @@ t_reg_label_cache2() -> register_types(TFromL), Tup = {'1', '1'}, Enc = gmser_dyn:encode_typed(lbl_tup2, Tup), - [<<0>>,<<1>>,[<<3,235>>,[[<<49>>],[<<49>>]]]] = Enc, + [<<0>>,<<1>>,[[<<3,235>>,<>],[[<>,[<<49>>]],[<>,[<<49>>]]]]] = Enc, _Tup = gmser_dyn:decode(Enc). t_reg_map() -> @@ -1053,11 +1241,7 @@ t_reg_map() -> }, register_types(Types), Enc0 = gmser_dyn:encode_typed(my_map, #{a => foo, b => 17}), - dbg:tracer(), - dbg:tpl(?MODULE,x), - dbg:p(all,[c]), - #{a := foo, b := 17} = try gmser_dyn:decode(Enc0) - after dbg:stop() end, + #{a := foo, b := 17} = gmser_dyn:decode(Enc0), ok. t_reg_options() -> @@ -1080,6 +1264,11 @@ t_typed_map() -> ?assertEqual(Term1, decode(Enc1)), ?assertEqual(Enc, encode_typed(#{items => Items}, Term1)). +t_typed_list() -> + Term = [1,2,3,4], + encode_typed(#{list => int}, Term), + ok. + t_alts() -> t_round_trip_typed(#{alt => [negint, int]}, -4), t_round_trip_typed(#{alt => [negint, int]}, 4), diff --git a/src/gmser_dyn_types.erl b/src/gmser_dyn_types.erl index 6117d6d..1a41f1c 100644 --- a/src/gmser_dyn_types.erl +++ b/src/gmser_dyn_types.erl @@ -1,6 +1,7 @@ -module(gmser_dyn_types). --export([ add_type/4 +-export([ add_type/3 %% (Tag, Code, Template) -> Types1 + , add_type/4 %% (Tag, Code, Template, Types) -> Types1 , from_list/2 , expand/1 ]). -export([ next_code/1 ]). @@ -8,6 +9,14 @@ next_code(#{codes := Codes}) -> lists:max(maps:keys(Codes)) + 1. +-spec add_type(Tag, Code, Template) -> Types + when Tag :: gmser_dyn:tag() + , Code :: gmser_dyn:code() + , Template :: gmser_dyn:template() + , Types :: gmser_dyn:types(). +add_type(Tag, Code, Template) -> + add_type(Tag, Code, Template, gmser_dyn:registered_types()). + add_type(Tag, Code, Template, Types) -> elem_to_type({Tag, Code, Template}, Types).