From a38d007012a8124de6fbe85af46cc13f439a1a47 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Wed, 13 May 2026 20:43:01 +0200 Subject: [PATCH] Validator extensions, zomp vsn 0.2.0 --- README.md | 47 ++++ ebin/gmconfig.app | 2 +- src/gmconfig.app.src | 2 +- src/gmconfig.app.src.script | 14 + src/gmconfig.erl | 3 +- src/gmconfig_schema_helpers.erl | 1 + src/gmconfig_schema_utils.erl | 250 +++++++++++++---- test/data/nested_refs_schema.json | 29 ++ test/data/recursion_data.json | 20 ++ test/data/recursion_data_faulty.json | 20 ++ test/data/recursion_schema.json | 10 + test/data/ref_loop_schema.json | 6 + test/gmconfig_schema_utils_tests.erl | 404 +++++++++++++++++++++++++++ zomp.meta | 2 +- 14 files changed, 756 insertions(+), 54 deletions(-) create mode 100644 src/gmconfig.app.src.script create mode 100644 test/data/nested_refs_schema.json create mode 100644 test/data/recursion_data.json create mode 100644 test/data/recursion_data_faulty.json create mode 100644 test/data/recursion_schema.json create mode 100644 test/data/ref_loop_schema.json create mode 100644 test/gmconfig_schema_utils_tests.erl diff --git a/README.md b/README.md index 835f006..6f3e1db 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,9 @@ management subsystem. It is based on JSON-Schema, and includes, among other thin * Caching of the user config (and schema) as persistent terms * Fast config lookups using key paths * Lookups can handle both schema defaults and user-provided defaults +* Optional type coercion during validation +* Optional conversion of enums to atoms +* Optional extensions with custom validator funs ## JSON-Schema validator @@ -78,3 +81,47 @@ there are surely other things that are unsupported. * `"merge"` (for objects, keeps and/or updates existing values) * `"suggest"` (adds value if not already present) +### Custom validation options + +Using `gmconfig_schema_utils:validate(Json, Schema, Opts)`, a few options are supported +to further enhance the validation (`Opts` is of type `map()`): + +`coerce => boolean()` converts strings to integers, null and booleans when needed. +`enums_to_atoms => boolean()` converts enum strings to atoms +`extensions => map()` supports mapping `x-...` properties to custom validators. + +#### Validator extensions + +See the following test case: +```erlang +t_nested_refs() -> + S = read("data/nested_refs_schema.json"), + F = fun(Str, #{<<"tags">> := Tags}) -> + true = lists:any( + fun(T) -> + nomatch =/= string:prefix(Str, T) + end, Tags) + end, + Opts = #{extensions => #{<<"x-serialization">> => F}}, + Vs = #{<<"tx">> => #{<<"from">> => <<"ak_good">>}}, + Vf = #{<<"tx">> => #{<<"from">> => <<"ac_bad">>}}, + validate(Vs, S, Opts), + fails(Vf, S, Opts, #{e => failing_schemas}), + ok. +``` +This simulates an encoding extension, where the example fun here simply checks +if tags specified under the `x-serialization` property are prefixes of the +given string value. + +In the test schema, we can see the following definition: +```json +"Pubkey": { + "type": "string", + "x-serialization": { + "tags": ["ak", "ct"] + } +``` + +Whenever the validator encounters an `x-...` property mapped to a validator fun, +this fun is called with the value and the schema part of the property. The return +value of the fun is ignored, and any normal return is treated as a validation success. diff --git a/ebin/gmconfig.app b/ebin/gmconfig.app index ff07263..fbcc3b0 100644 --- a/ebin/gmconfig.app +++ b/ebin/gmconfig.app @@ -1,6 +1,6 @@ {application,gmconfig, [{description,"Configuration management support"}, - {vsn,"0.1.0"}, + {vsn,"0.2.0"}, {registered,[]}, {applications,[kernel,stdlib,setup]}, {env,[]}, diff --git a/src/gmconfig.app.src b/src/gmconfig.app.src index 915905c..8c8e8f2 100644 --- a/src/gmconfig.app.src +++ b/src/gmconfig.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang; erlang-indent-level: 4; indent-tabs-mode: nil -*- {application, gmconfig, [{description, "Gajumaru configuration management support"}, - {vsn, "0.1.0"}, + {vsn, "zomp"}, {registered, []}, {applications, [ diff --git a/src/gmconfig.app.src.script b/src/gmconfig.app.src.script new file mode 100644 index 0000000..becbb46 --- /dev/null +++ b/src/gmconfig.app.src.script @@ -0,0 +1,14 @@ +%% -*- erlang-mode; erlang-indent-level: 4; indent-tabs-mode: nil -*- + +[{application, Name, Opts}] = CONFIG. +case lists:keyfind(vsn, 1, Opts) of + {vsn, "zomp"} -> + ZompMetaF = filename:join(filename:dirname(filename:dirname(SCRIPT)), "zomp.meta"), + {ok, ZMeta} = file:consult(ZompMetaF), + {_, {_, _, {Vmaj,Vmin,Vpatch}}} = lists:keyfind(package_id, 1, ZMeta), + VsnStr = unicode:characters_to_list(io_lib:fwrite("~w.~w.~w", [Vmaj, Vmin, Vpatch])), + Opts1 = lists:keyreplace(vsn, 1, Opts, {vsn, VsnStr}), + [{application, Name, Opts1}]; + _ -> + CONFIG +end. diff --git a/src/gmconfig.erl b/src/gmconfig.erl index de4db9f..338e432 100644 --- a/src/gmconfig.erl +++ b/src/gmconfig.erl @@ -7,6 +7,7 @@ %%% @end %%%------------------------------------------------------------------- -module(gmconfig). +-vsn("0.2.0"). -export([get_env/2, get_env/3]). @@ -100,7 +101,7 @@ mock_config() -> mock_config(Cfg) -> ensure_schema_loaded(), - store(Cfg, _Notify = false, _Mode = silent). + store(Cfg, Mode = silent). unmock_config() -> gmconfig_schema_utils:clear(), diff --git a/src/gmconfig_schema_helpers.erl b/src/gmconfig_schema_helpers.erl index bc1f9b8..4f09889 100644 --- a/src/gmconfig_schema_helpers.erl +++ b/src/gmconfig_schema_helpers.erl @@ -1,4 +1,5 @@ -module(gmconfig_schema_helpers). +-vsn("0.2.0"). -export( [ diff --git a/src/gmconfig_schema_utils.erl b/src/gmconfig_schema_utils.erl index a33c68a..e64a79a 100644 --- a/src/gmconfig_schema_utils.erl +++ b/src/gmconfig_schema_utils.erl @@ -1,5 +1,6 @@ %% -*- mode: erlang; erlang-indent-level: 4; indent-tabs-mode: nil -*- -module(gmconfig_schema_utils). +-vsn("0.2.0"). -export([get_config/0, @@ -7,20 +8,28 @@ get_schema/0, get_schema/1, %% (Default) set_schema/1, + use_schema/1, + use_schema/2, schema/1, %% (Path) schema/2, %% (Path, Schema) schema/3, %% (Path, Schema, Opts) clear/0, - expand_schema/1]). + expand_ref/2, + expand_schema/1, %% (Schema) %% expand whole schema + expand_schema/2]). %% (SubSchema, RootSchema) -export([ update_config/1 %% (Map) -> ok , merge/2 %% (Item1, Item2) -> Item3 , merge/3 %% (Item1, Item2, Schema) -> Item3 , valid/1 %% (Item) -> Item | error() , valid/2 %% (Item, Schema) -> Item | error() + , validate/3 %% (Item, Schema, Opts) -> Item | error(). ]). -export([in_properties/2]). +-export([normalize/0, + normalize/1]). + -type json_string() :: binary(). -type json_int() :: integer(). -type json_num() :: number(). @@ -37,15 +46,26 @@ -type schema() :: json(). +-type ext_fun() :: fun( (json(), schema()) -> any() | no_return() ). +-type extensions() :: #{ binary() => ext_fun() }. +-type options() :: #{coerce => boolean(), + enum_to_atom => boolean(), + extensions => extensions() }. + -record(st, { s :: schema() %% schema , r :: schema() %% root schema , p = [] , a = [] %% annotations , v :: json() | undefined %% value + , d = undefined :: list() | 'undefined' %% dynamic eval + , opts = #{} :: options() }). -type st() :: #st{}. +-export_type([ schema/0, json/0 ]). + +-include_lib("kernel/include/logger.hrl"). -spec set_schema(schema()) -> ok. set_schema(Schema) -> @@ -67,6 +87,26 @@ get_schema() -> get_schema(Default) -> persistent_term:get({?MODULE, '$schema'}, Default). +-spec use_schema(schema() | st()) -> st(). +use_schema(#st{} = St) -> St; +use_schema(S) -> #st{s = S, r = S}. + +use_schema(Schema, RootSchema) -> + #st{s = Schema, r = RootSchema}. + +normalize() -> + normalize(get_schema()). + +normalize(S) when is_map(S) -> + #{bin_key(K) => normalize(V) || K := V <- S}; +normalize(S) when is_list(S) -> + [normalize(Sx) || Sx <- S]; +normalize(S) -> + S. + +bin_key(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin_key(B) when is_binary(B) -> B. + clear() -> persistent_term:erase({?MODULE,'$schema'}), persistent_term:erase({?MODULE,'$config'}), @@ -138,12 +178,12 @@ schema_get(_, _, Default) -> %% let us pattern-match on a schema map %% all schemas that are not a map are converted to the empty map. -schema_map(Map) when is_map(Map) -> Map; -schema_map(_) -> #{}. +schema_map(Map, _) when is_map(Map) -> Map; +schema_map(_, _) -> #{}. -spec merge_(json(), json(), st()) -> json(). -merge_(A, B, #st{} = St) -> - Ss = schemas_from_dynamic_eval(A, St), +merge_(A, B, #st{} = St0) -> + {Ss, St} = schemas_from_dynamic_eval(A, St0#st{d = undefined}), case schema_prop(<<"readOnly">>, St, Ss, false) of true when B == null -> valid(A, St); @@ -163,7 +203,7 @@ merge_(A, B, St, Ss) -> end. update_semantics(A, St, Ss) -> - case maps:find(<<"$updateSemantics">>, schema_map(A)) of + case maps:find(<<"$updateSemantics">>, schema_map(A, St)) of {ok, _} = Ok -> {Ok, object}; error -> @@ -190,9 +230,9 @@ remove_props(O, Keys, Recurse) when is_map(O) -> remove_props(Other, _, _) -> Other. -get_type(#st{} = St, Value) -> - Ss = schemas_from_dynamic_eval(Value, St), - get_type(St, Ss, Value). +get_type(#st{} = St0, Value) -> + {Ss, St} = schemas_from_dynamic_eval(Value, St0), + {get_type(St, Ss, Value), St}. get_type(#st{} = St, Ss, Value) -> case any_schema_prop(<<"type">>, St, Ss) of @@ -272,7 +312,7 @@ update_object(A0, B, St, Ss) -> update_object_(New, Old, St, Ss) -> Dyn = acc_props(Ss), - SsOld = schemas_from_dynamic_eval(Old, St), + {SsOld, _} = schemas_from_dynamic_eval(Old, St#st{d = undefined}), PropSchemas = [{P, prop_schema(P, Dyn, St)} || P <- maps:keys(New)], try do_update_object(New, Old, St, PropSchemas) catch @@ -300,6 +340,55 @@ do_update_object(New, Old, St, PropSchemas) -> end, Old, PropSchemas), valid(Res, object, St). +validate(V, Schema, Opts) when is_map(Opts) -> + St0 = use_schema(Schema), + St = St0#st{opts = Opts}, + V1 = valid(V, St), + case Opts of + #{enum_to_atom := true} -> + convert_enums(V1, St); + _ -> + V1 + end. + +convert_enums(V, St0) when is_binary(V) -> + case get_type(St0, V) of + {string, St} -> + {Ss, St1} = schemas_from_dynamic_eval(V, St), + case any_schema_prop(<<"enum">>, St1, Ss) of + {ok, _} -> + binary_to_atom(V, utf8); + _ -> + V + end; + _ -> + V + end; +convert_enums(V, St0) when is_map(V) -> + {Ss, St} = schemas_from_dynamic_eval(V, St0), + Dyn = acc_props(Ss), + maps:map( + fun(P, Vp) -> + PSchema = prop_schema(P, Dyn, St), + convert_enums(Vp, push_path(P, s(PSchema, St))) + end, V); +convert_enums(V, St0) when is_list(V) -> + {Ss,St} = schemas_from_dynamic_eval(V, St0), + case any_schema_prop(<<"items">>, St, Ss) of + {ok, Is} -> + [convert_enums(Vi, push_path(items, s(Is, St))) + || Vi <- V]; + error -> + case any_schema_prop(<<"prefixItems">>, St, Ss) of + {ok, PfxIs} -> + [convert_enums(Vi, push_path(prefixItems, s(PfxIs, St))) + || Vi <- V]; + error -> + V + end + end; +convert_enums(V, _) -> + V. valid(V) -> valid(V, get_schema()). @@ -311,21 +400,21 @@ valid(V, Schema) -> valid_(V, #st{s = true}) -> V; valid_(_, #st{s = false} = St) -> fail(invalid, St); -valid_(V, St) -> - Type = get_type(St, V), +valid_(V, St0) -> + {Type, St} = get_type(St0, V), valid(V, Type, St). valid(V, _, #st{s = true}) -> V; valid(_, _, #st{s = false} = St) -> fail(invalid, St); -valid(V, Type, St) -> +valid(V, Type, St0) -> %% We run dynamic eval to find conditional parts of the schema. %% we keep these in a separate list. - Ss = schemas_from_dynamic_eval(V, St), + {Ss,St} = schemas_from_dynamic_eval(V, St0), valid(V, Type, St, Ss). valid(V, Type, St, Ss) -> - valid_const(V, Type, St, Ss), - valid_enum(V, Type, St, Ss), + _ = valid_const(V, Type, St, Ss), + _ = valid_enum(V, Type, St, Ss), %% Dynamic eval returns a list of matching schemas %% We pass them along as they may contain properties, %% but `V` has already been validated against them. @@ -342,7 +431,7 @@ valid(V, Type, St, Ss) -> split_valid(V, St, Ss) -> split_valid(V, 0, St, Ss, [], []). split_valid(V, Ix, St, [S|Ss], Yes, No) -> - try valid(V, push_path(Ix, St#st{s = S})) of + try valid(V, push_path(Ix, s(S, St))) of _ -> split_valid(V, Ix+1, St, Ss, [{Ix,S}|Yes], No) catch error:Err -> @@ -370,7 +459,8 @@ valid_enum(V, Type, St, Ss) -> case lists:any(fun(X) -> is_equal(Type, V, X) end, En) of - true -> V; + true -> + V; false -> fail(not_in_enum, push_path(enum, St)) end @@ -396,7 +486,7 @@ valid_object(O, St, Ss) when is_map(O) -> end, lists:foreach( fun({P, #st{} = S}) -> - valid(maps:get(P, O), push_path(P, S)) + valid(maps:get(P, O), push_path(P, S#st{d = undefined})) end, PropSchemas), O; valid_object(_, St, _) -> @@ -405,12 +495,15 @@ valid_object(_, St, _) -> -spec valid_boolean(json(), st(), [st()]) -> json_bool(). valid_boolean(V, #st{s = true}, []) -> V; valid_boolean(_, #st{s = false} = St, []) -> fail(invalid, St); +valid_boolean(<<"true">> , #st{opts = #{coerce := true}}, _) -> true; +valid_boolean(<<"false">>, #st{opts = #{coerce := true}}, _) -> false; valid_boolean(V, St, _) -> assert_type(fun is_boolean/1, V, St), V. valid_null(N, #st{s = true}, []) -> N; valid_null(_, #st{s = false} = St, []) -> fail(invalid, St); +valid_null(<<"null">>, #st{s = null, opts = #{coerce := true}}, _) -> null; valid_null(null, #st{s = null}, _) -> null; valid_null(_, St, _) -> @@ -432,15 +525,28 @@ valid_string(S, St, Ss) when is_binary(S) -> Lmax = schema_prop(<<"maxLength">>, St, Ss, Sz), assert_min(Sz, Lmin, min_length, St), assert_max(Sz, Lmax, max_length, St), - S; + valid_enum(S, string, St, Ss); valid_string(_, St, _) -> fail(wrong_type, St). valid_number(N, _, #st{s = true}, []) -> N; valid_number(_, _, #st{s = false} = St, []) -> fail(invalid, St); +valid_number(I, Sub, #st{opts = #{coerce := true}} = St, Ss) when is_binary(I) -> + try coerce_num(Sub, I) of + I1 -> + valid_number_(I1, Sub, St#st{v = I1}, Ss) + catch + error:_ -> + fail(wrong_type, St) + end; valid_number(I, Sub, St, Ss) when is_number(I) -> - [assert_type(fun is_integer/1, I, St) || Sub == integer], + valid_number_(I, Sub, St, Ss); +valid_number(_, _, St, _) -> + fail(wrong_type, St). + +valid_number_(I, Sub, St, Ss) when is_number(I) -> + [assert_type(fun is_integer/1, I, St) || Sub == integer], case any_schema_prop(<<"multipleOf">>, St, Ss) of error -> ok; {ok, X} when is_integer(X), X > 0 -> @@ -473,9 +579,16 @@ valid_number(I, Sub, St, Ss) when is_number(I) -> EMax = schema_prop(<<"exclusiveMaximum">>, St, Ss, I+1), test_range('>', EMax, I, add_anno(EMax, push_path(exclusiveMaximum, St))), test_range('<', EMin, I, add_anno(EMin, push_path(exclusiveMinimum, St))), - I; -valid_number(_, _, St, _) -> - fail(wrong_type, St). + I. + +coerce_num(integer, I) when is_binary(I) -> + binary_to_integer(I); +coerce_num(number, I) when is_binary(I) -> + try binary_to_integer(I) + catch + error:_ -> + binary_to_float(I) + end. valid_array(A, #st{s = true}, []) -> A; valid_array(_, #st{s = false} = St, []) -> fail(invalid, St); @@ -497,23 +610,23 @@ valid_array(A, #st{} = St, Ss) when is_list(A) -> {ok, PfxIs} -> assert_schema(fun is_list/1, PfxIs, push_path(prefixItems, St)), check_prefix_items( - PfxIs, A, Is, push_path(prefixItems, St#st{s = PfxIs})); + PfxIs, A, Is, push_path(prefixItems, s(PfxIs, St))); error -> - check_items(A, push_path(items, St#st{s = Is})) + check_items(A, push_path(items, s(Is, St))) end; error -> case PfxItems of {ok, PfxIs} -> assert_schema(fun is_list/1, PfxIs, push_path(prefixItems, St)), check_prefix_items( - PfxIs, A, true, push_path(prefixItems, St#st{s = PfxIs})); + PfxIs, A, true, push_path(prefixItems, s(PfxIs, St))); error -> ok end end, case any_schema_prop(<<"contains">>, St, Ss) of {ok, Cs} -> - check_contains(A, push_path(contains, St#st{s = Cs}), + check_contains(A, push_path(contains, s(Cs, St)), schema_prop(<<"minContains">>, St, Ss, null), schema_prop(<<"maxContains">>, St, Ss, null)); error -> @@ -540,12 +653,12 @@ check_prefix_items(_, _, _, St) -> fail(invalid, St). check_prefix_items([I|Is], [H|T], Ix, Items, St) -> - _ = valid(H, push_path(Ix, St#st{s = I})), + _ = valid(H, push_path(Ix, s(I, St))), check_prefix_items(Is, T, Ix+1, Items, St); check_prefix_items(_, [], _, _, _) -> ok; check_prefix_items([], Rest, Ix, Items, St) -> - check_items(Rest, Ix, push_path(items, St#st{s = Items})). + check_items(Rest, Ix, push_path(items, s(Items, St))). check_items(A, St) -> check_items(A, 0, St). @@ -577,13 +690,13 @@ check_contains([], _, St, Min, Max, Yes, _No) -> if is_integer(Max) -> _ = valid(YesLen, push_path(max, - St#st{s = #{<<"maximum">> => Max}})); + s(#{<<"maximum">> => Max}, St))); true -> ok end, if is_integer(Min) -> _ = valid(YesLen, push_path(min, - St#st{s = #{<<"minimum">> => Min}})); + s(#{<<"minimum">> => Min}, St))); true -> ok end @@ -660,14 +773,26 @@ any_pattern_({Pat, Schema, I}, P) -> any_pattern_(maps:next(I), P) end. -schemas_from_dynamic_eval(Obj, #st{s = Schema} = St) -> - SMap = schema_map(Schema), +maybe_expand_ref(#st{s = S} = St) -> + case S of + #{<<"$ref">> := Ref} = R when map_size(R) == 1 -> + St#st{s = expand_ref(Ref, St#st.r)}; + _ -> + St + end. + +schemas_from_dynamic_eval(_, #st{d = Ss} = St) when Ss =/= undefined -> + {Ss, St}; +schemas_from_dynamic_eval(Obj, #st{s = Schema} = St0) -> + St = maybe_expand_ref(St0), + SMap = schema_map(Schema, St), + Ss = maps:fold( fun(<<"allOf">>, Ss, Acc) -> St1 = push_path(allOf, St), case split_valid(Obj, St, Ss) of {ValidSs, []} -> - Acc ++ [St1#st{s = S} || {_, S} <- ValidSs]; + Acc ++ [s(S, St1) || {_, S} <- ValidSs]; {_, FailedSs} -> fail(failing_schemas, add_anno(FailedSs, St1)) end; @@ -675,7 +800,7 @@ schemas_from_dynamic_eval(Obj, #st{s = Schema} = St) -> St1 = push_path(anyOf, St), case split_valid(Obj, St1, Ss) of {[_|_] = ValidSs, _} -> - Acc ++ [St1#st{s = S} || {_, S} <- ValidSs]; + Acc ++ [s(S, St1) || {_, S} <- ValidSs]; {[], FailedSs} -> fail(no_matching_schema, add_anno(FailedSs, St1)) end; @@ -683,7 +808,7 @@ schemas_from_dynamic_eval(Obj, #st{s = Schema} = St) -> St1 = push_path(oneOf, St), case split_valid(Obj, St1, Ss) of {[{_, S}], _} -> - Acc ++ [St1#st{s = S}]; + Acc ++ [s(S, St1)]; {[_|_] = MoreValid, _} -> ValidIxs = [I || {I,_} <- MoreValid], fail(more_than_one, add_anno({valid, ValidIxs}, St1)); @@ -692,31 +817,49 @@ schemas_from_dynamic_eval(Obj, #st{s = Schema} = St) -> end; (<<"if">>, S, Acc) -> St1 = push_path('if', St), - try valid(Obj, St1#st{s = S}) of + try valid(Obj, s(S, St1)) of _ -> Sthen = push_path( - 'then', St1#st{s = maps:get(<<"then">>, SMap, #{})}), + 'then', s(maps:get(<<"then">>, SMap, #{}), St1)), _ = valid(Obj, Sthen), Acc ++ [Sthen] catch error:_ -> Selse = push_path( - 'else', St1#st{s = maps:get(<<"else">>, SMap, #{})}), + 'else', s(maps:get(<<"else">>, SMap, #{}), St1)), _ = valid(Obj, Selse), Acc ++ [Selse] end; (<<"not">>, S, Acc) -> - Snot = push_path('not', St#st{s = S}), + Snot = push_path('not', s(S, St)), try valid(Obj, Snot) of _ -> fail(invalid, Snot) catch error:_ -> Acc end; - (_, _, Acc) -> Acc - end, [], SMap). + (<<"x-", _/binary>> = Prop, SExt, Acc) -> + case St#st.opts of + #{extensions := #{Prop := ExtF}} -> + St1 = push_path(Prop, St), + call_extension(ExtF, Obj, SExt, Prop, St1), + Acc; + _ -> + Acc + end; + (_, _, Acc) -> + Acc + end, [], SMap), + {Ss, St#st{d = Ss}}. + +call_extension(F, Obj, S, Prop, St) -> + try F(Obj, S) + catch + error:E -> + fail(extended_check, add_anno({Prop, E}, St)) + end. acc_props(Ss) -> lists:foldl( @@ -728,6 +871,9 @@ acc_props(Ss) -> end end, #{}, Ss). +s(S, #st{} = St) -> + St#st{s = S, d = undefined}. + push_path(Ps, #st{p = P0} = St) when is_list(Ps) -> %% Assume Ps is in reverse order St#st{p = Ps ++ P0}; @@ -780,13 +926,15 @@ uniqueItems(L) -> USorted = lists:usort(L), [] == L -- USorted. -expand_schema(S0) -> - S = expand_definitions(S0), +expand_schema(S) -> + %% S = expand_definitions(S0), expand_schema(S, S). -expand_definitions(#{<<"definitions">> := D} = S) -> - S#{<<"definitions">> := expand_schema(D, S)}. +%% expand_definitions(#{<<"definitions">> := D} = S) -> +%% S#{<<"definitions">> := expand_schema(D, S)}. +expand_schema(#{<<"$ref">> := Path} = V, S0) when map_size(V) == 1 -> + expand_schema(expand_ref(Path, S0), S0); expand_schema(S, S0) when is_map(S) -> %% https://json-schema.org/understanding-json-schema/structuring#dollarref %% When $id is used in a subschema, it indicates an embedded schema. @@ -802,9 +950,9 @@ expand_schema(S, S0) when is_map(S) -> S0 end, maps:fold(fun(K, V, Acc) -> expand_schema_(K, V, Acc, S1) end, #{}, S); -expand_schema([#{<<"$ref">> := Path} = V], S0) when map_size(V) == 1 -> - D = expand_ref(Path, S0), - [D]; +%% expand_schema([#{<<"$ref">> := Path} = V], S0) when map_size(V) == 1 -> +%% D = expand_ref(Path, S0), +%% [expand_schema(D, S0)]; expand_schema(S, S0) when is_list(S) -> [expand_schema(E, S0) || E <- S]; expand_schema(S, _) -> @@ -853,6 +1001,8 @@ schema(Path) -> schema(Path, Schema) -> schema(Path, Schema, #{follow_refs => true}). +schema(Path, #st{s = Schema, r = RootSchema}, Opts) -> + schema_(Path, Schema, RootSchema, Opts); schema(Path, Schema, Opts) -> schema_(Path, Schema, Schema, Opts). diff --git a/test/data/nested_refs_schema.json b/test/data/nested_refs_schema.json new file mode 100644 index 0000000..82862fe --- /dev/null +++ b/test/data/nested_refs_schema.json @@ -0,0 +1,29 @@ +{ + "type": "object", + "properties": { + "tx" : { "$ref" : "#/components/schemas/Tx" } + }, + "components": { + "schemas": { + "Tx": { + "type": "object", + "properties": { + "from": { + "allOf": [ + { "$ref": "#/components/schemas/Pubkey" }, + { "x-serialization": { + "tags": ["ak"] + }} + ] + } + } + }, + "Pubkey": { + "type": "string", + "x-serialization": { + "tags": ["ak", "ct"] + } + } + } + } +} diff --git a/test/data/recursion_data.json b/test/data/recursion_data.json new file mode 100644 index 0000000..48f380f --- /dev/null +++ b/test/data/recursion_data.json @@ -0,0 +1,20 @@ +{ + "name": "Elizabeth", + "children": [ + { + "name": "Charles", + "children": [ + { + "name": "William", + "children": [ + { "name": "George" }, + { "name": "Charlotte" } + ] + }, + { + "name": "Harry" + } + ] + } + ] +} diff --git a/test/data/recursion_data_faulty.json b/test/data/recursion_data_faulty.json new file mode 100644 index 0000000..5fe6054 --- /dev/null +++ b/test/data/recursion_data_faulty.json @@ -0,0 +1,20 @@ +{ + "name": "Elizabeth", + "children": [ + { + "name": "Charles", + "children": [ + { + "name": "William", + "children": [ + { "name": [1] }, + { "name": "Charlotte" } + ] + }, + { + "name": "Harry" + } + ] + } + ] +} diff --git a/test/data/recursion_schema.json b/test/data/recursion_schema.json new file mode 100644 index 0000000..f20c882 --- /dev/null +++ b/test/data/recursion_schema.json @@ -0,0 +1,10 @@ +{ + "type": "object", + "properties": { + "name": { "type": "string" }, + "children": { + "type": "array", + "items": { "$ref": "#" } + } + } +} diff --git a/test/data/ref_loop_schema.json b/test/data/ref_loop_schema.json new file mode 100644 index 0000000..6a3ba40 --- /dev/null +++ b/test/data/ref_loop_schema.json @@ -0,0 +1,6 @@ +{ + "$defs": { + "alice": { "$ref": "#/$defs/bob" }, + "bob": { "$ref": "#/$defs/alice" } + } +} diff --git a/test/gmconfig_schema_utils_tests.erl b/test/gmconfig_schema_utils_tests.erl new file mode 100644 index 0000000..3ebba8f --- /dev/null +++ b/test/gmconfig_schema_utils_tests.erl @@ -0,0 +1,404 @@ +-module(gmconfig_schema_utils_tests). + +-export([tr/0]). + +-include_lib("eunit/include/eunit.hrl"). + +-import(gmconfig_schema_utils, [ valid/2 + , validate/3 ]). + +%%-define(t(Expr), ?_test(?debugVal(Expr))). +-define(t(Expr), ?_test(Expr)). + +tr() -> + dbg:tracer(), + dbg:tpl(?MODULE, x), + dbg:tpl(gmconfig_schema_utils,x), + dbg:p(all,[c]), + eunit:test(?MODULE). + +simple_type_test_() -> + {"Simple type tests", + [ + ?t(t_integer()) + , ?t(t_number()) + , ?t(t_boolean()) + , ?t(t_string()) + , ?t(t_array()) + , ?t(t_object()) + , ?t(t_shortcut_schema()) + ]}. + +update_test_() -> + {"Validated update tests", + [ + ?t(t_update_objects()) + ]}. + +schema_spec_examples_test_() -> + {"Examples from JSON-Schema docs", + [ + ?t(t_ref_loop()) + , ?t(t_recursive_def()) + , ?t(t_nested_refs()) + ]}. + +array() -> #{<<"type">> => <<"array">>}. +int() -> #{<<"type">> => <<"integer">>}. +str() -> #{<<"type">> => <<"string">>}. +obj(Ps) -> #{<<"type">> => <<"object">>, + <<"properties">> => Ps}. + + +t_update_objects() -> + S0 = obj( + #{<<"a">> => int(), + <<"b">> => obj(#{<<"b1">> => array(), + <<"b2">> => + obj(#{<<"b21">> => str()}) + }) + }), + %% + %% Only objects are merged; other types are replaced. + A0 = #{<<"a">> => 1, + <<"b">> => #{<<"b1">> => [1,2], + <<"b2">> => #{}}}, + B0 = #{<<"a">> => 2, + <<"b">> => #{<<"b1">> => [4,5], + <<"b2">> => #{<<"b21">> => <<"foo">>}}}, + #{<<"a">> := 1, + <<"b">> := #{<<"b1">> := [1,2], + <<"b2">> := #{<<"b21">> := <<"foo">>}}} = + gmconfig_schema_utils:merge(A0, B0, S0), + %% + %% Modified update semantics - replace instead of merge + S0Ps = maps:get(<<"properties">>, S0), + S01Ps = S0Ps#{<<"c">> => #{<<"type">> => <<"object">>, + <<"updateSemantics">> => <<"replace">>, + <<"properties">> => + #{<<"c1">> => #{<<"type">> => <<"integer">>}, + <<"c2">> => #{<<"type">> => <<"integer">>} + }}}, + S01 = S0#{<<"properties">> => S01Ps}, + A01 = A0#{<<"c">> => #{<<"c1">> => 1}}, + B01 = B0#{<<"c">> => #{<<"c2">> => 2}}, + #{<<"a">> := 1, + <<"b">> := #{<<"b1">> := [1,2], + <<"b2">> := #{<<"b21">> := <<"foo">>}}, + <<"c">> := #{<<"c1">> := 1}} = + gmconfig_schema_utils:merge(A01, B01, S01), + A0r = #{<<"a">> => 1, + <<"b">> => #{<<"$updateSemantics">> => <<"replace">>, + <<"b1">> => [1,2], + <<"b2">> => #{<<"$updateSemantics">> => <<"merge">>} + }}, + #{<<"a">> := 1, + <<"b">> := #{<<"b1">> := [1,2], + <<"b2">> := B2M}} = + gmconfig_schema_utils:merge(A0r, B0, S0), + %% B2M is the empty map (updateSemantics prop removed recursively) + ?assertEqual(#{}, B2M), + A0r1 = #{<<"a">> => 1, + <<"b">> => #{<<"$updateSemantics">> => <<"merge">>, + <<"b1">> => [1,2], + <<"b2">> => #{<<"$updateSemantics">> => <<"replace">>} + }}, + #{<<"a">> := 1, + <<"b">> := #{<<"b1">> := [1,2], + <<"b2">> := B2M1}} = + gmconfig_schema_utils:merge(A0r1, B0, S0), + %% B2M is the empty map (updateSemantics prop removed recursively) + ?assertEqual(#{}, B2M1), + %% + %% With updateSemantics : suggest, the offered data is accepted if + %% the existing is missing or 'null'. + S02Ps = S0Ps#{<<"c">> => #{<<"type">> => <<"object">>, + <<"updateSemantics">> => <<"suggest">>, + <<"properties">> => + #{<<"c1">> => #{<<"type">> => <<"integer">>}, + <<"c2">> => #{<<"type">> => <<"integer">>} + }}}, + S02 = S0#{<<"properties">> => S02Ps}, + A02 = A0#{<<"c">> => #{<<"c1">> => 1}}, + B02 = B0#{<<"c">> => #{<<"c2">> => 2}}, + #{<<"a">> := 1, + <<"b">> := #{<<"b1">> := [1,2], + <<"b2">> := #{<<"b21">> := <<"foo">>}}, + <<"c">> := #{<<"c2">> := 2}} = + gmconfig_schema_utils:merge(A02, B02, S02), + B03 = B0#{<<"c">> => null}, + #{<<"a">> := 1, + <<"b">> := #{<<"b1">> := [1,2], + <<"b2">> := #{<<"b21">> := <<"foo">>}}, + <<"c">> := #{<<"c1">> := 1}} = + gmconfig_schema_utils:merge(A02, B0, S02), + #{<<"a">> := 1, + <<"b">> := #{<<"b1">> := [1,2], + <<"b2">> := #{<<"b21">> := <<"foo">>}}, + <<"c">> := #{<<"c1">> := 1}} = + gmconfig_schema_utils:merge(A02, B03, S02), + ok. + +t_integer() -> + S0 = #{<<"type">> => <<"integer">>}, + is_valid(17, S0), + all_fail([17.0, [], #{}, <<>>], S0, wrong_type), + S1 = S0#{<<"minimum">> => 1, <<"maximum">> => 4}, + all_valid([1, 2, 4], S1), + all_fail([0, 5], S1, not_in_range), + S2 = S0#{<<"exclusiveMinimum">> => 1, <<"exclusiveMaximum">> => 3}, + is_valid(2, S2), + all_fail([1, 3], S2, not_in_range), + S3 = S0#{<<"multipleOf">> => 2}, + is_valid(4, S3), + fails(3, S3, not_a_multiple), + ok. + +t_number() -> + S0 = #{<<"type">> => <<"number">>}, + all_valid([17, 17.0], S0), + all_fail([#{}, [], <<>>, true], S0, wrong_type), + S1 = S0#{<<"minimum">> => 1, <<"maximum">> => 4.0}, + all_valid([1, 1.0, 2, 2.3, 3.999, 4, 4.0], S1), + all_fail([0, 5], S1, not_in_range), + S2 = S0#{<<"exclusiveMinimum">> => 1, <<"exclusiveMaximum">> => 3}, + is_valid(2, S2), + all_fail([1, 3], S2, not_in_range), + S3 = S0#{<<"multipleOf">> => 2}, + is_valid(4, S3), + all_fail([3, 4.0], S3, not_a_multiple), + ok. + +t_boolean() -> + S0 = #{<<"type">> => <<"boolean">>}, + all_valid([true, false], S0), + %% truthy and falsy types not accepted + all_fail([1, 0, null, [], <<>>, #{}], S0, wrong_type), + ok. + +t_string() -> + S0 = #{<<"type">> => <<"string">>}, + all_valid([<<>>, <<"foo">>], S0), + fails("foo", S0, wrong_type), + ok. + +t_array() -> + S0 = #{<<"type">> => <<"array">>}, + all_valid([ [], [1,1], [1,2,3], [#{}] ], S0), + all_fail([1, false, #{}, <<"foo">>], S0, wrong_type), + S1 = S0#{<<"minItems">> => 2}, + all_valid([[1,2], [1,2,3,4]], S1), + all_fail([[], [1]], S1, not_in_range), + S2 = S0#{<<"maxItems">> => 4}, + all_valid([[], [1], [1,2], [1,2,3,4]], S2), + fails([1,2,3,4,5], S2, not_in_range), + S3 = maps:merge(S1, S2), + all_valid([ [1,2], [1,2,3], [1,2,3,4] ], S3), + all_fail([ [1], [1,2,3,4,5] ], S3, not_in_range), + S4 = S0#{<<"uniqueItems">> => true}, + all_valid([ [], [1], [1,2] ], S4), + all_fail([ [1,1], [2,1,2] ], S4, not_unique), + S5 = S0#{<<"items">> => #{<<"type">> => <<"integer">>}}, + all_valid([ [], [1], [1,2,3] ], S5), + fails([<<"foo">>], S5, #{e => wrong_type, p => [array,items,0,integer]}), + fails([1,2,false], S5, #{e => wrong_type, p => [array,items,2,integer]}), + S6 = S0#{<<"prefixItems">> => [#{<<"type">> => <<"string">>}, + #{<<"type">> => <<"integer">>}, + #{<<"type">> => <<"integer">>}]}, + is_valid([<<"foo">>, 1], S6), + is_valid([<<"foo">>, 1, 2], S6), + is_valid([<<"foo">>, 1, 2, 3], S6), + fails([1,2,3], S6, #{e => wrong_type, p => [array,prefixItems,0,string]}), + S7 = S6#{<<"items">> => false}, + fails([<<"foo">>, 1, 2, 3], S7, #{e => invalid, + p => [array,prefixItems,items,3]}), + S71 = S6#{<<"items">> => #{<<"type">> => <<"integer">>}}, + is_valid([<<>>, 1,2,3,4,5,6], S71), + fails([<<>>, 1,2,3,4,5,[]], S71, + #{e => wrong_type, + p => [array,prefixItems,items,6,integer]}), + S8 = S0#{<<"contains">> => #{<<"type">> => <<"integer">>}}, + is_valid([1,2,3], S8), + is_valid([true,true,1], S8), + fails([true], S8, contains), + fails([], S8, contains), % not sure where specifed, but makes sense + S81 = S8#{<<"minContains">> => 2}, + is_valid([1,2,3], S81), + is_valid([true,1,2], S81), + fails([1], S81, #{e => not_in_range, p => [array, contains, min, + integer, minimum]}), + S82 = S8#{<<"maxContains">> => 4}, + all_valid([ [1], [1,2,3], [true,1,2,3,4] ], S82), + S83 = maps:merge(S81, S82), + is_valid([1,2,3], S83), + fails([1], S83, #{e => not_in_range, p => [array, contains, min, + integer, minimum]}), + fails([1,2,3,4,5], S83, #{e => not_in_range, p => [array, contains, max, + integer, maximum]}), + is_valid([], S8#{<<"minContains">> => 0}), + ok. + +t_object() -> + S0 = #{<<"type">> => <<"object">>}, + all_valid([ #{}, #{<<"a">> => 1} ], S0), + all_fail([ true, 17, 17.0, [], <<"foo">> ], S0, #{e => wrong_type}), + S1 = S0#{<<"properties">> => #{<<"a">> => #{<<"type">> => <<"integer">>}}}, + all_valid([ #{<<"a">> => 1}, #{}, #{<<"b">> => 1} ], S1), + fails(#{<<"a">> => true}, S1, #{e => wrong_type, + p => [object, <<"a">>, integer]}), + is_valid(#{<<"b">> => 1}, S1#{<<"additionalProperties">> => true}), + fails(#{<<"b">> => 1}, S1#{<<"additionalProperties">> => false}, + #{e => invalid, + p => [object, <<"b">>], + a => [additionalProperties]}), + fails(#{<<"b">> => 1}, S1#{<<"required">> => [<<"a">>]}, + #{e => required, p => [object,required], a => [[<<"a">>]]}), + all_valid([ #{<<"a">> => 1}, #{<<"a">> => true} ], + S0#{<<"properties">> => + #{<<"a">> => #{<<"oneOf">> => + [#{<<"type">> => <<"integer">>}, + #{<<"type">> => <<"boolean">>}]} + }}), + S2 = S0#{<<"properties">> => + #{<<"a">> => #{<<"type">> => <<"integer">>, + <<"allOf">> => + [#{<<"minimum">> => 2}, + #{<<"maximum">> => 5}, + #{<<"multipleOf">> => 2} + ]}} + }, + is_valid(#{<<"a">> => 4}, S2), + fails(#{<<"a">> => 3}, S2, #{e => failing_schemas, + a => [[{2,'_'}]|'_'], + p => [object, <<"a">>, allOf]}), + ok. + +t_shortcut_schema() -> + Vs = [<<"foo">>, + #{<<"a">> => 1}, + 17], + all_valid(Vs, #{}), + all_valid(Vs, true), + all_fail(Vs, false, invalid), + ok. + +is_valid(V, S) -> + {V, V} = {valid(V, S), V}. + %% ?assertEqual(V, valid(V, S)). + +fails(V, S, Reason) -> + fails(V, S, #{}, Reason). + +fails(V, S, Opts, Reason) when is_atom(Reason) -> + fails(V, S, Opts, #{e => Reason}); +fails(V, S, Opts, Expect) -> + try validate(V, S, Opts) of + _ -> + error({expected_exception, #{v => V, + s => S, + e => Expect}}) + catch + error:R -> + match_expected(Expect, R) + end. + %% ?assertError({Reason, [], V}, valid(V, S)). + +match_expected(E, R) -> + case maps:fold( + fun(K, V, Acc) -> + case maps:find(K, R) of + {ok, V} -> Acc; + {ok, Other} -> + case match_other(V, Other) of + true -> Acc; + false -> + Acc#{K => Other} + end; + error -> + Acc#{K => '$not_found'} + end + end, #{}, E) of + M when map_size(M) == 0 -> + ok; + Unexpected -> + error({expected, E, Unexpected}) + end. + +match_other(V, V) -> true; +match_other([H1|T1], [H2|T2]) -> + case match_other(H1, H2) of + true -> + match_other(T1, T2); + false -> + false + end; +match_other(T1, T2) when tuple_size(T1) =:= tuple_size(T2) -> + lists:all(fun({A,B}) -> match_other(A, B) end, + lists:zip(tuple_to_list(T1), + tuple_to_list(T2))); +match_other('_', _) -> + true; +match_other(_, _) -> + false. + +all_valid(Vs, S) -> + [is_valid(V, S) || V <- Vs]. + +all_fail(Vs, S, Reason) -> + [fails(V, S, Reason) || V <- Vs]. + +read(F) -> + FullF = filename:join( + filename:dirname(code:which(?MODULE)), F), + ?debugFmt("FullF = ~s~n", [FullF]), + {ok, Bin} = file:read_file(FullF), + dec(Bin). + +dec(Str) -> + try json:decode(Str) + catch + error:E -> + error(#{type => json_decode_error, + error => E}) + end. + + +t_ref_loop() -> + S = read("data/ref_loop_schema.json"), + try gmconfig_schema_utils:schema([<<"$defs">>,<<"alice">>], S) of + Unexpected -> + error({unexpected, Unexpected}) + catch + error:nested_references -> + ok + end. + +t_recursive_def() -> + S = read("data/recursion_schema.json"), + D = read("data/recursion_data.json"), + Df = read("data/recursion_data_faulty.json"), + is_valid(D, S), + fails(Df, S, #{e => wrong_type, p => [object, <<"children">>, + array, items, 0, + object, <<"children">>, + array, items, 0, + object,<<"children">>, + array, items, 0, + object, <<"name">>, string + ]}), + ok. + +t_nested_refs() -> + S = read("data/nested_refs_schema.json"), + F = fun(Str, #{<<"tags">> := Tags}) -> + true = lists:any( + fun(T) -> + nomatch =/= string:prefix(Str, T) + end, Tags) + end, + Opts = #{extensions => #{<<"x-serialization">> => F}}, + Vs = #{<<"tx">> => #{<<"from">> => <<"ak_good">>}}, + Vf = #{<<"tx">> => #{<<"from">> => <<"ac_bad">>}}, + validate(Vs, S, Opts), + fails(Vf, S, Opts, #{e => failing_schemas}), + ok. diff --git a/zomp.meta b/zomp.meta index 3c0e814..d46adab 100644 --- a/zomp.meta +++ b/zomp.meta @@ -4,7 +4,7 @@ {prefix,"gmconfig"}. {author,"Ulf Wiger"}. {desc,"Configuration management support"}. -{package_id,{"uwiger","gmconfig",{0,1,2}}}. +{package_id,{"uwiger","gmconfig",{0,2,0}}}. {deps,[{"uwiger","setup",{2,2,4}}]}. {key_name,none}. {a_email,"ulf@wiger.net"}.