From de5d2bfb7d419178e0beeb5695c84818b0268390 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Wed, 2 Apr 2025 16:35:38 +0900 Subject: [PATCH] Make config file parsers pluggable (#1) In order to simplify deps, this PR makes config file formats pluggable. The extensions supported by default are "json" and "eterm". The default json decoder is the one from OTP, but if one prefers another (e.g. `zj`), it's possible to choose that when installing the gmconfig env. Example of adding a yaml decoder: ```erlang instrument_gmconfig() -> gmconfig:set_gmconfig_env(gmconfig_env()). -spec gmconfig_env() -> gmconfig:gmconfig(). gmconfig_env() -> #{ ... , config_formats => add_yamerl() }. add_yamerl() -> Default = maps:get(config_formats, gmconfig:default_gmconfig_env()), case maps:is_key("yaml", Default) of true -> Default; false -> Default#{"yaml" => fun yamerl_decode/1} end. yamerl_decode(Bin) -> yamerl:decode(Bin, [{str_node_as_binary, true}, {map_node_format, map}]). ``` Co-authored-by: Ulf Wiger Reviewed-on: https://git.qpq.swiss/QPQ-AG/gmconfig/pulls/1 --- rebar.config | 6 +- rebar.lock | 19 ++--- src/gmconfig.app.src | 2 +- src/gmconfig.erl | 166 ++++++++++++++++++++++++++----------------- 4 files changed, 107 insertions(+), 86 deletions(-) diff --git a/rebar.config b/rebar.config index 912074a..e359a6a 100644 --- a/rebar.config +++ b/rebar.config @@ -3,9 +3,7 @@ {plugins, [rebar3_hex]}. {deps, [ - {zj, {git, "https://gitlab.com/zxq9/zj.git", {tag, "1.1.2"}}} - , {yamerl, "0.10.0"} - , {setup, "2.1.2"} + {setup, "2.2.1"} ]}. {profiles, [ @@ -17,5 +15,5 @@ deprecated_function_calls, deprecated_functions]}. {dialyzer, [ {warnings, [unknown]} - , {base_plt_apps, [erts, kernel, stdlib, yamerl, zj, setup]} + , {base_plt_apps, [erts, kernel, stdlib, setup]} ]}. diff --git a/rebar.lock b/rebar.lock index 37a8aaa..d8a17e9 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,15 +1,4 @@ -{"1.2.0", -[{<<"setup">>,{pkg,<<"setup">>,<<"2.1.2">>},0}, - {<<"yamerl">>,{pkg,<<"yamerl">>,<<"0.10.0">>},0}, - {<<"zj">>, - {git,"https://gitlab.com/zxq9/zj.git", - {ref,"090a43d23edc481695664f16763f147a78c45afc"}}, - 0}]}. -[ -{pkg_hash,[ - {<<"setup">>, <<"43C0BBFE9160DE7925BF2FC2FE4396A99DC4EE1B73F0DC46ACC3F10E27B07A9C">>}, - {<<"yamerl">>, <<"4FF81FEE2F1F6A46F1700C0D880B24D193DDB74BD14EF42CB0BCF46E81EF2F8E">>}]}, -{pkg_hash_ext,[ - {<<"setup">>, <<"596713D48D8241DF31821C08A9F7BAAF3E7CDD042C8396BC956CC7AE056925DC">>}, - {<<"yamerl">>, <<"346ADB2963F1051DC837A2364E4ACF6EB7D80097C0F53CBDC3046EC8EC4B4E6E">>}]} -]. +[{<<"setup">>, + {git,"https://github.com/uwiger/setup.git", + {ref,"9675f9a25c2087e676660911987935caeff9fef8"}}, + 0}]. diff --git a/src/gmconfig.app.src b/src/gmconfig.app.src index 1c6d6bd..915905c 100644 --- a/src/gmconfig.app.src +++ b/src/gmconfig.app.src @@ -7,7 +7,7 @@ [ kernel , stdlib - , zj + , setup ]}, {env, []}, {modules, []}, diff --git a/src/gmconfig.erl b/src/gmconfig.erl index 371e0d1..1a76f4f 100644 --- a/src/gmconfig.erl +++ b/src/gmconfig.erl @@ -48,6 +48,7 @@ , gmconfig_env/1 , gmconfig_env/2 , set_gmconfig_env/1 + , default_gmconfig_env/0 ]). -ifdef(TEST). @@ -62,11 +63,19 @@ -include_lib("kernel/include/logger.hrl"). +-type extension() :: string(). %% file extension (without leading dot) + +-type decoder_return() :: config_tree() + | {ok, config_tree()} + | {error, any()}. +-type decoder_fun() :: fun( (binary()) -> decoder_return() ). + -type gmconfig() :: #{ os_env_prefix => string() , config_file_basename => string() | 'undefined' , config_file_os_env => string() | 'undefined' , config_file_search_path => [string() | fun(() -> string())] - , system_suffix => string() + , system_suffix => string() + , config_formats => #{ extension() => decoder_fun() } , schema => string() | map() | fun(() -> map())}. -type basic_type() :: number() | binary() | boolean(). @@ -110,12 +119,13 @@ mock_system_defaults(Config) -> -endif. -spec set_gmconfig_env(gmconfig()) -> ok. -set_gmconfig_env(Env) when is_map(Env) -> +set_gmconfig_env(Env0) when is_map(Env0) -> + Env = maps:merge(default_gmconfig_env(), Env0), persistent_term:put({?MODULE, gmconfig_env}, Env). -spec gmconfig_env() -> gmconfig(). gmconfig_env() -> - persistent_term:get({?MODULE, gmconfig_env}, default_gmconfig()). + persistent_term:get({?MODULE, gmconfig_env}, default_gmconfig_env()). -spec gmconfig_env(atom()) -> any(). gmconfig_env(Key) -> @@ -142,11 +152,13 @@ search_path(Kind) -> end, Path0) end. -default_gmconfig() -> +default_gmconfig_env() -> #{ os_env_prefix => "GM" , config_file_basename => "gmconfig" , config_file_os_env => undefined , config_file_search_path => ["."] + , config_formats => #{ "json" => fun json_decode/1 + , "eterm" => fun eterm_consult/1 } , system_defaults_search_path => [fun setup:data_dir/0] , system_suffix => "" }. @@ -552,8 +564,8 @@ coerce_type(Key, Value, Schema) -> <<"integer">> -> to_integer(Value); <<"string">> -> to_string(Value); <<"boolean">> -> to_bool(Value); - <<"array">> -> json_decode(list_to_binary(Value)); - <<"object">> -> json_decode(list_to_binary(Value)) + <<"array">> -> json:decode(list_to_binary(Value)); + <<"object">> -> json:decode(list_to_binary(Value)) end; _ -> error({unknown_key, Key}) @@ -642,10 +654,21 @@ search_default_config() -> Basename -> Dirs = search_path(), SystemSuffix = get_system_suffix(), - Fname = Basename ++ SystemSuffix ++ ".{json,yaml}", + ExtPattern = extension_pattern(), + Fname = Basename ++ SystemSuffix ++ ExtPattern, search_for_config_file(Dirs, Fname) end. +config_formats() -> + gmconfig_env(config_formats). + +extension_pattern() -> + Exts = maps:keys(config_formats()), + lists:flatten([".{", intersperse(Exts),"}"]). + +intersperse([H|T]) -> + [H | [[",", X] || X <- T]]. + search_for_config_file(Dirs, FileWildcard) -> lists:foldl( fun(D0, undefined) -> @@ -665,13 +688,16 @@ to_list_string(S) when is_list(S) -> binary_to_list(iolist_to_binary(S)). do_load_user_config(F, Action, Mode) -> - case {filename:extension(F), Action} of - {".json", store} -> store(read_json(F, Mode), Mode); - {".yaml", store} -> store(read_yaml(F, Mode), Mode); - {".json", check} -> check_config_(catch read_json(F, Mode)); - {".yaml", check} -> check_config_(catch read_yaml(F, Mode)) + {Ext, Decoder} = pick_decoder(F), + case Action of + store -> store(read_file(F, Ext, Decoder, Mode), Mode); + check -> check_config_(catch read_file(F, Ext, Decoder, Mode)) end. +pick_decoder(F) -> + "." ++ Ext = filename:extension(F), + {Ext, maps:get(Ext, config_formats())}. + store(Vars, Mode) when is_map(Vars) -> case pt_get_config() of Map when map_size(Map) == 0 -> @@ -726,12 +752,15 @@ silent_as_report(silent) -> report; silent_as_report(Mode ) -> Mode. schema_string(Schema, Mode) -> - JSONSchema = json:encode(Schema), + JSONSchema = json_encode(Schema), case Mode of check -> json:format(JSONSchema, #{indent => 2}); report -> JSONSchema end. +json_encode(J) -> + iolist_to_binary(json:encode(J)). + error_format(Fmt, Args, check) -> io:format(Fmt, Args); error_format(Fmt, Args, report) -> @@ -906,70 +935,64 @@ update_map(With, Map) when is_map(With), is_map(Map) -> end end, Map, With). -read_json(F, Mode) -> - validate( - try_decode(F, fun(F1) -> - json_consult(F1) - end, "JSON", Mode), F, Mode). - -json_consult(F) -> +read_file(F, Type, Decoder, Mode) -> case setup_file:read_file(F) of {ok, Bin} -> - [json_decode(Bin)]; + validate( + try_decode_bin(Bin, Decoder, Type, Mode), + F, Mode); {error, Reason} -> - ?LOG_ERROR("Read error ~s - ~p", [F, Reason]), + error_msg(Mode, "Read error ~s - ~p", [F, Reason]), error({read_error, F, Reason}) end. -json_decode(Bin) -> - case json_decode_(Bin) of - {ok, Value} -> - Value; - {error, Reason} -> - ?LOG_ERROR("CAUGHT: ~p", [Reason]), - error(Reason) +json_decode(Str) when is_list(Str) -> + json:decode(iolist_to_binary(Str)); +json_decode(Bin) when is_binary(Bin) -> + json:decode(Bin). + +eterm_consult(Bin) -> + case setup_file:consult_binary(Bin) of + {ok, [Map]} when is_map(Map) -> + {ok, normalize_config(Map)}; + {ok, Other} -> + error({unknown_data, Other}); + {error, _} = Error -> + Error end. -json_decode_(Bin) -> - case zj:binary_decode(Bin) of - {ok, _} = Ok -> - Ok; - {error, _Parsed, Remainder} -> - {error, {json_decode_error, string_slice(Remainder)}} - end. - --define(SliceSz, 80). -string_slice(B) -> - case size(B) of - Sz when Sz > ?SliceSz -> - Sl = string:slice(B, 0, ?SliceSz - 4), - <>; - _ -> - B - end. - -read_yaml(F, Mode) -> - validate( - try_decode( - F, - fun(F1) -> - yamerl:decode_file(F1, [{str_node_as_binary, true}, - {map_node_format, map}]) - end, "YAML", Mode), - F, Mode). - -try_decode(F, DecF, Fmt, Mode) -> - try DecF(F) +normalize_config(Map) when is_map(Map) -> + try + json:decode(iolist_to_binary(json:encode(Map))) catch error:E -> - error_msg(Mode, "Error reading ~s file: ~s~n", [Fmt, F]), - erlang:error(E) + error({cannot_normalize, E, Map}) end. -validate(JSON, F, Mode) when is_list(JSON) -> - check_validation([validate_(J) || J <- JSON], JSON, F, Mode). -%% validate(JSON, F, Mode) when is_map(JSON) -> -%% validate([JSON], F, Mode). +try_decode_bin(Bin, Decoder, Fmt, Mode) -> + try decode_bin(Bin, Decoder) + catch + error:E:T -> + error_msg(Mode, "CAUGHT for ~p: ~p / ~p", [Fmt, E, T]), + error(E) + end. + +decode_bin(Str, Decoder) when is_list(Str) -> + decode_bin(iolist_to_binary(Str), Decoder); +decode_bin(Bin, Decoder) when is_binary(Bin), is_function(Decoder, 1) -> + case Decoder(Bin) of + {ok, Map} when is_map(Map) -> + Map; + Map when is_map(Map) -> + Map; + {error, E} -> + error(E); + Other -> + error({bad_decoder_return, Other}) + end. + +validate(JSON, F, Mode) when is_map(JSON) -> + check_validation([validate_(JSON)], JSON, F, Mode). vinfo(Mode, Res, F) -> case Res of @@ -1071,7 +1094,8 @@ ensure_schema_loaded() -> load_schema() -> JSON = gmconfig_schema(), - Schema = json_decode(JSON), + Decode = maps:get("json", config_formats()), + Schema = decode_bin(JSON, Decode), pt_set_schema(Schema), Schema. @@ -1084,3 +1108,13 @@ gmconfig_schema() -> F when is_function(F, 0) -> F() end. + +json_consult(F) -> + Decode = maps:get("json", config_formats()), + case setup_file:read_file(F) of + {ok, Bin} -> + decode_bin(Bin, Decode); + {error, Reason} -> + ?LOG_ERROR("Read error ~s - ~p", [F, Reason]), + error({read_error, F, Reason}) + end.