diff --git a/zx b/zx index e1a35e3..4573149 100755 --- a/zx +++ b/zx @@ -21,6 +21,10 @@ {realm = "otpr" :: name(), name = none :: none | name(), version = {z, z, z} :: version(), + type = app :: app | lib + deps = [] :: [package_id()], + dir = none :: none | file:filename(), + socket = none :: none | gen_tcp:socket(), pid = none :: none | pid(), mon = none :: none | reference()}). @@ -39,7 +43,8 @@ %-type keybin() :: {ID :: key_id(), % Type :: public | private, % DER :: binary()}. --type key_id() :: {realm(), KeyName :: lower0_9()}. +-type key_id() :: {realm(), key_name()}. +-type key_name() :: lower0_9(). -type lower0_9() :: [$a..$z | $0..$9 | $_]. %-type label() :: [$a..$z | $0..$9 | $_ | $- | $.]. @@ -64,7 +69,7 @@ main(Args) -> start(["help"]) -> usage_exit(0); start(["run", PackageString | Args]) -> - execute(PackageString, Args); + run(PackageString, Args); start(["init", "app", PackageString]) -> PackageID = package_id(PackageString), initialize(app, PackageID); @@ -84,9 +89,7 @@ start(["drop", "key", KeyID]) -> drop_key(KeyID); start(["verup", Level]) -> verup(Level); -start(["runlocal"]) -> - run_local([]); -start(["runlocal", Args]) -> +start(["runlocal" | Args]) -> run_local(Args); start(["package"]) -> {ok, TargetDir} = file:get_cwd(), @@ -115,7 +118,7 @@ start(_) -> %%% Execution of application --spec execute(Identifier, Args) -> no_return() +-spec run(Identifier, Args) -> no_return() when Identifier :: string(), Args :: [string()]. %% @private @@ -123,7 +126,7 @@ start(_) -> %% dependencies and run the program. This implies determining whether the program and %% its dependencies are installed, available, need to be downloaded, or are inaccessible %% given the current system condition (they could also be bogus, of course). The -%% Identifier provided should be a valid PackageString of the form `realm-appname-version' +%% Identifier should be a valid PackageString of the form `realm-appname-version' %% where the realm and appname should follow standard realm and app package naming %% conventions and the version should be represented as a semver in string form (where %% ommitted elements of the version always default to whatever is most current). @@ -135,15 +138,21 @@ start(_) -> %% If there is a problem anywhere in the locationg, discovery, building, and loading %% procedure the runtime will halt with an error message. -execute(Identifier, Args) -> - true = register(zx, self()), - ok = inets:start(), +run(Identifier, Args) -> PackageID = {Realm, Name, Version} = package_id(Identifier), ok = file:set_cwd(zomp_dir()), PackageRoot = filename:join("lib", Identifier), - ok = ensure_installed(PackageID), - {ok, Meta} = file:consult(filename:join(PackageRoot, "zomp.meta")), - {deps, Deps} = lists:keyfind(deps, 1, Meta), + State = #s{realm = Realm, + name = Name, + version = Version, + dir = PackageRoot}, + NextState = ensure_installed(State), + Meta = read_meta(PackageRoot), + Deps = maps:get(deps, Meta), + NewState = ensure_deps(NextState#s{deps = Deps}), + execute(State). + + Required = [PackageID | Deps], Needed = scrub(Required), Host = {"localhost", 11411}, @@ -152,8 +161,10 @@ execute(Identifier, Args) -> ok = lists:foreach(fun install/1, Needed), ok = lists:foreach(fun build/1, Required), ok = file:set_cwd(PackageRoot), - case lists:keyfind(type, 1, Meta) of - {type, app} -> + case maps:get(type, Meta) of + app -> + true = register(zx, self()), + ok = inets:start(), ok = log(info, "Starting ~ts", [package_string(PackageID)]), PackageMod = list_to_atom(Name), {ok, Pid} = PackageMod:start(normal, Args), @@ -166,7 +177,7 @@ execute(Identifier, Args) -> pid = Pid, mon = Mon}, exec_wait(State); - {type, lib} -> + lib -> Message = "Lib ~ts is available on the system, but is not a standalone app.", ok = log(info, Message, [package_string(PackageID)]), halt(0) @@ -195,7 +206,7 @@ initialize(Type, PackageID) -> Meta = [{package_id, PackageID}, {deps, []}, {type, Type}], - ok = write_terms("zomp.meta", Meta), + ok = write_meta(Meta), ok = log(info, "Project ~tp initialized.", [PackageString]), Message = "NOTICE:~n" @@ -224,10 +235,10 @@ assimilate(PackageFile) -> ok = file:set_cwd(zomp_dir()), {"zomp.meta", MetaBin} = lists:keyfind("zomp.meta", 1, Files), Meta = binary_to_term(MetaBin), - {package_id, PackageID} = lists:keyfind(package_id, 1, Meta), + PackageID = maps:get(package_id, Meta), TgzFile = namify_tgz(PackageID), {TgzFile, TgzData} = lists:keyfind(TgzFile, 1, Files), - {sig, {KeyID, Signature}} = lists:keyfind(sig, 1, Meta), + {KeyID, Signature} = maps:get(sig, Meta), {ok, PubKey} = loadkey(public, KeyID), ok = case public_key:verify(TgzData, sha512, Signature, PubKey) of @@ -259,7 +270,7 @@ assimilate(PackageFile) -> set_dep(PackageString) -> PackageID = package_id(PackageString), Meta = read_meta(), - {deps, Deps} = lists:keyfind(deps, 1, Meta), + Deps = maps:get(deps, Meta), case lists:member(PackageID, Deps) of true -> ok = log(info, "~ts is already a dependency", [PackageString]), @@ -279,11 +290,9 @@ set_dep(PackageString) -> %% such a dependency is not already present. Then write the project meta back to its %% file and exit. -set_dep(PackageID = {Realm, Name, NewVersion}, Deps, Meta) -> +set_dep(PackageID = {Realm, Name, Version}, Deps, Meta) -> {ok, CWD} = file:get_cwd(), - ok = file:set_cwd(zomp_dir()), - ok = ensure_installed(PackageID), - ok = file:set_cwd(CWD), + LatestVersion = latest_version(PackageID), ExistingPackageIDs = fun ({R, N, _}) -> {R, N} == {Realm, Name} end, NewDeps = case lists:partition(ExistingPackageIDs, Deps) of @@ -297,18 +306,37 @@ set_dep(PackageID = {Realm, Name, NewVersion}, Deps, Meta) -> ok = log(info, "Adding dep ~ts", [package_string(PackageID)]), [PackageID | Deps] end, - NewMeta = lists:keystore(deps, 1, Meta, {deps, NewDeps}), - ok = write_terms("zomp.meta", NewMeta), + NewMeta = maps:put(deps, NewDeps, Meta), + ok = write_meta(NewMeta), halt(0). --spec ensure_installed(package_id()) -> ok | no_return(). +-spec latest_version(package_id()) -> version(). +%% @private +%% Query the relevant realm for the latest version of a given package. + +latest_version({Realm, Name, Version}) -> + Socket = connect(Realm), + ok = send(Socket, {latest, Realm, Name, Version}), + Response = + receive + {tcp, Socket, Bin} -> binary_to_term(Bin, [safe]) + after 5000 -> {error, timeout} + end, + ok = disconnect(Socket). + + +-spec ensure_installed(State) -> NewState | no_return() + when State :: state(), + NewState :: state(). %% @private %% Given a PackageID, check whether it is installed on the system, and if not, ensure %% that the package is either in the cache or can be downloaded. If all attempts at %% locating or acquiring the package fail, then exit with an error. -ensure_installed(PackageID) -> +ensure_installed(State = #s{realm = Realm, name = Name, version = Version}) -> + % If the startup style is to always check first, match for the exact version, + % If the startup style is to run a matching latest and then check PackageString = package_string(PackageID), PackageDir = filename:join("lib", PackageString), case filelib:is_dir(PackageDir) of @@ -421,7 +449,7 @@ update_version(NewVersion, {Realm, Name, OldVersion}, OldMeta) -> update_version(Realm, Name, OldVersion, NewVersion, OldMeta) -> PackageID = {Realm, Name, NewVersion}, NewMeta = lists:keystore(package_id, 1, OldMeta, {package_id, PackageID}), - ok = write_terms("zomp.meta", NewMeta), + ok = write_meta(NewMeta), ok = log(info, "Version changed from ~s to ~s.", [version_to_string(OldVersion), version_to_string(NewVersion)]), @@ -444,7 +472,7 @@ drop_dep(PackageID) -> true -> NewDeps = lists:delete(PackageID, Deps), NewMeta = lists:keystore(deps, 1, Meta, {deps, NewDeps}), - ok = write_terms("zomp.meta", NewMeta), + ok = write_meta(NewMeta), Message = "~ts removed from dependencies.", ok = log(info, Message, [PackageString]), halt(0); @@ -508,15 +536,51 @@ verup(_) -> usage_exit(22). %% and use zx commands to add or drop dependencies made available via zomp. run_local(Args) -> - true = register(zx, self()), - ok = inets:start(), - {ok, ProjectRoot} = file:get_cwd(), Meta = read_meta(), {package_id, PackageID} = lists:keyfind(package_id, 1, Meta), {Realm, Name, Version} = PackageID, - ok = build(), - ok = file:set_cwd(zomp_dir()), + {type, Type} = lists:keyfind(type, 1, Meta), {deps, Deps} = lists:keyfind(deps, 1, Meta), + ok = build(), + {ok, Dir} = file:get_cwd(), + ok = file:set_cwd(zomp_dir()), + State = #s{realm = Realm, + name = Name, + version = Version, + type = Type, + deps = Deps, + dir = Dir}, + NewState = ensure_deps(State), + execute(State). + + +ensure_deps(Deps, State) -> + + execute(State). + + +execute(State = #s{type = app}) -> + true = register(zx, self()), + ok = inets:start(), + ok = log(info, "Starting ~ts", [package_string(PackageID)]), + AppMod = list_to_atom(Name), + {ok, Pid} = AppMod:start(normal, Args), + Mon = monitor(process, Pid), + Shell = spawn(shell, start, []), + ok = log(info, "Your shell is ~p, application is: ~p", [Shell, Pid]), + State = #s{realm = Realm, + name = Name, + version = Version, + pid = Pid, + mon = Mon}, + exec_wait(State); +execute(State = #s{type = lib}) -> + Message = "Lib ~ts is available on the system, but is not a standalone app", + ok = log(info, Message, [package_string(PackageID)]), + halt(0). + + + Needed = scrub(Deps), Host = {"localhost", 11411}, Socket = connect(Host, user), @@ -524,25 +588,6 @@ run_local(Args) -> ok = lists:foreach(fun install/1, Needed), ok = lists:foreach(fun build/1, Deps), ok = file:set_cwd(ProjectRoot), - case lists:keyfind(type, 1, Meta) of - {type, app} -> - ok = log(info, "Starting ~ts", [package_string(PackageID)]), - AppMod = list_to_atom(Name), - {ok, Pid} = AppMod:start(normal, Args), - Mon = monitor(process, Pid), - Shell = spawn(shell, start, []), - ok = log(info, "Your shell is ~p, application is: ~p", [Shell, Pid]), - State = #s{realm = Realm, - name = Name, - version = Version, - pid = Pid, - mon = Mon}, - exec_wait(State); - {type, lib} -> - Message = "Lib ~ts is available on the system, but is not a standalone app", - ok = log(info, Message, [package_string(PackageID)]), - halt(0) - end. @@ -556,7 +601,7 @@ run_local(Args) -> package(TargetDir) -> ok = log(info, "Packaging ~ts", [TargetDir]), - {ok, Meta} = file:consult(filename:join(TargetDir, "zomp.meta")), + Meta = read_meta(TargetDir), {package_id, {Realm, _, _}} = lists:keyfind(package_id, 1, Meta), KeyDir = filename:join([zomp_dir(), "key", Realm]), ok = force_dir(KeyDir), @@ -585,7 +630,7 @@ package(TargetDir) -> %% build a zrp package file ready to be submitted to a repository. package(KeyID, TargetDir) -> - {ok, Meta} = file:consult(filename:join(TargetDir, "zomp.meta")), + Meta = read_meta(TargetDir), {package_id, PackageID} = lists:keyfind(package_id, 1, Meta), true = element(1, PackageID) == element(1, KeyID), PackageString = package_string(PackageID), @@ -704,11 +749,9 @@ submit(PackageFile) -> {"zomp.meta", MetaBin} = lists:keyfind("zomp.meta", 1, Files), Meta = binary_to_term(MetaBin), {package_id, {Realm, Package, Version}} = lists:keyfind(package_id, 1, Meta), - {sig, {KeyID, _}} = lists:keyfind(sig, 1, Meta), + {sig, {KeyID = {Realm, KeyName}, _}} = lists:keyfind(sig, 1, Meta), true = ensure_keypair(KeyID), - RealmData = realm_data(Realm), - {prime, Prime} = lists:keyfind(prime, 1, RealmData), - Socket = connect(Prime, {auth, KeyID}), + {ok, Socket = connect_auth(Realm, KeyName), ok = send(Socket, {submit, {Realm, Package, Version}}), ok = receive @@ -730,8 +773,6 @@ submit(PackageFile) -> receive {tcp, Socket, Response2} -> log(info, "Response: ~tp", [Response2]); - Other -> - log(warning, "Unexpected message: ~tp", [Other]) after 5000 -> log(warning, "Server timed out!") end, @@ -750,67 +791,196 @@ send(Socket, Message) -> gen_tcp:send(Socket, Bin). --spec connect(Node, Type) -> gen_tcp:socket() | no_return() - when Node :: host(), - Type :: user | {auth, key_id()}. +-spec connect_user(realm()) -> gen_tcp:socket() | no_return(). %% @private -%% Connect to one of the servers in the realm constellation. +%% Connect to a given realm, whatever method is required. -connect({Host, Port}, Type) -> - Options = [{packet, 4}, {mode, binary}, {active, true}], - case gen_tcp:connect(Host, Port, Options) of +connect_user(Realm) -> + Hosts = + case file:consult(hosts_cache_file(Realm)) of + {ok, Cached} -> Cached; + {error, enoent} -> [] + end, + connect_user(Realm, Hosts). + + +-spec connect_user(realm(), [host()]) -> gen_tcp:socket() | no_return(). +%% @private +%% Try to connect to a subordinate host, if there are none then connect to prime. + +connect_user(Realm, []) -> + {Host, Port} = get_prime(Realm), + case gen_tcp:connect(Host, Port, connect_options(), 5000) of {ok, Socket} -> - confirm_server(Socket, Type); + confirm_user(Realm, Socket, []); {error, Error} -> - ok = log(warning, "Connection problem: ~tp", [Error]), + ok = log(warning, "Connection problem with prime: ~tp", [Error]), halt(0) + end; +connect_user(Realm, Hosts = [Node = {Host, Port} | Rest]) -> + case gen_tcp:connect(Host, Port, connect_options(), 5000) of + {ok, Socket} -> + confirm_user(Realm, Socket, Hosts); + {error, Error} -> + ok = log(warning, "Connection problem with ~tp: ~tp", [Node, Error]), + connect_user(Realm, Rest) end. --spec confirm_server(Socket, Type) -> gen_tcp:socket() | no_return() - when Socket :: gen_tcp:socket(), - Type :: user | {auth, key_id()}. +-spec confirm_user(Realm, Socket, Hosts) -> Socket | no_return() + when Realm :: realm(), + Socket :: gen_tcp:socket(), + Hosts :: [host()]. %% @private -%% Send a protocol ID string to notify the server what we're up to, disconnect -%% if it does not return an "OK" response within 5 seconds. +%% Confirm the zomp node can handle "OTPR USER 1" and is accepting connections or try another node. -confirm_server(Socket, user) -> +confirm_user(Realm, Socket, Hosts) -> {ok, {Addr, Port}} = inet:peername(Socket), Host = inet:ntoa(Addr), ok = gen_tcp:send(Socket, <<"OTPR USER 1">>), receive - {tcp, Socket, <<"OK">>} -> - ok = log(info, "Connected to ~s:~p", [Host, Port]), - Socket; - Other -> - Message = "Unexpected response from ~s:~p:~n~tp", - ok = log(warning, Message, [Host, Port, Other]), - ok = disconnect(Socket), - halt(0) + {tcp, Socket, Bin} -> + case binary_to_term(Bin, [safe]) of + ok -> + ok = log(info, "Connected to ~s:~p", [Host, Port]), + confirm_serial(Realm, Socket, Hosts, user); + {redirect, Next} -> + ok = log(info, "Redirected..."), + ok = disconnect(Socket), + connect_user(Realm, Next ++ Hosts) + end after 5000 -> ok = log(warning, "Host ~s:~p timed out.", [Host, Port]), - halt(0) - end; -confirm_server(Socket, {auth, KeyID}) -> - ok = log(info, "Would now be trying to connect as AUTH using ~tp", [KeyID]), + ok = disconnect(Socket), + connect_user(Realm, tl(Hosts)) + end. + + +-spec confirm_serial(Realm, Socket, Hosts) -> Socket | no_return() + when Realm :: realm(), + Socket :: gen_tcp:socket(), + Hosts :: [host()]. +%% @private +%% Confirm that the connected host has a valid serial for the realm zx is trying to +%% reach, and if not retry on another node. + +confirm_serial(Realm, Socket, Hosts) -> + SerialFile = filename:join(zomp_dir(), "realm.serials"), + Serials = + case file:consult(SerialFile) of + {ok, Serials} -> Serials; + {error, enoent} -> [] + end, + Serial = + case lists:keyfind(Realm, 1, Serials) of + false -> 1; + {Realm, S} -> S + end, + ok = send(Socket, {latest, Realm}), + receive + {tcp, Socket, Bin} -> + case binary_to_term(Bin, [safe]) of + {ok, Serial} -> + ok = log(info, "Node's serial same as ours."), + Socket; + {ok, Current} when Current > Serial -> + ok = log(info, "Node's serial newer than ours. Storing."), + NewSerials = lists:keystore(Realm, 1, Current, Serials), + ok = write_terms(hosts_cache_file(Realm), Hosts), + ok = write_terms(SerialFile, NewSerials), + Socket; + {ok, Current} when Current < Serial -> + ok = log(info, "Node's serial older than ours. Trying another."), + ok = disconnect(Socket), + connect_user(Realm, tl(Hosts)); + {error, bad_realm} -> + ok = log(info, "Node is no longer serving realm. Trying another."), + ok = disconnect(Socket), + connect_user(Realm, tl(Hosts)) + end + after 5000 -> + ok = log(info, "Host timed out on confirm_serial. Trying another."), + ok = disconnect(Socket), + connect_user(Realm, tl(Hosts)) + end. + + +-spec connect_auth(Realm, KeyName) -> Result + when Realm :: realm(), + KeyName :: key_name(), + Result :: {ok, gen_tcp:socket()} + | {error, Reason :: term()}. +%% @private +%% Connect to one of the servers in the realm constellation. + +connect_auth(Realm, KeyName) -> + {ok, Key} = loadkey(private, {Realm, KeyName}), + {Host, Port} = get_prime(Realm), + case gen_tcp:connect(Host, Port, connect_options(), 5000) of + {ok, Socket} -> + ok = log(info, "Connected to ~tp prime.", [Realm]), + confirm_auth(Socket, Key); + Error = {error, E} -> + ok = log(warning, "Connection problem: ~tp", [E]), + {error, Error} + end. + + + +-spec confirm_auth(Socket, Key) -> Result + when Socket :: gen_tcp:socket(), + Key :: term(), + Result :: {ok, gen_tcp:socket()} + | {error, timeout}. +%% @private +%% Send a protocol ID string to notify the server what we're up to, disconnect +%% if it does not return an "OK" response within 5 seconds. + +confirm_auth(Socket, Key) -> + ok = log(info, "Would be using key ~tp now", [Key]), {ok, {Addr, Port}} = inet:peername(Socket), - Host = inet:ntoa(Addr), ok = gen_tcp:send(Socket, <<"OTPR AUTH 1">>), receive {tcp, Socket, <<"OK">>} -> - ok = log(info, "Connected to ~s:~p", [Host, Port]), - Socket; - Other -> - Message = "Unexpected response from ~s:~p:~n~tp", - ok = log(warning, Message, [Host, Port, Other]), - ok = disconnect(Socket), - halt(0) + {ok, Socket} after 5000 -> ok = log(warning, "Host ~s:~p timed out.", [Host, Port]), - halt(0) + {error, auth_timeout} end. +-spec connect_options() -> [gen_tcp:connect_option()]. +%% @private +%% Hide away the default options used for TCP connections. + +connect_options() -> + [{packet, 4}, {mode, binary}, {active, true}]. + + +-spec get_prime(realm()) -> host(). +%% @private +%% Check the given Realm's config file for the current prime node and return it. + +get_prime(Realm) -> + RealmFile = filename:join(zomp_dir(), Realm ++ ".realm"), + case file:consult(RealmFile) of + {ok, RealmMeta} -> + {prime, Prime} = lists:keyfind(prime, 1, RealmMeta), + Prime; + {error, enoent} -> + ok = log(error, "Missing realm file for ~tp (~tp).", [Realm, RealmFile]), + halt(1) + end. + + +-spec hosts_cache_file(realm()) -> file:filename(). +%% @private +%% Given a Realm name, construct a realm's .hosts filename and return it. + +hosts_cache_file(Realm) -> + filename:join(zomp_dir(), Realm ++ ".hosts"). + + -spec disconnect(gen_tcp:socket()) -> ok. %% @private %% Gracefully shut down a socket, logging (but sidestepping) the case when the socket @@ -1219,22 +1389,6 @@ extract_zrp(FileName) -> end. --spec read_meta() -> [term()] | no_return(). -%% @private -%% Read the `zomp.meta' file from the current directory, if possible. If not possible -%% then halt execution with an appropriate error message. - -read_meta() -> - case file:consult("zomp.meta") of - {ok, Meta} -> - Meta; - Error -> - ok = log(error, "Failed to open \"zomp.meta\" with ~tp", [Error]), - ok = log(error, "Wrong directory?"), - halt(1) - end. - - -spec verify(Data, Signature, PubKey) -> ok | no_return() when Data :: binary(), Signature :: binary(), @@ -1300,6 +1454,52 @@ fetch(Socket, Needed) -> %%% Utility functions +-spec read_meta() -> package_meta() | no_return(). +%% @private +%% @equiv read_meta(".") + +read_meta() -> + read_meta("."). + + +-spec read_meta(Dir) -> package_meta() | no_return() + when Dir :: file:filename(). +%% @private +%% Read the `zomp.meta' file from the indicated directory, if possible. If not possible +%% then halt execution with an appropriate error message. + +read_meta(Dir) -> + Path = filename:join(Dir, "zomp.meta"), + case file:consult(Path) of + {ok, Meta} -> + maps:from_list(Meta); + Error -> + ok = log(error, "Failed to open \"zomp.meta\" with ~tp", [Error]), + ok = log(error, "Wrong directory?"), + halt(1) + end. + + +-spec write_meta(package_meta()) -> ok. +%% @private +%% @equiv write_meta(".") + +write_meta() -> + write_meta("."). + + +-spec write_meta(Dir, Meta) -> ok + when Dir :: file:filename(), + Meta :: package_meta(). +%% @private +%% Write the contents of the provided meta structure (a map these days) as a list of +%% Erlang K/V terms. + +write_meta(Dir, Meta) -> + Path = filename:join(Dir, "zomp.meta"), + ok = write_terms(Path, maps:to_list(Meta)). + + -spec write_terms(Filename, Terms) -> ok when Filename :: file:filename(), Terms :: [term()]. @@ -1351,6 +1551,8 @@ build() -> %% Take a list of dependencies and return a list of dependencies that are not yet %% installed on the system. +scrub([]) -> + []; scrub(Deps) -> {ok, Names} = file:list_dir("lib"), Existing = lists:map(fun package_id/1, Names), @@ -1382,7 +1584,8 @@ valid_lower0_9([$_ | _], $_) -> false; valid_lower0_9([Char | Rest], _) when $a =< Char, Char =< $z; - $0 =< Char, Char =< $9 -> + $0 =< Char, Char =< $9; + Char == $_ -> valid_lower0_9(Rest, Char); valid_lower0_9([], _) -> true;