pharpend/zx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

messy

Browse Source
This commit is contained in:
Craig Everett 2017-11-14 08:05:24 +09:00
parent fafea57954
commit 34656bef64
1 changed files with 318 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

433
zx
View File

@ -21,6 +21,10 @@
{realm = "otpr" :: name(),
name = none :: none | name(),
version = {z, z, z} :: version(),
type = app :: app | lib
deps = [] :: [package_id()],
dir = none :: none | file:filename(),
socket = none :: none | gen_tcp:socket(),
pid = none :: none | pid(),
mon = none :: none | reference()}).
@ -39,7 +43,8 @@
%-type keybin() :: {ID :: key_id(),
% Type :: public | private,
% DER :: binary()}.
-type key_id() :: {realm(), KeyName :: lower0_9()}.
-type key_id() :: {realm(), key_name()}.
-type key_name() :: lower0_9().
-type lower0_9() :: [$a..$z | $0..$9 | $_].
%-type label() :: [$a..$z | $0..$9 | $_ | $- | $.].
@ -64,7 +69,7 @@ main(Args) ->
start(["help"]) ->
usage_exit(0);
start(["run", PackageString | Args]) ->
execute(PackageString, Args);
run(PackageString, Args);
start(["init", "app", PackageString]) ->
PackageID = package_id(PackageString),
initialize(app, PackageID);
@ -84,9 +89,7 @@ start(["drop", "key", KeyID]) ->
drop_key(KeyID);
start(["verup", Level]) ->
verup(Level);
start(["runlocal"]) ->
run_local([]);
start(["runlocal", Args]) ->
start(["runlocal" | Args]) ->
run_local(Args);
start(["package"]) ->
{ok, TargetDir} = file:get_cwd(),
@ -115,7 +118,7 @@ start(_) ->
%%% Execution of application
-spec execute(Identifier, Args) -> no_return()
-spec run(Identifier, Args) -> no_return()
when Identifier :: string(),
Args :: [string()].
%% @private
@ -123,7 +126,7 @@ start(_) ->
%% dependencies and run the program. This implies determining whether the program and
%% its dependencies are installed, available, need to be downloaded, or are inaccessible
%% given the current system condition (they could also be bogus, of course). The
%% Identifier provided should be a valid PackageString of the form `realm-appname-version'
%% Identifier should be a valid PackageString of the form `realm-appname-version'
%% where the realm and appname should follow standard realm and app package naming
%% conventions and the version should be represented as a semver in string form (where
%% ommitted elements of the version always default to whatever is most current).
@ -135,15 +138,21 @@ start(_) ->
%% If there is a problem anywhere in the locationg, discovery, building, and loading
%% procedure the runtime will halt with an error message.
execute(Identifier, Args) ->
true = register(zx, self()),
ok = inets:start(),
run(Identifier, Args) ->
PackageID = {Realm, Name, Version} = package_id(Identifier),
ok = file:set_cwd(zomp_dir()),
PackageRoot = filename:join("lib", Identifier),
ok = ensure_installed(PackageID),
{ok, Meta} = file:consult(filename:join(PackageRoot, "zomp.meta")),
{deps, Deps} = lists:keyfind(deps, 1, Meta),
State = #s{realm = Realm,
name = Name,
version = Version,
dir = PackageRoot},
NextState = ensure_installed(State),
Meta = read_meta(PackageRoot),
Deps = maps:get(deps, Meta),
NewState = ensure_deps(NextState#s{deps = Deps}),
execute(State).
Required = [PackageID | Deps],
Needed = scrub(Required),
Host = {"localhost", 11411},
@ -152,8 +161,10 @@ execute(Identifier, Args) ->
ok = lists:foreach(fun install/1, Needed),
ok = lists:foreach(fun build/1, Required),
ok = file:set_cwd(PackageRoot),
case lists:keyfind(type, 1, Meta) of
{type, app} ->
case maps:get(type, Meta) of
app ->
true = register(zx, self()),
ok = inets:start(),
ok = log(info, "Starting ~ts", [package_string(PackageID)]),
PackageMod = list_to_atom(Name),
{ok, Pid} = PackageMod:start(normal, Args),
@ -166,7 +177,7 @@ execute(Identifier, Args) ->
pid = Pid,
mon = Mon},
exec_wait(State);
{type, lib} ->
lib ->
Message = "Lib ~ts is available on the system, but is not a standalone app.",
ok = log(info, Message, [package_string(PackageID)]),
halt(0)
@ -195,7 +206,7 @@ initialize(Type, PackageID) ->
Meta = [{package_id, PackageID},
{deps, []},
{type, Type}],
ok = write_terms("zomp.meta", Meta),
ok = write_meta(Meta),
ok = log(info, "Project ~tp initialized.", [PackageString]),
Message =
"NOTICE:~n"
@ -224,10 +235,10 @@ assimilate(PackageFile) ->
ok = file:set_cwd(zomp_dir()),
{"zomp.meta", MetaBin} = lists:keyfind("zomp.meta", 1, Files),
Meta = binary_to_term(MetaBin),
{package_id, PackageID} = lists:keyfind(package_id, 1, Meta),
PackageID = maps:get(package_id, Meta),
TgzFile = namify_tgz(PackageID),
{TgzFile, TgzData} = lists:keyfind(TgzFile, 1, Files),
{sig, {KeyID, Signature}} = lists:keyfind(sig, 1, Meta),
{KeyID, Signature} = maps:get(sig, Meta),
{ok, PubKey} = loadkey(public, KeyID),
ok =
case public_key:verify(TgzData, sha512, Signature, PubKey) of
@ -259,7 +270,7 @@ assimilate(PackageFile) ->
set_dep(PackageString) ->
PackageID = package_id(PackageString),
Meta = read_meta(),
{deps, Deps} = lists:keyfind(deps, 1, Meta),
Deps = maps:get(deps, Meta),
case lists:member(PackageID, Deps) of
true ->
ok = log(info, "~ts is already a dependency", [PackageString]),
@ -279,11 +290,9 @@ set_dep(PackageString) ->
%% such a dependency is not already present. Then write the project meta back to its
%% file and exit.
set_dep(PackageID = {Realm, Name, NewVersion}, Deps, Meta) ->
set_dep(PackageID = {Realm, Name, Version}, Deps, Meta) ->
{ok, CWD} = file:get_cwd(),
ok = file:set_cwd(zomp_dir()),
ok = ensure_installed(PackageID),
ok = file:set_cwd(CWD),
LatestVersion = latest_version(PackageID),
ExistingPackageIDs = fun ({R, N, _}) -> {R, N} == {Realm, Name} end,
NewDeps =
case lists:partition(ExistingPackageIDs, Deps) of
@ -297,18 +306,37 @@ set_dep(PackageID = {Realm, Name, NewVersion}, Deps, Meta) ->
ok = log(info, "Adding dep ~ts", [package_string(PackageID)]),
[PackageID | Deps]
end,
NewMeta = lists:keystore(deps, 1, Meta, {deps, NewDeps}),
ok = write_terms("zomp.meta", NewMeta),
NewMeta = maps:put(deps, NewDeps, Meta),
ok = write_meta(NewMeta),
halt(0).
-spec ensure_installed(package_id()) -> ok | no_return().
-spec latest_version(package_id()) -> version().
%% @private
%% Query the relevant realm for the latest version of a given package.
latest_version({Realm, Name, Version}) ->
Socket = connect(Realm),
ok = send(Socket, {latest, Realm, Name, Version}),
Response =
receive
{tcp, Socket, Bin} -> binary_to_term(Bin, [safe])
after 5000 -> {error, timeout}
end,
ok = disconnect(Socket).
-spec ensure_installed(State) -> NewState | no_return()
when State :: state(),
NewState :: state().
%% @private
%% Given a PackageID, check whether it is installed on the system, and if not, ensure
%% that the package is either in the cache or can be downloaded. If all attempts at
%% locating or acquiring the package fail, then exit with an error.
ensure_installed(PackageID) ->
ensure_installed(State = #s{realm = Realm, name = Name, version = Version}) ->
% If the startup style is to always check first, match for the exact version,
% If the startup style is to run a matching latest and then check
PackageString = package_string(PackageID),
PackageDir = filename:join("lib", PackageString),
case filelib:is_dir(PackageDir) of
@ -421,7 +449,7 @@ update_version(NewVersion, {Realm, Name, OldVersion}, OldMeta) ->
update_version(Realm, Name, OldVersion, NewVersion, OldMeta) ->
PackageID = {Realm, Name, NewVersion},
NewMeta = lists:keystore(package_id, 1, OldMeta, {package_id, PackageID}),
ok = write_terms("zomp.meta", NewMeta),
ok = write_meta(NewMeta),
ok = log(info,
"Version changed from ~s to ~s.",
[version_to_string(OldVersion), version_to_string(NewVersion)]),
@ -444,7 +472,7 @@ drop_dep(PackageID) ->
true ->
NewDeps = lists:delete(PackageID, Deps),
NewMeta = lists:keystore(deps, 1, Meta, {deps, NewDeps}),
ok = write_terms("zomp.meta", NewMeta),
ok = write_meta(NewMeta),
Message = "~ts removed from dependencies.",
ok = log(info, Message, [PackageString]),
halt(0);
@ -508,15 +536,51 @@ verup(_) -> usage_exit(22).
%% and use zx commands to add or drop dependencies made available via zomp.
run_local(Args) ->
true = register(zx, self()),
ok = inets:start(),
{ok, ProjectRoot} = file:get_cwd(),
Meta = read_meta(),
{package_id, PackageID} = lists:keyfind(package_id, 1, Meta),
{Realm, Name, Version} = PackageID,
ok = build(),
ok = file:set_cwd(zomp_dir()),
{type, Type} = lists:keyfind(type, 1, Meta),
{deps, Deps} = lists:keyfind(deps, 1, Meta),
ok = build(),
{ok, Dir} = file:get_cwd(),
ok = file:set_cwd(zomp_dir()),
State = #s{realm = Realm,
name = Name,
version = Version,
type = Type,
deps = Deps,
dir = Dir},
NewState = ensure_deps(State),
execute(State).
ensure_deps(Deps, State) ->
execute(State).
execute(State = #s{type = app}) ->
true = register(zx, self()),
ok = inets:start(),
ok = log(info, "Starting ~ts", [package_string(PackageID)]),
AppMod = list_to_atom(Name),
{ok, Pid} = AppMod:start(normal, Args),
Mon = monitor(process, Pid),
Shell = spawn(shell, start, []),
ok = log(info, "Your shell is ~p, application is: ~p", [Shell, Pid]),
State = #s{realm = Realm,
name = Name,
version = Version,
pid = Pid,
mon = Mon},
exec_wait(State);
execute(State = #s{type = lib}) ->
Message = "Lib ~ts is available on the system, but is not a standalone app",
ok = log(info, Message, [package_string(PackageID)]),
halt(0).
Needed = scrub(Deps),
Host = {"localhost", 11411},
Socket = connect(Host, user),
@ -524,25 +588,6 @@ run_local(Args) ->
ok = lists:foreach(fun install/1, Needed),
ok = lists:foreach(fun build/1, Deps),
ok = file:set_cwd(ProjectRoot),
case lists:keyfind(type, 1, Meta) of
{type, app} ->
ok = log(info, "Starting ~ts", [package_string(PackageID)]),
AppMod = list_to_atom(Name),
{ok, Pid} = AppMod:start(normal, Args),
Mon = monitor(process, Pid),
Shell = spawn(shell, start, []),
ok = log(info, "Your shell is ~p, application is: ~p", [Shell, Pid]),
State = #s{realm = Realm,
name = Name,
version = Version,
pid = Pid,
mon = Mon},
exec_wait(State);
{type, lib} ->
Message = "Lib ~ts is available on the system, but is not a standalone app",
ok = log(info, Message, [package_string(PackageID)]),
halt(0)
end.
@ -556,7 +601,7 @@ run_local(Args) ->
package(TargetDir) ->
ok = log(info, "Packaging ~ts", [TargetDir]),
{ok, Meta} = file:consult(filename:join(TargetDir, "zomp.meta")),
Meta = read_meta(TargetDir),
{package_id, {Realm, _, _}} = lists:keyfind(package_id, 1, Meta),
KeyDir = filename:join([zomp_dir(), "key", Realm]),
ok = force_dir(KeyDir),
@ -585,7 +630,7 @@ package(TargetDir) ->
%% build a zrp package file ready to be submitted to a repository.
package(KeyID, TargetDir) ->
{ok, Meta} = file:consult(filename:join(TargetDir, "zomp.meta")),
Meta = read_meta(TargetDir),
{package_id, PackageID} = lists:keyfind(package_id, 1, Meta),
true = element(1, PackageID) == element(1, KeyID),
PackageString = package_string(PackageID),
@ -704,11 +749,9 @@ submit(PackageFile) ->
{"zomp.meta", MetaBin} = lists:keyfind("zomp.meta", 1, Files),
Meta = binary_to_term(MetaBin),
{package_id, {Realm, Package, Version}} = lists:keyfind(package_id, 1, Meta),
{sig, {KeyID, _}} = lists:keyfind(sig, 1, Meta),
{sig, {KeyID = {Realm, KeyName}, _}} = lists:keyfind(sig, 1, Meta),
true = ensure_keypair(KeyID),
RealmData = realm_data(Realm),
{prime, Prime} = lists:keyfind(prime, 1, RealmData),
Socket = connect(Prime, {auth, KeyID}),
{ok, Socket = connect_auth(Realm, KeyName),
ok = send(Socket, {submit, {Realm, Package, Version}}),
ok =
receive
@ -730,8 +773,6 @@ submit(PackageFile) ->
receive
{tcp, Socket, Response2} ->
log(info, "Response: ~tp", [Response2]);
Other ->
log(warning, "Unexpected message: ~tp", [Other])
after 5000 ->
log(warning, "Server timed out!")
end,
@ -750,67 +791,196 @@ send(Socket, Message) ->
gen_tcp:send(Socket, Bin).
-spec connect(Node, Type) -> gen_tcp:socket() | no_return()
when Node :: host(),
Type :: user | {auth, key_id()}.
-spec connect_user(realm()) -> gen_tcp:socket() | no_return().
%% @private
%% Connect to one of the servers in the realm constellation.
%% Connect to a given realm, whatever method is required.
connect({Host, Port}, Type) ->
Options = [{packet, 4}, {mode, binary}, {active, true}],
case gen_tcp:connect(Host, Port, Options) of
connect_user(Realm) ->
Hosts =
case file:consult(hosts_cache_file(Realm)) of
{ok, Cached} -> Cached;
{error, enoent} -> []
end,
connect_user(Realm, Hosts).
-spec connect_user(realm(), [host()]) -> gen_tcp:socket() | no_return().
%% @private
%% Try to connect to a subordinate host, if there are none then connect to prime.
connect_user(Realm, []) ->
{Host, Port} = get_prime(Realm),
case gen_tcp:connect(Host, Port, connect_options(), 5000) of
{ok, Socket} ->
confirm_server(Socket, Type);
confirm_user(Realm, Socket, []);
{error, Error} ->
ok = log(warning, "Connection problem: ~tp", [Error]),
ok = log(warning, "Connection problem with prime: ~tp", [Error]),
halt(0)
end;
connect_user(Realm, Hosts = [Node = {Host, Port} | Rest]) ->
case gen_tcp:connect(Host, Port, connect_options(), 5000) of
{ok, Socket} ->
confirm_user(Realm, Socket, Hosts);
{error, Error} ->
ok = log(warning, "Connection problem with ~tp: ~tp", [Node, Error]),
connect_user(Realm, Rest)
end.
-spec confirm_server(Socket, Type) -> gen_tcp:socket() | no_return()
when Socket :: gen_tcp:socket(),
Type :: user | {auth, key_id()}.
-spec confirm_user(Realm, Socket, Hosts) -> Socket | no_return()
when Realm :: realm(),
Socket :: gen_tcp:socket(),
Hosts :: [host()].
%% @private
%% Send a protocol ID string to notify the server what we're up to, disconnect
%% if it does not return an "OK" response within 5 seconds.
%% Confirm the zomp node can handle "OTPR USER 1" and is accepting connections or try another node.
confirm_server(Socket, user) ->
confirm_user(Realm, Socket, Hosts) ->
{ok, {Addr, Port}} = inet:peername(Socket),
Host = inet:ntoa(Addr),
ok = gen_tcp:send(Socket, <<"OTPR USER 1">>),
receive
{tcp, Socket, <<"OK">>} ->
ok = log(info, "Connected to ~s:~p", [Host, Port]),
Socket;
Other ->
Message = "Unexpected response from ~s:~p:~n~tp",
ok = log(warning, Message, [Host, Port, Other]),
ok = disconnect(Socket),
halt(0)
{tcp, Socket, Bin} ->
case binary_to_term(Bin, [safe]) of
ok ->
ok = log(info, "Connected to ~s:~p", [Host, Port]),
confirm_serial(Realm, Socket, Hosts, user);
{redirect, Next} ->
ok = log(info, "Redirected..."),
ok = disconnect(Socket),
connect_user(Realm, Next ++ Hosts)
end
after 5000 ->
ok = log(warning, "Host ~s:~p timed out.", [Host, Port]),
halt(0)
end;
confirm_server(Socket, {auth, KeyID}) ->
ok = log(info, "Would now be trying to connect as AUTH using ~tp", [KeyID]),
ok = disconnect(Socket),
connect_user(Realm, tl(Hosts))
end.
-spec confirm_serial(Realm, Socket, Hosts) -> Socket | no_return()
when Realm :: realm(),
Socket :: gen_tcp:socket(),
Hosts :: [host()].
%% @private
%% Confirm that the connected host has a valid serial for the realm zx is trying to
%% reach, and if not retry on another node.
confirm_serial(Realm, Socket, Hosts) ->
SerialFile = filename:join(zomp_dir(), "realm.serials"),
Serials =
case file:consult(SerialFile) of
{ok, Serials} -> Serials;
{error, enoent} -> []
end,
Serial =
case lists:keyfind(Realm, 1, Serials) of
false -> 1;
{Realm, S} -> S
end,
ok = send(Socket, {latest, Realm}),
receive
{tcp, Socket, Bin} ->
case binary_to_term(Bin, [safe]) of
{ok, Serial} ->
ok = log(info, "Node's serial same as ours."),
Socket;
{ok, Current} when Current > Serial ->
ok = log(info, "Node's serial newer than ours. Storing."),
NewSerials = lists:keystore(Realm, 1, Current, Serials),
ok = write_terms(hosts_cache_file(Realm), Hosts),
ok = write_terms(SerialFile, NewSerials),
Socket;
{ok, Current} when Current < Serial ->
ok = log(info, "Node's serial older than ours. Trying another."),
ok = disconnect(Socket),
connect_user(Realm, tl(Hosts));
{error, bad_realm} ->
ok = log(info, "Node is no longer serving realm. Trying another."),
ok = disconnect(Socket),
connect_user(Realm, tl(Hosts))
end
after 5000 ->
ok = log(info, "Host timed out on confirm_serial. Trying another."),
ok = disconnect(Socket),
connect_user(Realm, tl(Hosts))
end.
-spec connect_auth(Realm, KeyName) -> Result
when Realm :: realm(),
KeyName :: key_name(),
Result :: {ok, gen_tcp:socket()}
| {error, Reason :: term()}.
%% @private
%% Connect to one of the servers in the realm constellation.
connect_auth(Realm, KeyName) ->
{ok, Key} = loadkey(private, {Realm, KeyName}),
{Host, Port} = get_prime(Realm),
case gen_tcp:connect(Host, Port, connect_options(), 5000) of
{ok, Socket} ->
ok = log(info, "Connected to ~tp prime.", [Realm]),
confirm_auth(Socket, Key);
Error = {error, E} ->
ok = log(warning, "Connection problem: ~tp", [E]),
{error, Error}
end.
-spec confirm_auth(Socket, Key) -> Result
when Socket :: gen_tcp:socket(),
Key :: term(),
Result :: {ok, gen_tcp:socket()}
| {error, timeout}.
%% @private
%% Send a protocol ID string to notify the server what we're up to, disconnect
%% if it does not return an "OK" response within 5 seconds.
confirm_auth(Socket, Key) ->
ok = log(info, "Would be using key ~tp now", [Key]),
{ok, {Addr, Port}} = inet:peername(Socket),
Host = inet:ntoa(Addr),
ok = gen_tcp:send(Socket, <<"OTPR AUTH 1">>),
receive
{tcp, Socket, <<"OK">>} ->
ok = log(info, "Connected to ~s:~p", [Host, Port]),
Socket;
Other ->
Message = "Unexpected response from ~s:~p:~n~tp",
ok = log(warning, Message, [Host, Port, Other]),
ok = disconnect(Socket),
halt(0)
{ok, Socket}
after 5000 ->
ok = log(warning, "Host ~s:~p timed out.", [Host, Port]),
halt(0)
{error, auth_timeout}
end.
-spec connect_options() -> [gen_tcp:connect_option()].
%% @private
%% Hide away the default options used for TCP connections.
connect_options() ->
[{packet, 4}, {mode, binary}, {active, true}].
-spec get_prime(realm()) -> host().
%% @private
%% Check the given Realm's config file for the current prime node and return it.
get_prime(Realm) ->
RealmFile = filename:join(zomp_dir(), Realm ++ ".realm"),
case file:consult(RealmFile) of
{ok, RealmMeta} ->
{prime, Prime} = lists:keyfind(prime, 1, RealmMeta),
Prime;
{error, enoent} ->
ok = log(error, "Missing realm file for ~tp (~tp).", [Realm, RealmFile]),
halt(1)
end.
-spec hosts_cache_file(realm()) -> file:filename().
%% @private
%% Given a Realm name, construct a realm's .hosts filename and return it.
hosts_cache_file(Realm) ->
filename:join(zomp_dir(), Realm ++ ".hosts").
-spec disconnect(gen_tcp:socket()) -> ok.
%% @private
%% Gracefully shut down a socket, logging (but sidestepping) the case when the socket
@ -1219,22 +1389,6 @@ extract_zrp(FileName) ->
end.
-spec read_meta() -> [term()] | no_return().
%% @private
%% Read the `zomp.meta' file from the current directory, if possible. If not possible
%% then halt execution with an appropriate error message.
read_meta() ->
case file:consult("zomp.meta") of
{ok, Meta} ->
Meta;
Error ->
ok = log(error, "Failed to open \"zomp.meta\" with ~tp", [Error]),
ok = log(error, "Wrong directory?"),
halt(1)
end.
-spec verify(Data, Signature, PubKey) -> ok | no_return()
when Data :: binary(),
Signature :: binary(),
@ -1300,6 +1454,52 @@ fetch(Socket, Needed) ->
%%% Utility functions
-spec read_meta() -> package_meta() | no_return().
%% @private
%% @equiv read_meta(".")
read_meta() ->
read_meta(".").
-spec read_meta(Dir) -> package_meta() | no_return()
when Dir :: file:filename().
%% @private
%% Read the `zomp.meta' file from the indicated directory, if possible. If not possible
%% then halt execution with an appropriate error message.
read_meta(Dir) ->
Path = filename:join(Dir, "zomp.meta"),
case file:consult(Path) of
{ok, Meta} ->
maps:from_list(Meta);
Error ->
ok = log(error, "Failed to open \"zomp.meta\" with ~tp", [Error]),
ok = log(error, "Wrong directory?"),
halt(1)
end.
-spec write_meta(package_meta()) -> ok.
%% @private
%% @equiv write_meta(".")
write_meta() ->
write_meta(".").
-spec write_meta(Dir, Meta) -> ok
when Dir :: file:filename(),
Meta :: package_meta().
%% @private
%% Write the contents of the provided meta structure (a map these days) as a list of
%% Erlang K/V terms.
write_meta(Dir, Meta) ->
Path = filename:join(Dir, "zomp.meta"),
ok = write_terms(Path, maps:to_list(Meta)).
-spec write_terms(Filename, Terms) -> ok
when Filename :: file:filename(),
Terms :: [term()].
@ -1351,6 +1551,8 @@ build() ->
%% Take a list of dependencies and return a list of dependencies that are not yet
%% installed on the system.
scrub([]) ->
[];
scrub(Deps) ->
{ok, Names} = file:list_dir("lib"),
Existing = lists:map(fun package_id/1, Names),
@ -1382,7 +1584,8 @@ valid_lower0_9([$_ | _], $_) ->
false;
valid_lower0_9([Char | Rest], _)
when $a =< Char, Char =< $z;
$0 =< Char, Char =< $9 ->
$0 =< Char, Char =< $9;
Char == $_ ->
valid_lower0_9(Rest, Char);
valid_lower0_9([], _) ->
true;