diff --git a/README.md b/README.md index b759e8e..bd21514 100644 --- a/README.md +++ b/README.md @@ -1,1428 +1,15 @@ -# gex = gajumaru exchange +gex: gajumaru exchange +===================================================================== -Currently there is only one thing, which is the Gajumaru HTTP Daemon. +Currently everything is in `gex_httpd`, which is the HTTP Daemon. -## How to run `gex_httpd` +Only application so far. -Last updated: September 23, 2025 (PRH). +Branches +--------------------------------------------------------------------- -### Prereq: Install Erlang and zx/zomp - -Source: [*Building Erlang 26.2.5 on Ubuntu 24.04*](https://zxq9.com/archives/2905) - -Adapt this to your Linux distribution. - -1. **Install necessary build tools** - - ```bash - sudo apt update - sudo apt upgrade - sudo apt install \ - gcc curl g++ dpkg-dev build-essential automake autoconf \ - libncurses-dev libssl-dev flex xsltproc libwxgtk3.2-dev \ - wget vim git - ``` - -2. **Put [Kerl](https://github.com/kerl/kerl) somewhere - in your `$PATH`**. This is a tool to build Erlang releases. - - ```bash - wget -O ~/bin/kerl https://raw.githubusercontent.com/kerl/kerl/master/kerl - chmod u+x ~/bin/kerl - ``` - -3. **Build Erlang from source using Kerl** - - ```bash - kerl update releases - ## use the most recent one that looks stable - ## you do need to type the number twice, that's not a typo - kerl build 28.1 28.1 - kerl install 28.1 ~/.erts/28.1 - ``` - -4. **Put Erlang in your `$PATH`** - - Update .bashrc or .zshrc or whatever with the following line: - - ```bash - . $HOME/.erts/28.1/activate - ``` - - -5. **Install zx** - - ```bash - wget -q https://zxq9.com/projects/zomp/get_zx && bash get_zx - ``` - -6. **Test zx works** - - zx installs itself to `~/bin`, so make sure that's in your - `$PATH`. - - ```bash - zx run erltris - ``` - -### Running the server - -``` -zxh runlocal -``` - -Then navigate to in your browser. - -![](./etc/hello-world.png) - -## Notes - -### Convention: \[brackets\] for jargon - -- You know how sometimes people will intermix technical jargon which has a very - specific context-local definition with common parlance? -- Our notational convention is to put \[jargon terms\] in square braces to warn - the reader that the word is meant in some extremely precise technical sense, - and the word doesn't necessarily mean what the dictionary says it means. -- Specifically, \[supervisor\] is a jargon term that is standard in Erlang. -- Do not confuse \[supervisor\] with \[manager\]. \[Manager\] is AFAIK - Craig-specific nomenclature. -- A \[supervisor\] is (roughly) a process that is in charge of a bunch of child - processes. It is responsible for restarting processes that crash. (Yes Craig - I know it's more nuanced than that). -- The other common pattern is a \[`gen_server`\]. -- If all you take away from this document is that erlang has things called - \[supervisor\]s and things called \[`gen_server`\]s, consider that a good - day. - -### Big Picture: telnet chat server -> HTTP server - -- The default project (see initial commit) is a telnet echo server. It's like - the most ghetto low-budget chat server imaginable. - - ![screenshot of `gh_httpd` running](./etc/gh-telnet.png) - - Lefty and middley can chat just like normal. - - However, righty (`curl`) foolishly thinks he is talking to an HTTP server. - His request is echoed to lefty and middley. - - Curl crashed because instead of a valid HTTP response back, he got something - like - - ``` - MESSAGE from YOU: GET / HTTP/1.1 - ``` - -- We make this into an HTTP server by replacing the "echo my message to - everyone else" logic with "parse this message as an HTTP request and send - back an HTTP response" logic. -- Our "application logic" or "business logic" or whatever is contained in that - process of how the request is mapped to a response. -- It really is not more complicated than that. - -### Basics of Erlang Processes - -These are heuristics that are good starting points - -- each module ~= 1 process -- it helps to think of erlang as an operating system, and erlang modules as - shell scripts that run in that operating system. -- some modules correspond to fungible processes, some are non-fungible -- in Observer (`observer:start()`) - - named processes are non-fungible (e.g. `gh_client_sup`) - - the name can be anything, but conventionally it's the module name - - fungible processes have numbers (PIDs) (e.g. the - `gh_client` code, which is the Erlang process that was - on the other end of the conversation with the `telnet` - windows) - - named processes also have PIDs, they just also have names - - ![](./etc/observer.png) -- you will want to get in the habit of any time you read code, always asking - what process context the code is running in. -- it is **NOT** the case that all code in module `foo` runs inside the context - of process `foo`. It is **very** important that you make sure you understand - that distinction, and always know where code is running. - - -### Following the call chain of `gex_httpd:listen(8080)` - -- Reference commit: `49a09d192c6f2380c5186ec7d81e98785d667214` -- By default, the telnet server doesn't occupy a port -- `gex_httpd:listen(8080)` tells it to listen on port 8080 - - ```erlang - %% gex_httpd.erl - - -spec listen(PortNum) -> Result - when PortNum :: inet:port_num(), - Result :: ok - | {error, {listening, inet:port_num()}}. - %% @doc - %% Make the server start listening on a port. - %% Returns an {error, Reason} tuple if it is already listening. - - listen(PortNum) -> - gh_client_man:listen(PortNum). - - - %% gh_client_man.erl - - listen(PortNum) -> - gen_server:call(?MODULE, {listen, PortNum}). - ``` - -- So this is a bit tricky. -- The code inside that function runs in the context of the `gex_httpd` - process (or whatever calling process) - - - the effect of that code is to send a message to the `gh_client_man` - process (`= ?MODULE`) - - that message is `{listen, 8080}` - - in general, it's `gen_server:call(PID, Message)` - - every process has a "mailbox" of messages. the process usually just sits - there doing nothing until it gets a message, and then does something - deterministically in response to the message. - - `gen_server`, \[`supervisor`\], etc, all are standard library factoring-outs - of common patterns of process configuration - - The low-level primitive to receive messages is `receive`. We'll see its - use later when we look at the `gh_client` code. - - See [pingpong example](./etc/pingpong.erl) for a simplified example. - [Permalink.](https://git.qpq.swiss/QPQ-AG/gex/src/commit/28193491e7dabadac9bac756b44ee2ea3869eede/etc/pingpong.erl) - - All of this `gen_server` nonsense is a bunch of boilerplate that rewrites - to a bunch of `receive`s - -#### Very Important: casts v. calls - -- **Very important:** sometimes you will also see `gen_server:cast(PID, - Message).` - - It's very important that you understand the difference - -- So in our example - - `gex_httpd` makes a call to `gh_client_man` - - `gex_httpd` sends the message `{listen, 8080}` to `gh_client_man` - - he is going to sit there and wait until `gh_client_man` sends him a - message back. this is what makes a call a call. if `gh_client_man` never - responds, `gex_httpd` will just sit there forever waiting, and never move - on with his life. - -- in a cast, the message is sent and you move on with your day -- think of calls like actual phone calls, where the other person has to - answer, but if they don't, there's no voicemail, the phone just rings - forever and you're just stuck listening to the phone ringing like sisyphus - (there's an option of course to call with a timeout, etc... simplifying). -- casts are like text messages. You send them. maybe you get a text back. who - knows. - -#### Continuing - -- Inside of `gh_client_man`'s own process context, it listens for calls in the - `handle_call` function - - ```erlang - %% gh_client_man.erl - - -spec handle_call(Message, From, State) -> Result - when Message :: term(), - From :: {pid(), reference()}, - State :: state(), - Result :: {reply, Response, NewState} - | {noreply, State}, - Response :: ok - | {error, {listening, inet:port_number()}}, - NewState :: state(). - %% @private - %% The gen_server:handle_call/3 callback. - %% See: http://erlang.org/doc/man/gen_server.html#Module:handle_call-3 - - handle_call({listen, PortNum}, _, State) -> - {Response, NewState} = do_listen(PortNum, State), - {reply, Response, NewState}; - handle_call(Unexpected, From, State) -> - ok = io:format("~p Unexpected call from ~tp: ~tp~n", [self(), From, Unexpected]), - {noreply, State}. - ``` - -- following the call chain, we look at `gh_client_man:do_listen/2` - - This is running inside the `gh_client_man` process context - - ```erlang - %% gh_client_man.erl - - -spec do_listen(PortNum, State) -> {Result, NewState} - when PortNum :: inet:port_number(), - State :: state(), - Result :: ok - | {error, Reason :: {listening, inet:port_number()}}, - NewState :: state(). - %% @private - %% The "doer" procedure called when a "listen" message is received. - - do_listen(PortNum, State = #s{port_num = none}) -> - SocketOptions = - [inet6, - {packet, line}, - {active, once}, - {mode, binary}, - {keepalive, true}, - {reuseaddr, true}], - {ok, Listener} = gen_tcp:listen(PortNum, SocketOptions), - {ok, _} = gh_client:start(Listener), - {ok, State#s{port_num = PortNum, listener = Listener}}; - do_listen(_, State = #s{port_num = PortNum}) -> - ok = io:format("~p Already listening on ~p~n", [self(), PortNum]), - {{error, {listening, PortNum}}, State}. - ``` - - - If we're already listening (i.e. our state already has a port), we tell - the calling process to fuck off. - - - If we don't have a port number, we - - make a TCP listen socket on that port - - start a `gh_client` process which spawns an acceptor socket on the - listen socket (kind of a "subsocket") - - the `gh_client` process is the erlang process that talks to either - the telnet chat clients, or eventually web browsers. - - send `ok` back to whomever called us - - -- Next let's look at how clients are started up - -- `gh_client` is called `gh_client` because from the perspective of our HTTP - daemon, that is a client. `gh_client` is the representation of clients within - the context of our HTTP server. -- analogously, to disambiguate directionality re "encode"/"decode", usually the - directionality is from the perspective of the program. This can be - counterintuitive, because the program's perspective is usually the opposite - of a human's; e.g. Binary data is clear to a program, but its representation - as plain text is opaque. -- In Erlang you always need to think about perspective - -- Last call in the call chain was `gh_client:start()`. We expect it to return - the PID of the process that talks to clients. - -``` -%% gh_client.erl - --spec start(ListenSocket) -> Result - when ListenSocket :: gen_tcp:socket(), - Result :: {ok, pid()} - | {error, Reason}, - Reason :: {already_started, pid()} - | {shutdown, term()} - | term(). -%% @private -%% How the gh_client_man or a prior gh_client kicks things off. -%% This is called in the context of gh_client_man or the prior gh_client. - -start(ListenSocket) -> - gh_client_sup:start_acceptor(ListenSocket). - - -%% gh_client_sup.erl - --spec start_acceptor(ListenSocket) -> Result - when ListenSocket :: gen_tcp:socket(), - Result :: {ok, pid()} - | {error, Reason}, - Reason :: {already_started, pid()} - | {shutdown, term()} - | term(). -%% @private -%% Spawns the first listener at the request of the gh_client_man when -%% gex_httpd:listen/1 is called, or the next listener at the request of the -%% currently listening gh_client when a connection is made. -%% -%% Error conditions, supervision strategies and other important issues are -%% explained in the supervisor module docs: -%% http://erlang.org/doc/man/supervisor.html - -start_acceptor(ListenSocket) -> - supervisor:start_child(?MODULE, [ListenSocket]). -``` - -- Reference: - ![](./etc/observer.png) - -- `gh_client_sup` is the \[supervisor\] responsible for restarting client - processes when they crash. -- he is tasked at this moment with starting one. Let's see how that goes - - ```erlang - %% gh_client_sup.erl - - -spec start_acceptor(ListenSocket) -> Result - when ListenSocket :: gen_tcp:socket(), - Result :: {ok, pid()} - | {error, Reason}, - Reason :: {already_started, pid()} - | {shutdown, term()} - | term(). - %% @private - %% Spawns the first listener at the request of the gh_client_man when - %% gex_httpd:listen/1 is called, or the next listener at the request of the - %% currently listening gh_client when a connection is made. - %% - %% Error conditions, supervision strategies and other important issues are - %% explained in the supervisor module docs: - %% http://erlang.org/doc/man/supervisor.html - - start_acceptor(ListenSocket) -> - supervisor:start_child(?MODULE, [ListenSocket]). - ``` - -- If we look in the configuration for `gh_client_sup`, we see this: - - ```erlang - -spec init(none) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. - %% @private - %% The OTP init/1 function. - - init(none) -> - RestartStrategy = {simple_one_for_one, 1, 60}, - Client = {gh_client, - {gh_client, start_link, []}, - temporary, - brutal_kill, - worker, - [gh_client]}, - {ok, {RestartStrategy, [Client]}}. - ``` - -- my eyes are drawn to `{gh_client, start_link, []}` -- probably that's what's called to spawn one of the worker processes -- let's look - - - ```erlang - %% gh_client.erl - - -spec start_link(ListenSocket) -> Result - when ListenSocket :: gen_tcp:socket(), - Result :: {ok, pid()} - | {error, Reason}, - Reason :: {already_started, pid()} - | {shutdown, term()} - | term(). - %% @private - %% This is called by the gh_client_sup. While start/1 is called to iniate a startup - %% (essentially requesting a new worker be started by the supervisor), this is - %% actually called in the context of the supervisor. - - start_link(ListenSocket) -> - proc_lib:start_link(?MODULE, init, [self(), ListenSocket]). - ``` - -- Any time you see a 3-tuple of `{Module, FunctionName, ArgumentList}`, - probably that's information about how to call some function - -- In this case, this is saying "to start one of the `gh_client` processes, we - call `gh_client:init(SupervisorPID, ListenSocket)`" - - Let's take a look - - ```erlang - %% gh_client.erl - - -spec init(Parent, ListenSocket) -> no_return() - when Parent :: pid(), - ListenSocket :: gen_tcp:socket(). - %% @private - %% This is the first code executed in the context of the new worker itself. - %% This function does not have any return value, as the startup return is - %% passed back to the supervisor by calling proc_lib:init_ack/2. - %% We see the initial form of the typical arity-3 service loop form here in the - %% call to listen/3. - - init(Parent, ListenSocket) -> - ok = io:format("~p Listening.~n", [self()]), - Debug = sys:debug_options([]), - ok = proc_lib:init_ack(Parent, {ok, self()}), - listen(Parent, Debug, ListenSocket). - ``` - -- Ok let's look at the `listen/3` function - - ```erlang - -spec listen(Parent, Debug, ListenSocket) -> no_return() - when Parent :: pid(), - Debug :: [sys:dbg_opt()], - ListenSocket :: gen_tcp:socket(). - %% @private - %% This function waits for a TCP connection. The owner of the socket is still - %% the gh_client_man (so it can still close it on a call to gh_client_man:ignore/0), - %% but the only one calling gen_tcp:accept/1 on it is this process. Closing the socket - %% is one way a manager process can gracefully unblock child workers that are blocking - %% on a network accept. - %% - %% Once it makes a TCP connection it will call start/1 to spawn its successor. - - listen(Parent, Debug, ListenSocket) -> - case gen_tcp:accept(ListenSocket) of - {ok, Socket} -> - {ok, _} = start(ListenSocket), - {ok, Peer} = inet:peername(Socket), - ok = io:format("~p Connection accepted from: ~p~n", [self(), Peer]), - ok = gh_client_man:enroll(), - State = #s{socket = Socket}, - loop(Parent, Debug, State); - {error, closed} -> - ok = io:format("~p Retiring: Listen socket closed.~n", [self()]), - exit(normal) - end. - ``` - -- The lines that jump out to me are - - ```erlang - ok = gh_client_man:enroll(), - State = #s{socket = Socket}, - loop(Parent, Debug, State); - ``` - -- The `gh_client_man` module is responsible for keeping track of all the - running clients. So probably `gh_client_man:enroll(self())` is just - informing `gh_client_man` that this `gh_client` instance exists. - - If we look, that's precisely what's happening - - ```erlang - %% gh_client_man.erl - %% remember, enroll/0 is running in the context of the calling code, and - %% do_enroll/2 is running in the context of the gh_client_man process - - -spec enroll() -> ok. - %% @doc - %% Clients register here when they establish a connection. - %% Other processes can enroll as well. - - enroll() -> - gen_server:cast(?MODULE, {enroll, self()}). - - %% ... - -spec do_enroll(Pid, State) -> NewState - when Pid :: pid(), - State :: state(), - NewState :: state(). - - do_enroll(Pid, State = #s{clients = Clients}) -> - case lists:member(Pid, Clients) of - false -> - Mon = monitor(process, Pid), - ok = io:format("Monitoring ~tp @ ~tp~n", [Pid, Mon]), - State#s{clients = [Pid | Clients]}; - true -> - State - end. - ``` - -- Next line is `loop(Parent, Debug, State)`. Let's look at `gh_client:loop/3` - - ```erlang - -spec loop(Parent, Debug, State) -> no_return() - when Parent :: pid(), - Debug :: [sys:dbg_opt()], - State :: state(). - %% @private - %% The service loop itself. This is the service state. The process blocks on receive - %% of Erlang messages, TCP segments being received themselves as Erlang messages. - - loop(Parent, Debug, State = #s{socket = Socket}) -> - ok = inet:setopts(Socket, [{active, once}]), - receive - {tcp, Socket, <<"bye\r\n">>} -> - ok = io:format("~p Client saying goodbye. Bye!~n", [self()]), - ok = gen_tcp:send(Socket, "Bye!\r\n"), - ok = gen_tcp:shutdown(Socket, read_write), - exit(normal); - {tcp, Socket, Message} -> - ok = io:format("~p received: ~tp~n", [self(), Message]), - ok = gh_client_man:echo(Message), - loop(Parent, Debug, State); - {relay, Sender, Message} when Sender == self() -> - ok = gen_tcp:send(Socket, ["Message from YOU: ", Message]), - loop(Parent, Debug, State); - {relay, Sender, Message} -> - From = io_lib:format("Message from ~tp: ", [Sender]), - ok = gen_tcp:send(Socket, [From, Message]), - loop(Parent, Debug, State); - {tcp_closed, Socket} -> - ok = io:format("~p Socket closed, retiring.~n", [self()]), - exit(normal); - {system, From, Request} -> - sys:handle_system_msg(Request, From, Parent, ?MODULE, Debug, State); - Unexpected -> - ok = io:format("~p Unexpected message: ~tp", [self(), Unexpected]), - loop(Parent, Debug, State) - end. - ``` - -- I'll let you figure this one out -- I think the picture is clear. There's a lot of moving parts, but the basic - principle is as follows: - - - `gh_client` instances are like infinity-spawn receptionists. Every time a - web browser wants to talk to our server, we spawn a `gh_client` instance - that talks to the web browser. - - `gh_client_man` is responsible for any logic that spans across different - `gh_client` instances (e.g. relaying messages). - - Everything else is boilerplate -- So our task is to remove the relay-messages logic, and replace it with http - parse/respond logic. - - -### Auto-listening to port `8000` - -- Start commit: `9c5b332e009b88a9400a4e7008f760ce872b810a` - -```diff -diff --git a/gex_httpd/src/gex_httpd.erl b/gex_httpd/src/gex_httpd.erl -index 242c82f..3a724b0 100644 ---- a/gex_httpd/src/gex_httpd.erl -+++ b/gex_httpd/src/gex_httpd.erl -@@ -9,8 +9,11 @@ - -copyright("Peter Harpending "). - - -+%% for our edification - -export([listen/1, ignore/0]). ---export([start/0, start/1]). -+-export([start/0]). -+ -+%% erlang expects us to export these functions - -export([start/2, stop/1]). - - -@@ -45,17 +48,17 @@ start() -> - io:format("Starting..."). - - ---spec start(PortNum) -> ok -- when PortNum :: inet:port_number(). --%% @doc --%% Start the server and begin listening immediately. Slightly more convenient when --%% playing around in the shell. -- --start(PortNum) -> -- ok = start(), -- ok = gh_client_man:listen(PortNum), -- io:format("Startup complete, listening on ~w~n", [PortNum]). -- -+%-spec start(PortNum) -> ok -+% when PortNum :: inet:port_number(). -+%%% @doc -+%%% Start the server and begin listening immediately. Slightly more convenient when -+%%% playing around in the shell. -+% -+%start(PortNum) -> -+% ok = start(), -+% ok = gh_client_man:listen(PortNum), -+% io:format("Startup complete, listening on ~w~n", [PortNum]). -+% - - -spec start(normal, term()) -> {ok, pid()}. - %% @private -@@ -64,7 +67,10 @@ start(PortNum) -> - %% See: http://erlang.org/doc/apps/kernel/application.html - - start(normal, _Args) -> -- gh_sup:start_link(). -+ Result = gh_sup:start_link(), -+ % auto-listen to port 8000 -+ ok = listen(8000), -+ Result. - - - -spec stop(term()) -> ok. -``` - -### Deleting the relay logic - -- Start commit: `2037444f54d8f95a959c7e67673d55782fad5ad3` - -```diff -diff --git a/gex_httpd/src/gh_client.erl b/gex_httpd/src/gh_client.erl -index ea5b3ef..44d206d 100644 ---- a/gex_httpd/src/gh_client.erl -+++ b/gex_httpd/src/gh_client.erl -@@ -127,22 +127,10 @@ listen(Parent, Debug, ListenSocket) -> - loop(Parent, Debug, State = #s{socket = Socket}) -> - ok = inet:setopts(Socket, [{active, once}]), - receive -- {tcp, Socket, <<"bye\r\n">>} -> -- ok = io:format("~p Client saying goodbye. Bye!~n", [self()]), -- ok = gen_tcp:send(Socket, "Bye!\r\n"), -- ok = gen_tcp:shutdown(Socket, read_write), -- exit(normal); - {tcp, Socket, Message} -> - ok = io:format("~p received: ~tp~n", [self(), Message]), - ok = gh_client_man:echo(Message), - loop(Parent, Debug, State); -- {relay, Sender, Message} when Sender == self() -> -- ok = gen_tcp:send(Socket, ["Message from YOU: ", Message]), -- loop(Parent, Debug, State); -- {relay, Sender, Message} -> -- From = io_lib:format("Message from ~tp: ", [Sender]), -- ok = gen_tcp:send(Socket, [From, Message]), -- loop(Parent, Debug, State); - {tcp_closed, Socket} -> - ok = io:format("~p Socket closed, retiring.~n", [self()]), - exit(normal); -diff --git a/gex_httpd/src/gh_client_man.erl b/gex_httpd/src/gh_client_man.erl -index 9c78921..c913dcc 100644 ---- a/gex_httpd/src/gh_client_man.erl -+++ b/gex_httpd/src/gh_client_man.erl -@@ -17,7 +17,7 @@ - - - -export([listen/1, ignore/0]). ---export([enroll/0, echo/1]). -+-export([enroll/0]). - -export([start_link/0]). - -export([init/1, handle_call/3, handle_cast/2, handle_info/2, - code_change/3, terminate/2]). -@@ -74,17 +74,6 @@ enroll() -> - gen_server:cast(?MODULE, {enroll, self()}). - - ---spec echo(Message) -> ok -- when Message :: string(). --%% @doc --%% The function that tells the manager to broadcast a message to all clients. --%% This can broadcast arbitrary strings to clients from non-clients as well. -- --echo(Message) -> -- gen_server:cast(?MODULE, {echo, Message, self()}). -- -- -- - %%% Startup Functions - - -@@ -145,9 +134,6 @@ handle_call(Unexpected, From, State) -> - handle_cast({enroll, Pid}, State) -> - NewState = do_enroll(Pid, State), - {noreply, NewState}; --handle_cast({echo, Message, Sender}, State) -> -- ok = do_echo(Message, Sender, State), -- {noreply, State}; - handle_cast(ignore, State) -> - NewState = do_ignore(State), - {noreply, NewState}; -@@ -262,17 +248,6 @@ do_enroll(Pid, State = #s{clients = Clients}) -> - end. - - ---spec do_echo(Message, Sender, State) -> ok -- when Message :: string(), -- Sender :: pid(), -- State :: state(). --%% @private --%% The "doer" procedure called when an "echo" message is received. -- --do_echo(Message, Sender, #s{clients = Clients}) -> -- Send = fun(Client) -> Client ! {relay, Sender, Message} end, -- lists:foreach(Send, Clients). -- - - -spec handle_down(Mon, Pid, Reason, State) -> NewState - when Mon :: reference(), -``` - -### Adding QHL as a dependency - -[QHL](https://git.qpq.swiss/qPQ-AG/qhl) is our own internal parsing library for -HTTP/1.1 requests - -```bash -zx set dep otpr-qhl-0.1.0 -``` - -```diff -diff --git a/gex_httpd/zomp.meta b/gex_httpd/zomp.meta -index 7111f2d..09d84c2 100644 ---- a/gex_httpd/zomp.meta -+++ b/gex_httpd/zomp.meta -@@ -1,17 +1,17 @@ --{a_email,"peterharpending@qpq.swiss"}. -+{name,"Gajumaru DEX HTTP Daemon"}. -+{type,app}. -+{modules,[]}. -+{prefix,"gh"}. - {author,"Peter Harpending"}. -+{desc,"Gajumaru Exchange HTTP Daemon"}. -+{package_id,{"otpr","gex_httpd",{0,1,0}}}. -+{deps,[{"otpr","qhl",{0,1,0}}]}. -+{key_name,none}. -+{a_email,"peterharpending@qpq.swiss"}. - {c_email,"peterharpending@qpq.swiss"}. - {copyright,"Peter Harpending"}. --{deps,[]}. --{desc,"Gajumaru Exchange HTTP Daemon"}. - {file_exts,[]}. --{key_name,none}. - {license,skip}. --{modules,[]}. --{name,"Gajumaru DEX HTTP Daemon"}. --{package_id,{"otpr","gex_httpd",{0,1,0}}}. --{prefix,"gh"}. - {repo_url,"https://git.qpq.swiss/QPQ-AG/gex"}. - {tags,[]}. --{type,app}. - {ws_url,"https://git.qpq.swiss/QPQ-AG/gex"}. -``` - -### Adding the QHL header file - -```bash -wget https://git.qpq.swiss/QPQ-AG/QHL/raw/branch/master/include/http.hrl -O gex_httpd/include/http.hrl -``` - -```diff -diff --git a/gex_httpd/include/http.hrl b/gex_httpd/include/http.hrl -new file mode 100644 -index 0000000..27d24d2 ---- /dev/null -+++ b/gex_httpd/include/http.hrl -@@ -0,0 +1,25 @@ -+-record(request, -+ {method = undefined :: undefined | method(), -+ path = undefined :: undefined | binary(), -+ qargs = undefined :: undefined | #{Key :: binary() := Value :: binary()}, -+ fragment = undefined :: undefined | none | binary(), -+ version = undefined :: undefined | http10 | http11 | http20, -+ headers = undefined :: undefined | [{Key :: binary(), Value :: binary()}], -+ cookies = undefined :: undefined | #{Key :: binary() := Value :: binary()}, -+ enctype = undefined :: undefined | none | urlencoded | multipart(), -+ size = undefined :: undefined | none | non_neg_integer(), -+ body = undefined :: undefined | none | body()}). -+ -+-record(response, -+ {type = page :: page | {data, string()}, -+ version = http11 :: http11, -+ code = 200 :: pos_integer(), -+ slogan = "" :: string(), -+ headers = [] :: [{Key :: string(), Value :: iolist()}], -+ body = "" :: iolist()}). -+ -+-type method() :: get | post | options. -+-type multipart() :: {multipart, Boundary :: binary()}. -+-type body() :: {partial, binary()} | {multipart, [body_part()]} | binary(). -+-type body_part() :: {Field :: binary(), Data :: binary()} -+ | {Field :: binary(), Name :: binary(), Data :: binary()}. -``` - -### Making an index.html - -```diff -diff --git a/gex_httpd/priv/index.html b/gex_httpd/priv/index.html -new file mode 100644 -index 0000000..2ba545c ---- /dev/null -+++ b/gex_httpd/priv/index.html -@@ -0,0 +1,11 @@ -+ -+ -+ -+ -+ Hello, world! -+ -+ -+ -+

Hello, world!

-+ -+ -``` - -### Serving the index.html page - -- Big picture steps: - - have `gh_client.erl` parse whatever it gets on the socket into a `qhl:request()` - - write a function that takes in the `qhl:request()` and returns a `qhl:response()` - - write a function that takes the `qhl:response()` and render it into binary - - send that binary back over the socket - -- [QHL reference](https://git.qpq.swiss/QPQ-AG/QHL/src/commit/7f77f9e3b19f58006df88a2a601e85835d300c37/src/qhl.erl) - -```diff -diff --git a/gex_httpd/priv/404.html b/gex_httpd/priv/404.html -new file mode 100644 -index 0000000..bfb09f3 ---- /dev/null -+++ b/gex_httpd/priv/404.html -@@ -0,0 +1,11 @@ -+ -+ -+ -+ -+ QHL: 404 -+ -+ -+ -+

404 Not Found

-+ -+ -diff --git a/gex_httpd/priv/500.html b/gex_httpd/priv/500.html -new file mode 100644 -index 0000000..19d2057 ---- /dev/null -+++ b/gex_httpd/priv/500.html -@@ -0,0 +1,11 @@ -+ -+ -+ -+ -+ QHL: 500 -+ -+ -+ -+

500 Internal Server Error

-+ -+ -diff --git a/gex_httpd/src/gh_client.erl b/gex_httpd/src/gh_client.erl -index 44d206d..3abd46f 100644 ---- a/gex_httpd/src/gh_client.erl -+++ b/gex_httpd/src/gh_client.erl -@@ -24,11 +24,14 @@ - -export([system_continue/3, system_terminate/4, - system_get_state/1, system_replace_state/2]). - -+-include("http.hrl"). -+ - - %%% Type and Record Definitions - - ---record(s, {socket = none :: none | gen_tcp:socket()}). -+-record(s, {socket = none :: none | gen_tcp:socket(), -+ received = none :: none | binary()}). - - - %% An alias for the state record above. Aliasing state can smooth out annoyances -@@ -124,13 +127,38 @@ listen(Parent, Debug, ListenSocket) -> - %% The service loop itself. This is the service state. The process blocks on receive - %% of Erlang messages, TCP segments being received themselves as Erlang messages. - --loop(Parent, Debug, State = #s{socket = Socket}) -> -+loop(Parent, Debug, State = #s{socket = Socket, received = Received}) -> - ok = inet:setopts(Socket, [{active, once}]), - receive - {tcp, Socket, Message} -> - ok = io:format("~p received: ~tp~n", [self(), Message]), -- ok = gh_client_man:echo(Message), -- loop(Parent, Debug, State); -+ %% Received exists because web browsers usually use the same -+ %% acceptor socket for sequential requests -+ %% -+ %% QHL parses a request off the socket, and consumes all the data -+ %% pertinent to said task. Any additional data it finds on the -+ %% socket it hands back to us. -+ %% -+ %% That additional data, as I said, is usually the next request. -+ %% -+ %% We store that in our process state in the received=Received field -+ Message2 = -+ case Received of -+ none -> Message; -+ _ -> <> -+ end, -+ %% beware: wrong typespec in QHL 0.1.0 -+ %% see: https://git.qpq.swiss/QPQ-AG/QHL/pulls/1 -+ case qhl:parse(Socket, Message2) of -+ {ok, Request, NewReceived} -> -+ ok = handle_request(Socket, Request), -+ NewState = State#s{received = NewReceived}, -+ loop(Parent, Debug, NewState); -+ {error, Reason} -> -+ io:format("~p error: ~tp~n", [self(), Reason]), -+ ok = http_err(Socket, 500), -+ exit(normal) -+ end; - {tcp_closed, Socket} -> - ok = io:format("~p Socket closed, retiring.~n", [self()]), - exit(normal); -@@ -190,3 +218,85 @@ system_get_state(State) -> {ok, State}. - - system_replace_state(StateFun, State) -> - {ok, StateFun(State), State}. -+ -+ -+%%%------------------------------------------- -+%%% http request handling -+%%%------------------------------------------- -+ -+-spec handle_request(Socket, Request) -> ok -+ when Socket :: gen_tcp:socket(), -+ Request :: #request{}. -+ -+%% ref: https://git.qpq.swiss/QPQ-AG/QHL/src/commit/7f77f9e3b19f58006df88a2a601e85835d300c37/include/http.hrl -+ -+handle_request(Socket, #request{method = get, path = <<"/">>}) -> -+ IndexHtmlPath = filename:join([zx:get_home(), "priv", "index.html"]), -+ case file:read_file(IndexHtmlPath) of -+ {ok, ResponseBody} -> -+ %% see https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Messages#http_responses -+ Headers = [{"content-type", "text/html"}], -+ Response = #response{headers = Headers, -+ body = ResponseBody}, -+ respond(Socket, Response); -+ Error -> -+ io:format("~p error: ~p~n", [self(), Error]), -+ http_err(Socket, 500) -+ end; -+handle_request(Socket, _) -> -+ http_err(Socket, 404). -+ -+ -+http_err(Socket, 404) -> -+ HtmlPath = filename:join([zx:get_home(), "priv", "404.html"]), -+ {ok, ResponseBody} = file:read_file(HtmlPath), -+ Headers = [{"content-type", "text/html"}], -+ Response = #response{headers = Headers, -+ code = 404, -+ body = ResponseBody}, -+ respond(Socket, Response); -+% default error is 500 -+http_err(Socket, _) -> -+ HtmlPath = filename:join([zx:get_home(), "priv", "500.html"]), -+ {ok, ResponseBody} = file:read_file(HtmlPath), -+ Headers = [{"content-type", "text/html"}], -+ Response = #response{headers = Headers, -+ code = 500, -+ body = ResponseBody}, -+ respond(Socket, Response). -+ -+ -+respond(Socket, R = #response{code = Code, headers = Headers, body = Body}) -> -+ Slogan = slogan(Code), -+ ContentLength = byte_size(Body), -+ DefaultHeaders = [{"date", qhl:ridiculous_web_date()}, -+ {"content-length", integer_to_list(ContentLength)}], -+ Headers2 = merge_headers(DefaultHeaders, Headers), -+ really_respond(Socket, R#response{slogan = Slogan, -+ headers = Headers2}). -+ -+ -+really_respond(Socket, #response{code = Code, slogan = Slogan, headers = Headers, body = Body}) -> -+ Response = -+ ["HTTP/1.1 ", integer_to_list(Code), " ", Slogan, "\r\n", -+ render_headers(Headers), "\r\n", -+ Body], -+ gen_tcp:send(Socket, Response). -+ -+ -+merge_headers(Defaults, Overwrites) -> -+ DefaultsMap = proplists:to_map(Defaults), -+ OverwritesMap = proplists:to_map(Overwrites), -+ FinalMap = maps:merge(DefaultsMap, OverwritesMap), -+ proplists:from_map(FinalMap). -+ -+render_headers([{K, V} | Rest]) -> -+ [K, ": ", V, "\r\n", -+ render_headers(Rest)]; -+render_headers([]) -> -+ []. -+ -+ -+slogan(200) -> "OK"; -+slogan(404) -> "Not Found"; -+slogan(500) -> "Internal Server Error". -``` - -## Video 2: Talking to the gm testnet (2025-09-25) - -- - -### Adding hz as a dependency - -- [`hakuzaru`](https://git.qpq.swiss/QPQ-AG/hakuzaru/) is our Erlang - \[application\] (applications are the subset of what in another language you - would call "libraries" that spawn processes, essentially...) for talking to - the blockchain from an Erlang application. - -``` -$ zx list versions hakuzaru -0.6.1 -0.6.0 -0.5.2 -0.5.1 -0.5.0 -0.4.0 -0.3.2 -0.3.1 -0.3.0 -0.2.0 -0.1.0 -$ zx set dep otpr-hakuzaru-0.6.1 -``` - -If we just try to use this naively, we will run into errors whenever we try to -call code that calls into hz's dependencies. - -So we also have to manually include all of hz's dependencies. - -Thankfully, hz is also subject to this same constraint, so we only have to just -copy them from hz's zomp.meta file. - -```diff -diff --git a/gex_httpd/zomp.meta b/gex_httpd/zomp.meta -index 442e32e..eb32a14 100644 ---- a/gex_httpd/zomp.meta -+++ b/gex_httpd/zomp.meta -@@ -5,7 +5,18 @@ - {author,"Peter Harpending"}. - {desc,"Gajumaru Exchange HTTP Daemon"}. - {package_id,{"otpr","gex_httpd",{0,1,0}}}. --{deps,[{"otpr","hakuzaru",{0,6,1}},{"otpr","qhl",{0,1,0}}]}. -+{deps,[ -+ {"otpr","hakuzaru",{0,6,1}}, -+ {"otpr","sophia",{9,0,0}}, -+ {"otpr","gmserialization",{0,1,3}}, -+ {"otpr","gmbytecode",{3,4,1}}, -+ {"otpr","base58",{0,1,1}}, -+ {"otpr","eblake2",{1,0,1}}, -+ {"otpr","ec_utils",{1,0,0}}, -+ {"otpr","zj",{1,1,0}}, -+ {"otpr","getopt",{1,0,2}}, -+ {"otpr","qhl",{0,1,0}} -+]}. - {key_name,none}. - {a_email,"peterharpending@qpq.swiss"}. - {c_email,"peterharpending@qpq.swiss"}. -``` - -### talking to testnet - -```diff -diff --git a/gex_httpd/src/gex_httpd.erl b/gex_httpd/src/gex_httpd.erl -index 3a724b0..a2ed213 100644 ---- a/gex_httpd/src/gex_httpd.erl -+++ b/gex_httpd/src/gex_httpd.erl -@@ -69,10 +69,28 @@ start() -> - start(normal, _Args) -> - Result = gh_sup:start_link(), - % auto-listen to port 8000 -+ ok = hz(), - ok = listen(8000), -+ ok = io:format("~p~n", [hz:status()]), - Result. - - -+hz() -> -+ ok = application:ensure_started(hakuzaru), -+ ok = hz:chain_nodes([testnet_node()]), -+ ok = zx:tell("hz status: ~tp", [hz:status()]), -+ ok. -+ -+testnet_ip() -> -+ {84, 46, 242, 9}. -+ -+testnet_port() -> -+ 3013. -+ -+testnet_node() -> -+ {testnet_ip(), testnet_port()}. -+ -+ - -spec stop(term()) -> ok. - %% @private - %% Similar to start/2 above, this is to be called by the "application" part of OTP, -``` - -```erlang -hz status: {ok,#{"difficulty" => 3172644578, - "finalized" => - #{"hash" => - "kh_2vPQ8Q8QYQF1tSWRrSjPJ5YKSCfLD4TKEfeAvuNsdkWzVkFspp", - "height" => 215186,"type" => "height"}, - "genesis_key_block_hash" => - "kh_Qdi5MTuuhJm7xzn5JUAbYG12cX3qoLMnXrBxPGzBkMWJ4K8vq", - "hashrate" => 953086,"listening" => true, - "network_id" => "groot.testnet", - "node_revision" => "7b3cc1db3bb36053023167b86f7d6f2d5dcbd01d", - "node_version" => "0.1.0+203.7b3cc1db3", - "peer_connections" => #{"inbound" => 1,"outbound" => 3}, - "peer_count" => 4, - "peer_pubkey" => - "pp_2nQHucGyEt5wkYruNuRkg19cbZuEeyR9BZfvtv49F3AoyNSYMT", - "pending_transactions_count" => 0, - "protocols" => [#{"effective_at_height" => 0,"version" => 1}], - "solutions" => 0,"sync_progress" => 100.0,"syncing" => false, - "top_block_height" => 215287, - "top_hash" => - "kh_2TX5p81WtTX3y82NPdfWwv7yuehDh6aMRh1Uy6GBS5JsdkaGXu", - "top_key_block_hash" => - "kh_2TX5p81WtTX3y82NPdfWwv7yuehDh6aMRh1Uy6GBS5JsdkaGXu"}} -``` - -### Testnet info - -- Explorer: (status) -- Faucet: -- Middleware: (status) -- Endpoint: (status) - -### Deploying a contract to testnet - -- Deployed to: `ct_2PbZyDvyECxnwVvL5Y1ryHciY9J1EJmNmGWtP1uEJW3dn73MEv` -- [Explorer link](http://84.46.242.9:5001/contract/ct_2PbZyDvyECxnwVvL5Y1ryHciY9J1EJmNmGWtP1uEJW3dn73MEv) - -#### Goal - -Take this contract: - -```c -/** - * Hello world contract in sophia - * - * Copyright (C) 2025, QPQ AG - */ - -@compiler 9.0.0 - -contract Hello = - type state = () - - entrypoint - init : () => state - init() = () - - entrypoint - hello : () => string - hello() = "hello" -``` - -Deploy it to the testnet, call `hello` entrypoint, and get the result back - -#### Big picture - -Steps: -1. Need a keypair (someone who owns the contract) -> [`ec_utils` library](https://github.com/hanssv/ec_utils) -2. Give our public key to the [faucet][tn-faucet] and get some gas money -3. Use [`hz:contract_create/3`](https://git.qpq.swiss/QPQ-AG/hakuzaru/src/commit/b13af3d0822762df167ac56da89e30cf8372c673/src/hz.erl#L859-L884) - to make a `ContractCreateTx` -4. Use [`hz:prepare_contract/1`](https://git.qpq.swiss/QPQ-AG/hakuzaru/src/commit/b13af3d0822762df167ac56da89e30cf8372c673/src/hz.erl#L1395-L1407) - to get an "AACI", which is a data structure that maps Erlang types - (integers, strings, tuples, lists, etc) to FATE types. - - Essentially when we go from Sophia (source language) to FATE (VM bytecode), - a lot of information is lost. The AACI is the information that is lost, - which is precisely what is needed to translate back and forth between Erlang - and FATE data. - -#### Deploying diff - -Start point: `d99fefcd1540d5ded0c000c5608992805217bd25` - -```diff -diff --git a/gex_httpd/src/gex_httpd.erl b/gex_httpd/src/gex_httpd.erl -index 0cb09bd..f9d7a4f 100644 ---- a/gex_httpd/src/gex_httpd.erl -+++ b/gex_httpd/src/gex_httpd.erl -@@ -8,7 +8,6 @@ - -author("Peter Harpending "). - -copyright("Peter Harpending "). - -- - %% for our edification - -export([listen/1, ignore/0]). - -export([start/0]). -@@ -17,6 +16,12 @@ - -export([start/2, stop/1]). - - -+-include("$zx_include/zx_logger.hrl"). -+ -+%------------------------------------------------------ -+% API -+%------------------------------------------------------ -+ - - -spec listen(PortNum) -> Result - when PortNum :: inet:port_num(), -@@ -77,7 +82,7 @@ start(normal, _Args) -> - hz() -> - ok = application:ensure_started(hakuzaru), - ok = hz:chain_nodes([testnet_node()]), -- ok = zx:tell("hz status: ~tp", [hz:status()]), -+ ok = tell("hz status: ~tp", [hz:status()]), - ok. - - testnet_ip() -> -diff --git a/gex_httpd/src/gh_ct.erl b/gex_httpd/src/gh_ct.erl -new file mode 100644 -index 0000000..212a679 ---- /dev/null -+++ b/gex_httpd/src/gh_ct.erl -@@ -0,0 +1,164 @@ -+% @doc miscellaneous contract functions -+% -+% mostly wrappers for ec_utils and hakuzaru -+-module(gh_ct). -+ -+ -+-export_type([ -+ keypair/0 -+]). -+ -+-export([ -+ deploy/2, -+ get_pubkey_akstr/0, get_keypair/0, -+ keypair_file/0, -+ read_keypair_from_file/1, write_keypair_to_file/2, fmt_keypair/1, -+ fmt_pubkey_api/1, -+ gen_keypair/0 -+]). -+ -+-include("$zx_include/zx_logger.hrl"). -+ -+%------------------------------------------------------ -+% API: types -+%------------------------------------------------------ -+ -+ -+-type keypair() :: #{public := binary(), -+ secret := binary()}. -+ -+ -+%------------------------------------------------------ -+% API: functions -+%------------------------------------------------------ -+ -+-spec deploy(ContractSrcPath, InitArgs) -> Result -+ when ContractSrcPath :: string(), -+ InitArgs :: term(), -+ Result :: {ok, term()} -+ | {error, term()}. %% FIXME -+ -+deploy(ContractSrcPath, InitArgs) -> -+ CreatorId = get_pubkey_akstr(), -+ case hz:contract_create(CreatorId, ContractSrcPath, InitArgs) of -+ {ok, ContractCreateTx} -> -+ push(ContractCreateTx); -+ Error -> -+ tell(error, "gh_ct:deploy(~tp, ~tp) error: ~tp", [ContractSrcPath, InitArgs, Error]), -+ Error -+ end. -+ -+push(ContractCreateTx) -> -+ #{secret := SecretKey} = get_keypair(), -+ SignedTx = hz:sign_tx(ContractCreateTx, SecretKey), -+ tell(info, "pushing signed tx: ~tp", [SignedTx]), -+ hz:post_tx(SignedTx). -+ -+ -+ -+-spec get_pubkey_akstr() -> string(). -+% @doc -+% get our pubkey as an ak_... string -+ -+get_pubkey_akstr() -> -+ #{public := PK} = get_keypair(), -+ unicode:characters_to_list(fmt_pubkey_api(PK)). -+ -+ -+-spec get_keypair() -> keypair(). -+% @doc -+% if can read keypair from `keypair_file()`, do so -+% otherwise generate one -+% -+% prints warnings if IO ops fail -+ -+get_keypair() -> -+ case read_keypair_from_file(keypair_file()) of -+ {ok, KP} -> -+ KP; -+ % probably file -+ ReadError -> -+ tell(warning, "gh_ct:get_keypair(): read error: ~tp", [ReadError]), -+ KP = gen_keypair(), -+ % try writing to file -+ %tell(info, "gh_ct:get_keypair(): attempting to write keypair to file...", []), -+ %case write_keypair_to_file(keypair_file(), KP) of -+ % ok -> tell(info, "gh_ct:get_keypair(): write successful!", []); -+ % Error -> tell(warning, "gh_ct:get_keypair(): write error: ~tp", [Error]) -+ %end, -+ KP -+ end. -+ -+ -+-spec keypair_file() -> string(). -+% @doc -+% normal file where operating keypair is stored -+ -+keypair_file() -> -+ filename:join([zx:get_home(), "priv", "keypair.eterms"]). -+ -+ -+ -+-spec read_keypair_from_file(FilePath) -> Result -+ when FilePath :: string(), -+ Result :: {ok, keypair()} -+ | {error, Reason :: term()}. -+% @doc -+% try to read keypair from file in `file:consult/1` format. -+ -+read_keypair_from_file(FilePath) -> -+ case file:consult(FilePath) of -+ {ok, [{public, PK}, {secret, SK}]} -> -+ {ok, #{public => PK, secret => SK}}; -+ {ok, [{secret, SK}, {public, PK}]} -> -+ {ok, #{public => PK, secret => SK}}; -+ {ok, Bad} -> -+ tell(warning, "read malformed keypair from file ~tp: ~tp", [FilePath, Bad]), -+ {error, bad_keypair}; -+ Error -> -+ Error -+ end. -+ -+ -+ -+-spec write_keypair_to_file(FilePath, Keypair) -> Result -+ when FilePath :: string(), -+ Keypair :: keypair(), -+ Result :: ok -+ | {error, Reason :: term()}. -+% @doc -+% Write keypair to file as -+% -+% ``` -+% {public, <<...>>}. -+% {secret, <<..>>}. -+% ``` -+ -+write_keypair_to_file(FP, KP) -> -+ file:write_file(FP, fmt_keypair(KP)). -+ -+ -+ -+-spec fmt_pubkey_api(binary()) -> binary(). -+ -+fmt_pubkey_api(Bin) -> -+ gmser_api_encoder:encode(account_pubkey, Bin). -+ -+ -+-spec fmt_keypair(keypair()) -> iolist(). -+% @doc -+% format keypair in `file:consult/1` format -+ -+fmt_keypair(#{public := PK, secret := SK}) -> -+ io_lib:format("{public, ~tp}.~n" -+ "{secret, ~tp}.~n", -+ [PK, SK]). -+ -+ -+ -+-spec gen_keypair() -> keypair(). -+% @doc -+% Generate a keypair -+ -+gen_keypair() -> -+ ecu_eddsa:sign_keypair(). -``` - -[tn-explorer]: http://84.46.242.9:5001/ -[tn-faucet]: http://84.46.242.9:5000/ +- `master` = production. very stable. Should be able to `git clone` the repo + and everything just works. +- `staging` = release candidates +- `develop` = wild west +- `f-*` = brothel in the wild west