Compare commits

...

4 Commits

Author SHA1 Message Date
Peter Harpending
138c8eaaeb the shed biketh 2025-10-22 14:37:27 -07:00
Peter Harpending
b3599633f9 static file caching/serving seems to work 2025-10-22 14:25:59 -07:00
Peter Harpending
f3a107111f add query function 2025-10-22 13:33:08 -07:00
Peter Harpending
8b938fdd42 [wip] static file cache data structure works 2025-10-22 13:17:39 -07:00
10 changed files with 249 additions and 563 deletions

View File

@ -1,100 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Chat with Websockets</title>
<link rel="stylesheet" href="./default.css">
</head>
<body>
<div class="content">
<h1 class="content-title">Chat with websockets</h1>
<div class="content-body">
<input autofocus label="Nick" id="nick"></input>
<textarea hidden disabled id="wfc-output"></textarea>
<input hidden id="wfc-input"></input>
</div>
</div>
<script>
let nelt = document.getElementById('nick');
let ielt = document.getElementById('wfc-input');
let oelt = document.getElementById('wfc-output');
let ws = new WebSocket("/ws/chat");
let nick = '';
// when user hits any key while typing in nick
function on_nick(evt) {
if (evt.key === 'Enter') {
// don't do default thing
evt.preventDefault();
// grab contents
let contents = nelt.value;
let trimmed = contents.trim();
// if contents are nonempty
let nonempty_contents = trimmed.length > 0;
if (nonempty_contents) {
nick = trimmed;
let msg_obj = ['nick', nick];
let msg_str = JSON.stringify(msg_obj);
console.log('message to server:', contents.trim());
// query backend for result
ws.send(msg_str);
// delete element from dom
nelt.remove();
oelt.hidden = false;
ielt.hidden = false;
ielt.autofocus = true;
}
}
}
// when user hits any key while typing in ielt
function on_input_key(evt) {
if (evt.key === 'Enter') {
// don't do default thing
evt.preventDefault();
// grab contents
let contents = ielt.value;
let trimmed = contents.trim();
// if contents are nonempty
let nonempty_contents = trimmed.length > 0;
if (nonempty_contents) {
let msg_obj = ['chat', trimmed];
let msg_str = JSON.stringify(msg_obj);
console.log('message to server:', contents.trim());
// query backend for result
ws.send(msg_str);
// clear input
ielt.value = '';
// add to output
oelt.value += '> ';
oelt.value += trimmed;
oelt.value += '\n';
}
}
}
function main() {
nelt.addEventListener('keydown', on_nick);
ielt.addEventListener('keydown', on_input_key);
ws.onmessage =
function (msg_evt) {
console.log('message from server:', msg_evt);
let msg_str = msg_evt.data;
let msg_obj = JSON.parse(msg_str);
oelt.value += msg_obj.nick;
oelt.value += '> ';
oelt.value += msg_obj.msg;
oelt.value += '\n';
};
}
main();
</script>
</body>
</html>

View File

@ -1,95 +0,0 @@
/* color pallette */
* {
--white: #f7f8fb;
--lgray0: #f5fbfd;
--lgray1: #daddd6;
--lgray2: #d3d8d5;
--mgray1: #687864;
--dgreen1: #21341e;
--lgreen1: #e5eef2;
--black: #003d27;
--sans: Helvetica, Liberation Sans, FreeSans, Roboto, sans-serif;
--mono: Liberation Mono, FreeMono, Roboto Mono, monospace;
--fsdef: 12pt;
}
/* body */
body {
background: var(--white);
color: var(--dgray1);
font-size: var(--fsdef);
font-family: var(--sans);
margin: 0;
padding: 0;
line-height: 1.4;
}
.content {
max-width: 800px;
margin: 0 auto;
/*
background: #ff0;
*/
}
/* add some top padding to content */
.content-title {
text-align: center;
text-decoration: underline;
}
.content-body {
width: 100%;
margin: 0 auto;
/*
background: #f00;
*/
padding-left: 10px;
padding-right: 10px;
box-sizing: border-box;
}
/* element-specific styling */
a {
color: var(--mgray1);
}
#wfc-input {
font-family: var(--mono);
font-size: var(--fsdef);
background: var(--lgreen1);
width: 100%;
box-sizing: border-box;
border-radius: 6px;
border: 1px solid var(--lgray1);
padding: 5px;
}
#wfc-output {
background: var(--lgray0);
font-family: var(--mono);
font-size: var(--fsdef);
width: 100%;
resize: vertical;
box-sizing: border-box;
border-radius: 6px;
border: 1px solid var(--lgray1);
padding: 5px;
}

View File

@ -1,138 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>WF Compiler Demo</title>
<link rel="stylesheet" href="./default.css">
</head>
<body>
<div class="content">
<h1 class="content-title">WFC Demo</h1>
<ul>
<li><a href="/chat.html">Websocket Chatroom</a></li>
<li><a href="/ws-test-echo.html">Websocket Echo Test</a></li>
</ul>
<div class="content-body">
<textarea disabled id="wfc-output"></textarea>
<input autofocus id="wfc-input"></input>
<h2>Settings</h2>
<input type="checkbox" checked id="auto-resize-output">Auto-resize output</input> <br>
<input type="checkbox" checked id="auto-scroll" >Auto-scroll output to bottom</input>
</div>
</div>
<script>
let ielt = document.getElementById('wfc-input');
let oelt = document.getElementById('wfc-output');
let MAX_OELT_HEIGHT = 300;
function auto_resize_output() {
// if the user has manually resized their output, we do nothing
if (document.getElementById('auto-resize-output').checked) {
// resize it automagically up to 500px
if (oelt.scrollHeight < MAX_OELT_HEIGHT) {
oelt.style.height = String(oelt.scrollHeight) + 'px';
}
else {
oelt.style.height = String(MAX_OELT_HEIGHT) + 'px';
}
}
}
function auto_scroll_to_bottom() {
if (document.getElementById('auto-scroll').checked) {
// scroll to bottom
oelt.scrollTop = oelt.scrollHeight;
}
}
async function on_server_return(response) {
console.log('on_server_return:', response);
if (response.ok) {
let jsbs = await response.json();
console.log('jsbs', jsbs);
// jsbs: {ok: true, result: string} | {ok: false, error: string}
if (jsbs.ok) {
// this means got a result back from server
// put it in
oelt.value += jsbs.result;
oelt.value += '\n';
}
else {
// this is an error at the WFC level
oelt.value += jsbs.error;
oelt.value += '\n';
}
}
// this means we sent an invalid request
else {
oelt.value += 'HTTP ERROR, SEE BROWSER CONSOLE\n'
}
}
function on_some_bullshit(x) {
console.log('on_some_bullshit:', x);
oelt.value += 'NETWORK ERROR, SEE BROWSER CONSOLE\n'
}
function fetch_wfcin(user_line) {
let req_body_obj = {wfcin: user_line};
// let req_body_str = JSON.stringify(req_body_obj, undefined, 4);
let req_body_str = JSON.stringify(req_body_obj);
let req_options = {method: 'POST',
headers: {'content-type': 'application/json'},
body: req_body_str};
let response_promise = fetch('/wfcin', req_options);
response_promise.then(on_server_return, on_some_bullshit);
// this is a promise for a response
//console.log(response_promise);
}
// when user hits any key
function on_input_key(evt) {
if (evt.key === 'Enter') {
// don't do default thing
evt.preventDefault();
// grab contents
let contents = ielt.value;
// if contents are nonempty
let nonempty_contents = contents.trim().length > 0;
if (nonempty_contents) {
// put in output
// // if it's nonempty add a newline
// if (oelt.value.length > 0) {
// oelt.value += '\n';
// }
oelt.value += '> ' + contents + '\n';
oelt.hidden = false;
// query backend for result
fetch_wfcin(contents.trim());
// clear input
ielt.value = '';
// auto-resize
auto_resize_output();
auto_scroll_to_bottom();
}
}
}
function main() {
ielt.addEventListener('keydown', on_input_key);
}
main();
</script>
</body>
</html>

View File

@ -1,56 +0,0 @@
/* header */
div#header {
background: var(--lgray2);
height: 50px;
width: 100%;
}
img#header-logo {
height: 40px;
margin-top: 5px;
margin-bottom: 5px;
margin-left: 5px;
}
.main {
padding-top: 20px;
}
.content-diagram {
max-width: 100%;
margin: 0 auto;
padding: 0 auto;
}
/* Pandoc makes some wacky choices with how it lays out code blocks
*
* this means all <code> blocks that do not have a <pre> block as an ancestor
*
* ie the `inline code`
*/
code:not(pre *) {
border-radius: 6px;
border: 1px solid var(--lgray1);
padding-left: 3px;
padding-right: 3px;
padding-top: 1px;
padding-bottom: 1px;
background: var(--lgreen1);
}
/* this is specifically for ```fenced blocks``` */
pre {
border-radius: 6px;
border: 1px solid var(--lgray1);
padding: 5px;
background: var(--lgreen1);
overflow: scroll;
}
/* All `unfenced` or ```fenced``` blocks */
code {
font-family: Liberation Mono, Roboto Mono, monospace;
color: var(--dgreen1);
}

View File

@ -1,65 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Websockets echo test</title>
<link rel="stylesheet" href="./default.css">
</head>
<body>
<div class="content">
<h1 class="content-title">Websockets echo test</h1>
<div class="content-body">
<textarea id="wfc-output"
disabled></textarea>
<input autofocus id="wfc-input"></input>
</div>
</div>
<script>
let ielt = document.getElementById('wfc-input');
let oelt = document.getElementById('wfc-output');
let ws = new WebSocket("/ws/echo");
// when user hits any key while typing in ielt
function on_input_key(evt) {
if (evt.key === 'Enter') {
// don't do default thing
evt.preventDefault();
// grab contents
let contents = ielt.value;
let trimmed = contents.trim();
// if contents are nonempty
let nonempty_contents = trimmed.length > 0;
if (nonempty_contents) {
console.log('message to server:', contents.trim());
// query backend for result
ws.send(contents.trim());
// clear input
ielt.value = '';
// add to output
oelt.value += '> ';
oelt.value += trimmed;
oelt.value += '\n';
}
}
}
function main() {
ielt.addEventListener('keydown', on_input_key);
ws.onmessage =
function (msg_evt) {
console.log('message from server:', msg_evt);
let msg_str = msg_evt.data;
oelt.value += '< ';
oelt.value += msg_str;
oelt.value += '\n';
};
}
main();
</script>
</body>
</html>

View File

@ -224,12 +224,9 @@ handle_request(Sock, R = #request{method = M, path = P}) when M =/= undefined, P
route(Sock, get, Route, Request) ->
case Route of
<<"/">> -> home(Sock);
<<"/default.css">> -> default_css(Sock);
<<"/chat.html">> -> chat_html(Sock);
<<"/ws-test-echo.html">> -> ws_test_echo_html(Sock);
<<"/ws/echo">> -> ws_echo(Sock, Request);
_ -> http_err(Sock, 404)
<<"/">> -> route_static(Sock, <<"/index.html">>);
_ -> route_static(Sock, Route)
end;
route(Sock, post, Route, Request) ->
case Route of
@ -240,6 +237,28 @@ route(Sock, _, _, _) ->
http_err(Sock, 404).
route_static(Sock, Route) ->
respond_static(Sock, fd_sfc:query(Route)).
respond_static(Sock, {found, Entry}) ->
% -record(e, {fs_path :: file:filename(),
% last_modified :: file:date_time(),
% mime_type :: string(),
% encoding :: encoding(),
% contents :: binary()}).
Headers0 =
case fd_sfc_entry:encoding(Entry) of
gzip -> [{"content-encoding", "gzip"}];
none -> []
end,
Headers1 = [{"content-type", fd_sfc_entry:mime_type(Entry)} | Headers0],
Response = #response{headers = Headers1,
body = fd_sfc_entry:contents(Entry)},
respond(Sock, Response);
respond_static(Sock, not_found) ->
http_err(Sock, 404).
ws_echo(Sock, Request) ->
try
ws_echo2(Sock, Request)
@ -279,59 +298,6 @@ ws_echo_loop(Sock, Frames, Received) ->
error(Error)
end.
home(Sock) ->
%% fixme: cache
Path_IH = filename:join([zx:get_home(), "priv", "index.html"]),
case file:read_file(Path_IH) of
{ok, Body} ->
Resp = #response{headers = [{"content-type", "text/html"}],
body = Body},
respond(Sock, Resp);
Error ->
tell("error: ~p~n", [self(), Error]),
http_err(Sock, 500)
end.
default_css(Sock) ->
%% fixme: cache
Path_IH = filename:join([zx:get_home(), "priv", "default.css"]),
case file:read_file(Path_IH) of
{ok, Body} ->
Resp = #response{headers = [{"content-type", "text/css"}],
body = Body},
respond(Sock, Resp);
Error ->
io:format("~p error: ~p~n", [self(), Error]),
http_err(Sock, 500)
end.
ws_test_echo_html(Sock) ->
%% fixme: cache
Path_IH = filename:join([zx:get_home(), "priv", "ws-test-echo.html"]),
case file:read_file(Path_IH) of
{ok, Body} ->
Resp = #response{headers = [{"content-type", "text/html"}],
body = Body},
respond(Sock, Resp);
Error ->
io:format("~p error: ~p~n", [self(), Error]),
http_err(Sock, 500)
end.
chat_html(Sock) ->
%% fixme: cache
Path_IH = filename:join([zx:get_home(), "priv", "chat.html"]),
case file:read_file(Path_IH) of
{ok, Body} ->
Resp = #response{headers = [{"content-type", "text/html"}],
body = Body},
respond(Sock, Resp);
Error ->
io:format("~p error: ~p~n", [self(), Error]),
http_err(Sock, 500)
end.
wfcin(Sock, #request{enctype = json,
cookies = Cookies,
body = #{"wfcin" := Input}}) ->

View File

@ -5,6 +5,8 @@
-export([
%% caller context
base_path/0,
renew/0, query/1,
start_link/0,
%% process context
@ -14,19 +16,25 @@
-include("$zx_include/zx_logger.hrl").
-record(e, {fs_path :: file:filename(),
last_modified :: file:date_time(),
mime_type :: string(),
contents :: binary()}).
-type entry() :: #e{}.
-record(s, {entries = #{ :: [entry()]}).
-type state() :: #s{}.
-record(s, {base_path = base_path() :: file:filename(),
cache = fd_sfc_cache:new(base_path()) :: fd_sfc_cache:cache(),
auto_renew = 5_000 :: pos_integer()}).
%-type state() :: #s{}.
%%-----------------------------------------------------------------------------
%% caller context
%%-----------------------------------------------------------------------------
base_path() ->
filename:join([zx:get_home(), "priv", "static"]).
renew() ->
gen_server:cast(?MODULE, renew).
query(Path) ->
gen_server:call(?MODULE, {query, Path}).
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, none, []).
@ -41,19 +49,31 @@ start_link() ->
init(none) ->
log(info, "starting fd_cache"),
InitState = #s{},
erlang:send_after(InitState#s.auto_renew, self(), auto_renew),
{ok, InitState}.
handle_call({query, Path}, _, State = #s{cache = Cache}) ->
Reply = fd_sfc_cache:query(Path, Cache),
{reply, Reply, State};
handle_call(Unexpected, From, State) ->
tell("~tp: unexpected call from ~tp: ~tp", [?MODULE, Unexpected, From]),
{noreply, State}.
handle_cast(renew, State) ->
NewState = i_renew(State),
{noreply, NewState};
handle_cast(Unexpected, State) ->
tell("~tp: unexpected cast: ~tp", [?MODULE, Unexpected]),
{noreply, State}.
handle_info(auto_renew, State = #s{auto_renew = MS}) ->
log(info, "~tp: auto_renew", [?MODULE]),
erlang:send_after(MS, self(), auto_renew),
NewState = i_renew(State),
{noreply, NewState};
handle_info(Unexpected, State) ->
tell("~tp: unexpected info: ~tp", [?MODULE, Unexpected]),
{noreply, State}.
@ -66,43 +86,11 @@ terminate(_, _) ->
ok.
%%---------------------
%% doers
%%---------------------
%%-----------------------------------------------------------------------------
%% internals
%%-----------------------------------------------------------------------------
-spec refresh_entry(Entry) -> Result
when Entry :: entry(),
Result :: {found, NewEntry :: entry()}
| not_found.
refresh_entry(Entry) ->
refresh_entry(Entry, false).
-spec refresh_entry(Entry, Force) -> Result
when Entry :: entry(),
Force :: boolean(),
Result :: {found, NewEntry :: entry()}
| not_found.
% @private
% Force = even if file has not been modified, still refresh contents
refresh_entry(E = #e{fs_path = Path}, Force) ->
case file:find_file(Path) of
{ok, _} -> re2(E, Force);
{error, not_found} -> not_found.
end.
re2(E = #e{fs_path = Path, last_modified = LastModified}, _Force = false) ->
case file:last_modified(Path) > LastModified of
false -> {found, E};
true -> re2(E, true)
end;
re2(E = #e{http_path = HttpPath}, _Force = true) ->
new_entry(HttpPath).
-spec new_entry(HttpPath) -> Result
when HttpPath :: binary(),
FilePath ::
i_renew(State = #s{base_path = BasePath}) ->
NewCache = fd_sfc_cache:new(BasePath),
NewState = State#s{cache = NewCache},
NewState.

83
src/fd_sfc_cache.erl Normal file
View File

@ -0,0 +1,83 @@
% @doc
% cache data management
-module(fd_sfc_cache).
-export_type([
cache/0
]).
-export([
query/2,
new/0, new/1
]).
-include("$zx_include/zx_logger.hrl").
-type cache() :: #{HttpPath :: binary() := Entry :: fd_sfc_entry:entry()}.
-spec query(HttpPath, Cache) -> Result
when HttpPath :: binary(),
Cache :: cache(),
Result :: {found, Entry}
| not_found,
Entry :: fd_sfc_entry:entry().
query(HttpPath, Cache) ->
case maps:find(HttpPath, Cache) of
{ok, Entry} -> {found, Entry};
error -> not_found
end.
-spec new() -> cache().
new() -> #{}.
-spec new(BasePath) -> cache()
when BasePath :: file:filename().
% @doc
% if you give a file path it just takes the parent dir
%
% recursively crawls through file tree and picks
%
% IO errors will be logged but will result in cache misses
new(BasePath) ->
case filelib:is_file(BasePath) of
true -> new2(BasePath);
false ->
tell("~p:new(~p): no such file or directory, returning empty cache", [?MODULE, BasePath]),
#{}
end.
new2(BasePath) ->
BaseDir =
case filelib:is_dir(BasePath) of
true -> filename:absname(BasePath);
false -> filename:absname(filename:dirname(BasePath))
end,
%% hacky, fuck you
RemovePrefix =
fun (Prefix, Size, From) ->
<<Prefix:Size/bytes, Rest/bytes>> = From,
Rest
end,
BBaseDir = unicode:characters_to_binary(BaseDir),
BBS = byte_size(BBaseDir),
HandlePath =
fun(AbsPath, AccCache) ->
BAbsPath = unicode:characters_to_binary(AbsPath),
HttpPath = RemovePrefix(BBaseDir, BBS, BAbsPath),
NewCache =
case fd_sfc_entry:new(AbsPath) of
{found, Entry} -> maps:put(HttpPath, Entry, AccCache);
not_found -> AccCache
end,
NewCache
end,
filelib:fold_files(_dir = BaseDir,
_match = ".+",
_recursive = true,
_fun = HandlePath,
_init_acc = #{}).

View File

@ -1,2 +1,99 @@
% @doc non-servery functions for static file caching
-module(fd_sfc_entry)
%
% this spams the filesystem, so it's not "pure" code
-module(fd_sfc_entry).
-export_type([
encoding/0,
entry/0
]).
-export([
%% constructor
new/1,
%% accessors
fs_path/1, last_modified/1, mime_type/1, encoding/1, contents/1
]).
-include("$zx_include/zx_logger.hrl").
%% types
% id = not compressed
-type encoding() :: none | gzip.
-record(e, {fs_path :: file:filename(),
last_modified :: file:date_time(),
mime_type :: string(),
encoding :: encoding(),
contents :: binary()}).
-opaque entry() :: #e{}.
%% accessors
fs_path(#e{fs_path = X}) -> X.
last_modified(#e{last_modified = X}) -> X.
mime_type(#e{mime_type = X}) -> X.
encoding(#e{encoding = X}) -> X.
contents(#e{contents = X}) -> X.
%% API
-spec new(Path) -> Result
when Path :: file:filename(),
Result :: {found, entry()}
| not_found.
% @doc
% absolute file path stored in resulting record
%
% returns not_found if ANY I/O error occurs during the process. will be logged
new(Path) ->
log(info, "~tp:new(~tp)", [?MODULE, Path]),
case file:read_file(Path) of
{ok, Binary} ->
{found, new2(Path, Binary)};
Error ->
tell("~tp:new(~tp): file read error: ~tp", [?MODULE, Path, Error]),
not_found
end.
%% can assume file exists
new2(FsPath, FileBytes) ->
LastModified = filelib:last_modified(FsPath),
{Encoding, MimeType} = mimetype_compress(FsPath),
Contents =
case Encoding of
none -> FileBytes;
gzip -> zlib:gzip(FileBytes)
end,
#e{fs_path = FsPath,
last_modified = LastModified,
mime_type = MimeType,
encoding = Encoding,
contents = Contents}.
mimetype_compress(FsPath) ->
case string:casefold(filename:extension(FsPath)) of
%% only including the ones i anticipate encountering
%% plaintext formats
".css" -> {gzip, "text/css"};
".htm" -> {gzip, "text/html"};
".html" -> {gzip, "text/html"};
".js" -> {gzip, "text/javascript"};
".json" -> {gzip, "application/json"};
".map" -> {gzip, "application/json"};
".md" -> {gzip, "text/markdown"};
".ts" -> {gzip, "text/x-typescript"};
".txt" -> {gzip, "text/plain"};
%% binary formats
".gif" -> {none, "image/gif"};
".jpg" -> {none, "image/jpeg"};
".jpeg" -> {none, "image/jpeg"};
".mp4" -> {none, "video/mp4"};
".png" -> {none, "image/png"};
".webm" -> {none, "video/webm"};
".webp" -> {none, "image/webp"};
_ -> {none, "application/octet-stream"}
end.

View File

@ -48,11 +48,17 @@ init([]) ->
5000,
worker,
[fd_chat]},
FileCache = {fd_sfc,
{fd_sfc, start_link, []},
permanent,
5000,
worker,
[fd_sfc]},
Cache = {fd_cache,
{fd_cache, start_link, []},
permanent,
5000,
worker,
[fd_cache]},
Children = [Clients, Chat, Cache],
Children = [Clients, Chat, FileCache, Cache],
{ok, {RestartStrategy, Children}}.