Reset worker errors, retry failed workers

Reviewed-on: #8
Reviewed-by: Jarvis Carroll <jarviscarrol@qpq.swiss>

README.md

@@ -1,11 +1,213 @@
 # Gajumaru Hive Client
 
-When running tests locally:
+This application is primarily meant to be used by the [GajuMine](https://git.qpq.swiss/QPQ-AG/gajumine)
+application, but can also be run as a headless worker. It can be built either using `rebar3` or
+using [`zx`](https://zxq9.com/projects/zomp/) (recomended).
 
-`rebar3 shell`
+## Operation
+The `gmhive_client` needs at least a worker public key to operate.
+It first queries the [GajuMining website](https://gajumining.com) to find the address of
+the Gajumaru Hive server, which it then connects to. GajuMining won't provide the address
+unless the public key is a registered worker.
 
-If connecting to the `testnet` hive:
-`GMMP_CLIENT_CONFIG=gmhc_config-testnet.eterm rebar3 shell`
+## Configuration
 
-If connecting to a `testnet` hive running locally:
-`GMHC_CONFIG=gmhc_config-testnet-local.eterm rebar3 shell`
+The Hive Client uses a configuration schema ([see below](#json-schema)).
+At a minimum, a value for `"pubkey"` must be provided. All other configuration
+items have usable defaults.
+
+Examples:
+
+**.json**:
+```json
+{ "pubkey" : "ak_2NQoybA6uyvhu3cdzWFtZnxCZpZvKE5TxjYB3hB7kMWEquwLVY",
+  "extra_pubkeys" : [ "ak_2Dfwyb7ZFhcAELoxgWyMCkVnpZUhwUm6aftKcv6dakagePWyx9" ]
+}
+```
+
+**.eterm**:
+```erlang
+#{ <<"pubkey">> => <<"ak_2NQoybA6uyvhu3cdzWFtZnxCZpZvKE5TxjYB3hB7kMWEquwLVY">>,
+   <<"extra_pubkeys">> : [ <<"ak_2Dfwyb7ZFhcAELoxgWyMCkVnpZUhwUm6aftKcv6dakagePWyx9">> ]
+ }.
+```
+
+Configuration can be provided in a file, where the filename extension defines
+the format: `".json"` for JSON format, `".eterm"` for Erlang term format.
+In the case of Erlang term format, the data should be on "internal JSON form",
+i.e. nested maps, where all strings are of type `binary()`.
+
+By default, the Hive client looks for a configuration file named
+`"gmhive_client_config.[json|eterm]"` in the current working directory.
+A specific configuration file can be identified using the OS environment
+variable `GMHIVE_CLIENT_CONFIG`.
+
+On invocation, individual configuration items can be set in one of two ways:
+
+### OS environment variable overrides
+
+By creating an OS environment variable with the name `GMHC__<variable_name>`,
+an individual variable can be redefined. The name is composed from the 
+configuration key name, converted to uppercase, and with `__` as a delimiter
+for a nested key. The value is coerced into the expected type using the schema.
+Complex values are given in JSON format - be sure to quote it.
+
+Example:
+```
+GMHC__PUBKEY="ak_2Dfwyb7ZFhcAELoxgWyMCkVnpZUhwUm6aftKcv6dakagePWyx9" zx run uwiger-gmhive_client
+```
+
+```
+GMHC__WORKERS='[{"executable": "mean29-avx2"}]' zx run ...
+```
+
+### Command-line arguments
+
+Arguments on the form `-gmhc Key Value` can be added to the command line.
+The key name is derived from the schema, all lowercase and with `__` as delimiter.
+
+Example:
+```
+zx run uwiger-gmhive_client -gmhc pubkey ak_2Dfwyb...
+```
+
+### Considerations
+
+The values for `pool_admin` and `pool` should be left alone. The `"network"`
+configuration item is special. Its default value is `"mainnet"`, which
+indicates that the client should connect to
+[gajumining.com](https://gajumining.com) to find the hive server, but by
+changing its value to `"testnet"`, (or to other networks in the future,) not
+only does this inform the client of the network for which the client should
+mine, but it also changes what URL the client will use to find the hive
+server. (For `"testnet"` this will be `"test.gajumining.com"`.)
+
+## JSON Schema
+
+```json
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "additionalProperties": false,
+    "properties": {
+        "extra_pubkeys": {
+            "default": [],
+            "description": "Additional worker pubkeys, sharing rewards",
+            "items": {
+                "pattern": "^ak_[1-9A-HJ-NP-Za-km-z]*$",
+                "type": "string"
+            },
+            "type": "array"
+        },
+        "network": {
+            "default": "mainnet",
+            "type": "string"
+        },
+        "pool": {
+            "additionalProperties": false,
+            "properties": {
+                "host": {
+                    "default": "127.0.0.1",
+                    "description": "Hostname of hive server",
+                    "example": "0.0.0.0",
+                    "type": "string"
+                },
+                "id": {
+                    "description": "Pool contract id",
+                    "pattern": "^ct_[1-9A-HJ-NP-Za-km-z]*$",
+                    "type": "string"
+                },
+                "port": {
+                    "default": 17888,
+                    "description": "Hive server listen port",
+                    "minimum": 1,
+                    "type": "integer"
+                }
+            },
+            "type": "object"
+        },
+        "pool_admin": {
+            "additionalProperties": false,
+            "properties": {
+                "default_per_network": {
+                    "additionalProperties": false,
+                    "properties": {
+                        "mainnet": {
+                            "default": "https://gajumining.com/api/workers/{CLIENT_ID}",
+                            "type": "string"
+                        },
+                        "testnet": {
+                            "default": "https://test.gajumining.com/api/workers/{CLIENT_ID}",
+                            "type": "string"
+                        }
+                    },
+                    "type": "object"
+                },
+                "url": {
+                    "default": "https://test.gajumining.com/api/workers/{CLIENT_ID}",
+                    "description": "URL of Eureka worker api",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
+        "pubkey": {
+            "description": "Primary client pubkey",
+            "pattern": "^ak_[1-9A-HJ-NP-Za-km-z]*$",
+            "type": "string"
+        },
+        "type": {
+            "default": "worker",
+            "description": "monitor mode can be used to see if a pool is alive",
+            "enum": [
+                "worker",
+                "monitor"
+            ],
+            "type": "string"
+        },
+        "workers": {
+            "default": [
+                { "executable": "mean29-generic" }
+            ],
+            "description": "Definitions of workers' configurations. If no worker are configured one worker is used as default, i.e. 'mean29-generic' executable without any extra args.",
+            "items": {
+                "additionalProperties": false,
+                "properties": {
+                    "executable": {
+                        "default": "mean29-generic",
+                        "description": "Executable binary of the worker. Can be a fully qualified path, but the software may apply default logic to locate a plain basename.",
+                        "type": "string"
+                    },
+                    "extra_args": {
+                        "default": "",
+                        "description": "Extra arguments to pass to the worker executable binary. The safest choice is specifying no arguments i.e. empty string.",
+                        "type": "string"
+                    },
+                    "hex_encoded_header": {
+                        "default": false,
+                        "description": "Hexadecimal encode the header argument that is send to the worker executable. CUDA executables expect hex encoded header.",
+                        "type": "boolean"
+                    },
+                    "instances": {
+                        "description": "Instances used by the worker in case of Multi-GPU mining. Numbers on the configuration list represent GPU devices that are to be addressed by the worker.",
+                        "example": [0,1,2,3],
+                        "items": { "type": "integer" },
+                        "minItems": 1,
+                        "type": "array"
+                    },
+                    "repeats": {
+                        "default": 1,
+                        "description": "Number of tries to do in each worker context - WARNING: it should be set so the worker process runs for 3-5s or else the node risk missing out on new micro blocks.",
+                        "type": "integer"
+                    }
+                },
+                "required": [
+                    "executable"
+                ],
+                "type": "object"
+            },
+            "type": "array"
+        }
+    },
+    "type": "object"
+}
+```

ebin/gmhive_client.app

@@ -1,102 +1,16 @@
 {application,gmhive_client,
-             [{description,"Gajumaru Hive client"},
-              {vsn,"0.2.1"},
+             [{description,"Gajumaru Hive Client"},
+              {vsn,"0.4.8"},
               {registered,[]},
-              {applications,[kernel,stdlib,sasl,gproc,enoise,gmconfig,
-                             gmhive_protocol,gmhive_worker]},
-              {mod,{gmhive_client,[]}},
+              {applications,[kernel,stdlib,sasl,gproc,inets,ssl,enoise,
+                             gmconfig,gmhive_protocol,gmhive_worker]},
+              {mod,{gmhc_app,[]}},
               {start_phases,[{connect_to_primary,[]}]},
               {env,[]},
-              {modules,[base58,eblake2,eblake2_tests,timing,enacl_eqc,
-                        enacl_ext_eqc,enacl,enacl_ext,enacl_nif,enacl_SUITE,
-                        enoise,enoise_cipher_state,enoise_connection,
-                        enoise_crypto,enoise_hs_state,enoise_keypair,
-                        enoise_protocol,enoise_sym_state,
-                        enoise_bad_data_tests,enoise_chiper_state_tests,
-                        enoise_crypto_tests,enoise_hs_state_tests,
-                        enoise_protocol_tests,enoise_sym_state_tests,
-                        enoise_tests,enoise_utils,test_utils,gmconfig,
-                        gmconfig_schema_helpers,gmconfig_schema_utils,
-                        gmcuckoo,gmhc_app,gmhc_config,gmhc_config_schema,
+              {modules,[gmhc_app,gmhc_config,gmhc_config_schema,
                         gmhc_connector,gmhc_connectors_sup,gmhc_counters,
                         gmhc_eureka,gmhc_events,gmhc_handler,gmhc_server,
-                        gmhc_sup,gmhc_workers,gmhive_client,gmhp_msgs,
-                        gmhw_blake2b_256,gmhw_pow,gmhw_pow_cuckoo,
-                        gmhw_siphash24,gmcuckoo_SUITE,gmhw_pow_cuckoo_tests,
-                        gmhw_pow_tests,gmser_api_encoder,gmser_chain_objects,
-                        gmser_contract_code,gmser_delegation,gmser_dyn,
-                        gmser_id,gmser_rlp,gmserialization,
-                        gmser_api_encoder_tests,gmser_chain_objects_tests,
-                        gmser_contract_code_tests,gmser_delegation_tests,
-                        gmser_rlp_tests,gproc,gproc_app,gproc_bcast,
-                        gproc_dist,gproc_info,gproc_init,gproc_lib,
-                        gproc_monitor,gproc_pool,gproc_ps,gproc_pt,gproc_sup,
-                        setup,setup_app,setup_file,setup_file_io_server,
-                        setup_gen,setup_lib,setup_srv,setup_sup,setup_zomp,
-                        testapp_app,testapp_sup,testapp_app,testapp_sup,
-                        hex_api,hex_api_auth,hex_api_key,hex_api_organization,
-                        hex_api_organization_member,hex_api_package,
-                        hex_api_package_owner,hex_api_release,hex_api_user,
-                        hex_core,hex_erl_tar,hex_filename,hex_http,
-                        hex_http_httpc,hex_licenses,hex_pb_names,
-                        hex_pb_package,hex_pb_signed,hex_pb_versions,
-                        hex_registry,hex_repo,hex_tarball,safe_erl_term,pc,
-                        pc_compilation,pc_port_env,pc_port_specs,pc_prv_clean,
-                        pc_prv_compile,pc_util,rebar3_hex,rebar3_hex_app,
-                        rebar3_hex_build,rebar3_hex_client,rebar3_hex_config,
-                        rebar3_hex_cut,rebar3_hex_error,rebar3_hex_file,
-                        rebar3_hex_httpc_adapter,rebar3_hex_io,rebar3_hex_key,
-                        rebar3_hex_organization,rebar3_hex_owner,
-                        rebar3_hex_publish,rebar3_hex_results,
-                        rebar3_hex_retire,rebar3_hex_search,rebar3_hex_user,
-                        rebar3_hex_version,
-                        rebar_aecuckooprebuilt_app_with_priv_from_git_resource,
-                        rebar_aecuckooprebuilt_dep,verl,verl_parser,base58,
-                        eblake2,eblake2_tests,timing,enacl_eqc,enacl_ext_eqc,
-                        enacl,enacl_ext,enacl_nif,enacl_SUITE,enoise,
-                        enoise_cipher_state,enoise_connection,enoise_crypto,
-                        enoise_hs_state,enoise_keypair,enoise_protocol,
-                        enoise_sym_state,enoise_bad_data_tests,
-                        enoise_chiper_state_tests,enoise_crypto_tests,
-                        enoise_hs_state_tests,enoise_protocol_tests,
-                        enoise_sym_state_tests,enoise_tests,enoise_utils,
-                        test_utils,gmconfig,gmconfig_schema_helpers,
-                        gmconfig_schema_utils,gmcuckoo,gmhp_msgs,
-                        gmhw_blake2b_256,gmhw_pow,gmhw_pow_cuckoo,
-                        gmhw_siphash24,gmcuckoo_SUITE,gmhw_pow_cuckoo_tests,
-                        gmhw_pow_tests,gmser_api_encoder,gmser_chain_objects,
-                        gmser_contract_code,gmser_delegation,gmser_dyn,
-                        gmser_id,gmser_rlp,gmserialization,
-                        gmser_api_encoder_tests,gmser_chain_objects_tests,
-                        gmser_contract_code_tests,gmser_delegation_tests,
-                        gmser_rlp_tests,gproc,gproc_app,gproc_bcast,
-                        gproc_dist,gproc_info,gproc_init,gproc_lib,
-                        gproc_monitor,gproc_pool,gproc_ps,gproc_pt,gproc_sup,
-                        setup,setup_app,setup_file,setup_file_io_server,
-                        setup_gen,setup_lib,setup_srv,setup_sup,setup_zomp,
-                        testapp_app,testapp_sup,testapp_app,testapp_sup,
-                        hex_api,hex_api_auth,hex_api_key,hex_api_organization,
-                        hex_api_organization_member,hex_api_package,
-                        hex_api_package_owner,hex_api_release,hex_api_user,
-                        hex_core,hex_erl_tar,hex_filename,hex_http,
-                        hex_http_httpc,hex_licenses,hex_pb_names,
-                        hex_pb_package,hex_pb_signed,hex_pb_versions,
-                        hex_registry,hex_repo,hex_tarball,safe_erl_term,pc,
-                        pc_compilation,pc_port_env,pc_port_specs,pc_prv_clean,
-                        pc_prv_compile,pc_util,rebar3_hex,rebar3_hex_app,
-                        rebar3_hex_build,rebar3_hex_client,rebar3_hex_config,
-                        rebar3_hex_cut,rebar3_hex_error,rebar3_hex_file,
-                        rebar3_hex_httpc_adapter,rebar3_hex_io,rebar3_hex_key,
-                        rebar3_hex_organization,rebar3_hex_owner,
-                        rebar3_hex_publish,rebar3_hex_results,
-                        rebar3_hex_retire,rebar3_hex_search,rebar3_hex_user,
-                        rebar3_hex_version,
-                        rebar_aecuckooprebuilt_app_with_priv_from_git_resource,
-                        rebar_aecuckooprebuilt_dep,verl,verl_parser,gmhc_app,
-                        gmhc_config,gmhc_config_schema,gmhc_connector,
-                        gmhc_connectors_sup,gmhc_counters,gmhc_eureka,
-                        gmhc_events,gmhc_handler,gmhc_server,gmhc_sup,
-                        gmhc_workers,gmhive_client]},
+                        gmhc_sup,gmhc_workers,gmhive_client]},
               {maintainers,["QPQ IaaS AG"]},
               {licensens,["ISC"]},
               {links,[{"gitea","https://git.qpq.swiss/gmhive_client"}]}]}.

gmhc_config-testnet-local.eterm

@@ -3,9 +3,9 @@
  , pool => #{id => <<"ct_26xqeE3YKmZV8jrks57JSgZRCHSuG4RGzpnvdz6AAiSSTVbJRM">>,
              host => <<"127.0.0.1">>}
  , workers =>
-       [#{executable => <<"mean15-generic">>, <<"executable_group">> => <<"gmcuckoo">>}
-       ,#{executable => <<"mean15-generic">>, <<"executable_group">> => <<"gmcuckoo">>}
-       ,#{executable => <<"mean15-generic">>, <<"executable_group">> => <<"gmcuckoo">>}
-       ,#{executable => <<"mean15-generic">>, <<"executable_group">> => <<"gmcuckoo">>}
+       [#{executable => <<"mean15-generic">>}
+       ,#{executable => <<"mean15-generic">>}
+       ,#{executable => <<"mean15-generic">>}
+       ,#{executable => <<"mean15-generic">>}
        ]
  }.

rebar.config

@@ -10,11 +10,11 @@
         {gmhive_protocol,
          {git, "https://git.qpq.swiss/QPQ-AG/gmhive_protocol.git",
           {ref, "818ce33"}}},
-        {gmhive_worker, {git, "https://git.qpq.swiss/QPQ-AG/gmhive_worker", {ref, "708f1a6"}}},
+        {gmhive_worker, {git, "https://git.qpq.swiss/QPQ-AG/gmhive_worker", {ref, "fac460714f"}}},
         {gmconfig, {git, "https://git.qpq.swiss/QPQ-AG/gmconfig.git",
-                    {ref, "32c1ed5c4b"}}},
+                    {ref, "38620ff9e2"}}},
         {gproc, "1.0.0"},
-        {setup, {git, "https://github.com/uwiger/setup", {ref, "3ad83ed"}}}
+        {setup, {git, "https://github.com/uwiger/setup", {ref, "d9b0b51"}}}
        ]}.
 
 {relx, [

rebar.lock

@@ -1,49 +1,44 @@
 {"1.2.0",
-[{<<"aecuckooprebuilt">>,
-  {aecuckooprebuilt_app_with_priv_from_git,
-      {git,"https://github.com/aeternity/cuckoo-prebuilt.git",
-          {ref,"90afb699dc9cc41d033a7c8551179d32b3bd569d"}}},
-  1},
- {<<"base58">>,
+[{<<"base58">>,
   {git,"https://git.qpq.swiss/QPQ-AG/erl-base58.git",
-      {ref,"e6aa62eeae3d4388311401f06e4b939bf4e94b9c"}},
+       {ref,"e6aa62eeae3d4388311401f06e4b939bf4e94b9c"}},
   2},
  {<<"eblake2">>,
   {git,"https://git.qpq.swiss/QPQ-AG/eblake2.git",
-      {ref,"b29d585b8760746142014884007eb8441a3b6a14"}},
+       {ref,"b29d585b8760746142014884007eb8441a3b6a14"}},
   1},
  {<<"enacl">>,
   {git,"https://git.qpq.swiss/QPQ-AG/enacl.git",
-      {ref,"4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}},
+       {ref,"4eb7ec70084ba7c87b1af8797c4c4e90c84f95a2"}},
   2},
  {<<"enoise">>,
   {git,"https://git.qpq.swiss/QPQ-AG/enoise.git",
-      {ref,"029292817ea1c685b6377bfae667976c0f4a36bc"}},
+       {ref,"029292817ea1c685b6377bfae667976c0f4a36bc"}},
   0},
  {<<"gmconfig">>,
   {git,"https://git.qpq.swiss/QPQ-AG/gmconfig.git",
-      {ref,"32c1ed5c4bc0913adc175fb18b1c9251acc7b606"}},
+       {ref,"38620ff9e2289dff121377c382e4d5b08c719d87"}},
   0},
  {<<"gmcuckoo">>,
   {git,"https://git.qpq.swiss/QPQ-AG/gmcuckoo.git",
-      {ref,"9f3aef96d8660e83ef8b5a7ee64908060a8c6572"}},
+       {ref,"106e1cd2e4ff81286f6bc7d7c85f83bc20e14b82"}},
   1},
  {<<"gmhive_protocol">>,
   {git,"https://git.qpq.swiss/QPQ-AG/gmhive_protocol.git",
-      {ref,"818ce33cc1dec74c020515be48fb548a5207befd"}},
+       {ref,"818ce33cc1dec74c020515be48fb548a5207befd"}},
   0},
  {<<"gmhive_worker">>,
   {git,"https://git.qpq.swiss/QPQ-AG/gmhive_worker",
-      {ref,"708f1a62f9584f874eb9436a974401b5d4272d9f"}},
+       {ref,"fac460714fc228eb0b723a3f292a44aec77f094a"}},
   0},
  {<<"gmserialization">>,
   {git,"https://git.qpq.swiss/QPQ-AG/gmserialization.git",
-      {ref,"0288719ae15814f3a53114c657502a24376bebfa"}},
+       {ref,"0288719ae15814f3a53114c657502a24376bebfa"}},
   1},
  {<<"gproc">>,{pkg,<<"gproc">>,<<"1.0.0">>},0},
  {<<"setup">>,
   {git,"https://github.com/uwiger/setup",
-      {ref,"3ad83ed3cade6780897d11f5f0d987012b93345b"}},
+       {ref,"d9b0b51505ff75c459f1ab62bd907ca5e3a32291"}},
   0}]}.
 [
 {pkg_hash,[

src/gmhc_app.erl

@@ -1,5 +1,6 @@
 %% -*- mode: erlang; erlang-indent-level: 4; indent-tabs-mode: nil -*-
 -module(gmhc_app).
+-vsn("0.4.8").
 
 -behaviour(application).
 

src/gmhc_config.erl

@@ -1,8 +1,8 @@
 -module(gmhc_config).
+-vsn("0.4.8").
 
 -export([ load_config/0
         , get_config/1
-        , verify_cfg_props/1
         ]).
 
 -include_lib("kernel/include/logger.hrl").
@@ -13,6 +13,7 @@ load_config() ->
     gmconfig:load_user_config(report),
     gmconfig:apply_os_env(),
     gmconfig:process_plain_args(),
+    check_application_env(),
     ok.
 
 instrument_gmconfig() ->
@@ -31,37 +32,35 @@ gmconfig_env() ->
 get_config(Path) ->
     gmconfig:get_config(Path, cpath(Path)).
 
-cpath([<<"pubkey">>])                -> [user_config, env(pubkey        )                ];
-cpath([<<"extra_pubkeys">>])         -> [user_config, env(extra_pubkeys ), schema_default];
-cpath([<<"pool_admin">>, <<"url">>]) -> [user_config, env(pool_admin_url), schema_default];
-cpath([<<"workers">>])               -> [user_config, env(workers       ), schema_default];
-cpath(_)                             -> [user_config,                      schema_default].
-
-env2cpath(pubkey        ) -> [<<"pubkey">>];
-env2cpath(extra_pubkeys ) -> [<<"extra_pubkeys">>];
-env2cpath(pool_admin_url) -> [<<"pool_admin">>, <<"url">>];
-env2cpath(workers       ) -> [<<"workers">>].
-
-verify_cfg_props(PropList) ->
-    Cfg = lists:foldl(
-            fun({K,V}, M) ->
-                    CfgK = env2cpath(K),
-                    gmconfig:merge_config_maps(M, to_cfg_map(CfgK, V))
-            end, #{}, PropList),
-    case gmconfig_schema_utils:valid(Cfg) of
-        Ok when is_map(Ok) ->
+check_application_env() ->
+    case lists:foldl(fun appl_env/2, #{}, env_keys()) of
+        C when map_size(C) == 0 ->
             ok;
-        Other ->
-            error({invalid_config, Other})
+        C ->
+            gmconfig:update_config(C)
+    end.
+appl_env(K, C) ->
+    case application:get_env(gmhive_client, K) of
+        {ok, V}   -> appl_env(K, V, C);
+        undefined -> C
     end.
 
-to_cfg_map(K, V) ->
-    to_cfg_map(K, V, #{}).
+appl_env(bin_dir_hook  , H , C) -> set_bin_dir_hook(H), C;
+appl_env(pubkey        , K , C) -> C#{<<"pubkey">> => K};
+appl_env(extra_pubkeys , Ks, C) -> C#{<<"extra_pubkeys">> => Ks};
+appl_env(pool_admin_url, U , C) -> merge(C, #{<<"pool_admin">> => #{<<"url">> => U}});
+appl_env(workers       , Ws, C) -> C#{<<"workers">> => Ws}.
 
-to_cfg_map([H], V, M) ->
-    M#{H => V};
-to_cfg_map([H|T], V, M) ->
-    M#{H => to_cfg_map(T, V, M)}.
+set_bin_dir_hook({Mod, F}) when is_atom(Mod), is_atom(F) ->
+    application:set_env(gmhive_worker, bin_dir_hook, {Mod, F}).
 
-env(K) ->
-    {env, gmhive_client, K}.
+env_keys() -> [pubkey, extra_pubkeys, pool_admin_url, workers, network, bin_dir_hook].
+
+cpath([<<"pubkey">>])                -> [user_config                ];
+cpath([<<"extra_pubkeys">>])         -> [user_config, schema_default];
+cpath([<<"pool_admin">>, <<"url">>]) -> [user_config, schema_default];
+cpath([<<"workers">>])               -> [user_config, schema_default];
+cpath(_)                             -> [user_config, schema_default].
+
+merge(A, B) ->
+    gmconfig:merge_config_maps(A, B).

src/gmhc_config_schema.erl

@@ -1,4 +1,5 @@
 -module(gmhc_config_schema).
+-vsn("0.4.8").
 
 -export([ schema/0
         , to_json/0 ]).
@@ -24,7 +25,8 @@ to_json() ->
 schema() ->
     obj(schema_init(),
         #{
-           pubkey => str(#{pattern => ?ACCOUNT_PATTERN},
+           network => str(#{ default => <<"mainnet">> })
+         , pubkey => str(#{pattern => ?ACCOUNT_PATTERN},
                          <<"Primary client pubkey">>)
          , extra_pubkeys => array(#{ description =>
                                          <<"Additional worker pubkeys, sharing rewards">>
@@ -52,7 +54,14 @@ pool() ->
 pool_admin() ->
     obj(#{
           url => str(#{ default => <<"https://test.gajumining.com/api/workers/{CLIENT_ID}">>
-                      , description => <<"URL of Eureka worker api">> })
+                      , description => <<"URL of Eureka worker api">> }),
+          default_per_network =>
+              obj(#{
+                     testnet =>
+                        str(#{default => <<"https://test.gajumining.com/api/workers/{CLIENT_ID}">>})
+                   , mainnet =>
+                        str(#{default => <<"https://gajumining.com/api/workers/{CLIENT_ID}">>})
+                   })
          }).
 
 workers() ->
@@ -65,17 +74,8 @@ workers() ->
           #{executable =>
                 str(#{default => <<"mean29-generic">>,
                       description =>
-                          <<"Executable binary of the worker. Options are: \"mean29-generic\""
-                            " (memory-intensive), "
-                            "\"mean29-avx2\" (memory-intensive, benefits from faster CPU supporting"
-                            " AVX2 instructions), "
-                            "\"lean29-generic\" (CPU-intensive, useful if memory-constrained), "
-                            "\"lean29-avx2\" (CPU-intensive, useful if memory-constrained, benefits "
-                            "from faster CPU supporting AVX2 instructions).">>}),
-            executable_group =>
-                str(#{description => <<"Group of executable binaries of the worker.">>,
-                      enum => [ <<"aecuckoo">>, <<"aecuckooprebuilt">>, <<"gmcuckoo">>, <<"cuda">>, <<"gajumine">> ],
-                      default => <<"aecuckoo">>}),
+                          <<"Executable binary of the worker. Can be a fully qualified path, \n"
+                             "but the software may apply default logic to locate a plain basename.">>}),
             extra_args =>
                 str(#{description => <<"Extra arguments to pass to the worker executable binary. "
                                        "The safest choice is specifying no arguments i.e. empty string.">>,

src/gmhc_connector.erl

@@ -1,4 +1,5 @@
 -module(gmhc_connector).
+-vsn("0.4.8").
 
 -behaviour(gen_server).
 

src/gmhc_connectors_sup.erl

@@ -1,4 +1,5 @@
 -module(gmhc_connectors_sup).
+-vsn("0.4.8").
 -behavior(supervisor).
 
 -export([ start_link/0

src/gmhc_counters.erl

@@ -1,4 +1,5 @@
 -module(gmhc_counters).
+-vsn("0.4.8").
 
 -export([ initialize/0 ]).
 

src/gmhc_eureka.erl

@@ -1,4 +1,5 @@
 -module(gmhc_eureka).
+-vsn("0.4.8").
 
 -export([get_pool_address/0]).
 
@@ -6,14 +7,26 @@
 -include("gmhc_events.hrl").
 
 get_pool_address() ->
-    URL0 = gmhc_config:get_config([<<"pool_admin">>, <<"url">>]),
-    case expand_url(URL0) of
-        <<"local">> ->
-            #{<<"address">> => <<"127.0.0.1">>,
-              <<"port">>    => gmconfig:get_config([<<"pool">>, <<"port">>], [schema_default]),
-              <<"pool_id">> => gmhc_config:get_config([<<"pool">>, <<"id">>]) };
-        URL ->
-            ?LOG_INFO("Trying to connect to ~p", [URL]),
+    case gmconfig:find_config([<<"pool_admin">>, <<"url">>], [user_config]) of
+        {ok, URL0} ->
+            case expand_url(URL0) of
+                <<"local">> ->
+                    #{<<"address">> => <<"127.0.0.1">>,
+                      <<"port">>    => gmconfig:get_config(
+                                         [<<"pool">>, <<"port">>], [schema_default]),
+                      <<"pool_id">> => gmhc_config:get_config([<<"pool">>, <<"id">>]) };
+                URL ->
+                    ?LOG_INFO("Trying to connect to ~p", [URL]),
+                    connect1(URL)
+            end;
+        undefined ->
+            Network = gmconfig:get_config([<<"network">>]),
+            URL0 = gmconfig:get_config([ <<"pool_admin">>
+                                       , <<"default_per_network">>
+                                       , Network ],
+                                       [schema_default]),
+            URL = expand_url(URL0),
+            ?LOG_INFO("Using default for ~p: ~p", [Network, URL]),
             connect1(URL)
     end.
 

src/gmhc_events.erl

@@ -1,4 +1,5 @@
 -module(gmhc_events).
+-vsn("0.4.8").
 
 -export([subscribe/1,
          ensure_subscribed/1,

src/gmhc_handler.erl

@@ -1,4 +1,5 @@
 -module(gmhc_handler).
+-vsn("0.4.8").
 -behavior(gen_server).
 
 -export([ start_link/0
@@ -129,12 +130,15 @@ call_connector(Req0) ->
             gmhc_connector:send(ViaId, #{call => Req#{ id => Id }}),
             receive
                 {from_pool, #{reply := #{ id := Id, result := Result }}} ->
+                    erlang:demonitor(MRef),
                     Result;
                 {from_pool, #{error := #{ id := Id } = Error}} ->
+                    erlang:demonitor(MRef),
                     {error, maps:remove(id, Error)};
                 {'DOWN', MRef, _, _, _} ->
                     {error, no_connection}
             after 5000 ->
+                    erlang:demonitor(MRef),
                     {error, {timeout, process_info(self(), messages)}}
             end
     end.

src/gmhc_server.erl

@@ -1,4 +1,5 @@
 -module(gmhc_server).
+-vsn("0.4.8").
 
 -behaviour(gen_server).
 
@@ -44,7 +45,7 @@
 
 -define(CONNECTED(S), map_size(S#st.connected) > 0).
 
--define(MAX_ERRORS, 5).
+-define(MAX_ERRORS, 50).
 
 connected(Id, Type) ->
     gen_server:call(?MODULE, {connected, Id, Type}).
@@ -102,13 +103,26 @@ handle_cast({from_pool, #{via := Connector,
                               #{candidate := Cand0}}},
                           #st{workers = Workers} = S) ->
     Cand = maps:put(via, Connector, decode_candidate_hash(Cand0)),
-    %% ?LOG_DEBUG("Got new candidate; will mine it: ~p", [Cand]),
+    ?LOG_DEBUG("Got new candidate; will mine it: ~p", [Cand]),
+    %%
+    %% We could check whether we have already received the candidate ...
     %% For now, stop all workers, restart with new candidate
-    Workers1 = stop_workers(Workers),
-    {Workers2, Cand1} = assign_nonces(Workers1, Cand),
-    #st{candidate = Cand2} = S1 = maybe_request_nonces(S#st{candidate = Cand1}),
-    NewWorkers = [spawn_worker(W, Cand2) || W <- Workers2],
-    {noreply, S1#st{workers = NewWorkers}};
+    try
+        %% Most of the time we don't want to stop the worker. If we do, though, then
+        %% we need to do it more carefully than this, or memory usage will triple.
+        %% Workers1 = stop_workers(Workers),
+        %%
+        %% Nonces may be [], in which case we need to request new nonces first.
+        #st{candidate = Cand1} = S1 = maybe_request_nonces(S#st{candidate = Cand}),
+        {Workers2, Cand2} = assign_nonces(Workers, Cand1),
+        #st{candidate = Cand3} = S2 = maybe_request_nonces(S1#st{candidate = Cand2}),
+        NewWorkers = [spawn_worker(W, Cand3) || W <- Workers2],
+        {noreply, S2#st{workers = NewWorkers}}
+    catch
+       Cat:Err:St ->
+           ?LOG_ERROR("CAUGHT ~p:~p / ~p", [Cat, Err, St]),
+           {noreply, S}
+    end;
 handle_cast({disconnected, Id}, #st{connected = Conn} = S) ->
     ?LOG_DEBUG("disconnected: ~p", [Id]),
     Conn1 = maps:remove(Id, Conn),
@@ -122,9 +136,7 @@ handle_cast(_Msg, S) ->
     {noreply, S}.
 
 handle_info({'DOWN', MRef, process, Pid, Reason}, #st{ workers = Workers
-                                                     , connected = Connected
-                                                     , working = Working} = S)
-  when ?CONNECTED(S), Working ->
+                                                     , connected = Connected} = S) ->
     %% ?LOG_DEBUG("DOWN from ~p: ~p", [Pid, Reason]),
     case lists:keyfind(Pid, #worker.pid, Workers) of
         #worker{mref = MRef} = W ->
@@ -143,11 +155,23 @@ handle_info({'EXIT', Pid, Reason}, #st{ workers = Workers
             gmhc_events:publish(error, ?ERR_EVT(#{error => worker_error,
                                                    data  => Reason})),
             Ws1 = incr_worker_error(W, Workers),
+            erlang:start_timer(100, self(), check_workers),
             {noreply, S#st{workers = Ws1}};
         false ->
             %% ?LOG_DEBUG("EXIT apparently not from worker?? (~p)", [Pid]),
             {noreply, S}
     end;
+handle_info({timeout, _, check_workers}, #st{workers = Workers} = S) ->
+    case [W || #worker{cand = undefined} = W <- Workers] of
+        [] ->
+            {noreply, S};
+        Idle ->
+            S1 = maybe_request_nonces(S),
+            S2 = lists:foldl(fun(W, Sx) ->
+                                 maybe_restart_worker(W, Sx)
+                             end, S1, Idle),
+            {noreply, S2}
+    end;
 handle_info(Msg, St) ->
     ?LOG_DEBUG("Unknown msg: ~p", [Msg]),
     {noreply, St}.
@@ -215,14 +239,14 @@ handle_worker_result({worker_result, Result}, W, S) ->
     case Result of
         {solutions, Solutions} ->
             {Cont, S1} = report_solutions_(Solutions, W, S),
-            maybe_continue(Cont, W, S1);
+            maybe_continue(Cont, reset_errors(W), S1);
         {solution, Nonce, Solution} ->
             %% report_solution(Nonce, Solution, W, S),
             {Cont, S1} = report_solutions_([{Nonce, Solution}], W, S),
-            maybe_continue(Cont, W, S1);
+            maybe_continue(Cont, reset_errors(W), S1);
         {no_solution, Nonce} ->
             report_no_solution(Nonce, W, S),
-            maybe_restart_worker(W, S);
+            maybe_restart_worker(reset_errors(W), S);
         {error, S} ->
             ?LOG_DEBUG("Worker ~p reported error as normal", [W#worker.index]),
             gmhc_events:publish(error, ?ERR_EVT(#{error => worker_error,
@@ -240,15 +264,19 @@ handle_worker_result(Error, W, S) ->
 report_solutions_(Solutions, W, S) ->
     case report_solutions(Solutions, W, S) of
         ok ->
-            Ws = reset_worker(W, S#st.workers),
-            Ws1 = stop_workers(Ws),
-            {stopped, S#st{workers = Ws1}};
+            %% Ws = reset_worker(W, S#st.workers),
+            %% Ws1 = stop_workers(Ws),
+            %% {stopped, S#st{workers = Ws1}};
+            {continue, S};
         continue ->
             {continue, S};
         {error, _} ->
             {error, S}
     end.
 
+reset_errors(#worker{} = W) ->
+    W#worker{errors = 0}.
+
 reset_worker(#worker{index = I} = W, Ws) ->
     W1 = reset_worker_(W),
     lists:keyreplace(I, #worker.index, Ws, W1).
@@ -263,13 +291,14 @@ incr_worker_error(#worker{errors = Es, index = I} = W, Ws) ->
     W1 = reset_worker_(W#worker{errors = Es+1}),
     lists:keyreplace(I, #worker.index, Ws, W1).
 
-maybe_continue(stopped, _, S) ->
-    S;
+%% maybe_continue(stopped, _, S) ->
+%%     S;
 maybe_continue(continue, W, S) ->
     maybe_restart_worker(W, S);
 maybe_continue(error, W, S) ->
     ?LOG_INFO("Won't restart worker ~p due to error", [W#worker.index]),
-    S.
+    Ws = reset_worker(W, S#st.workers),
+    S#st{workers = Ws}.
 
 maybe_restart_worker(#worker{index = I} = W, #st{candidate = C} = S) ->
     case maps:get(nonces, C) of
@@ -302,8 +331,10 @@ stop_workers_for_seq(Seq, Workers) ->
     [stop_worker(W) || #worker{cand = #{seq := Seq1}} = W <- Workers,
                        Seq1 =:= Seq].
 
-stop_worker(#worker{pid = Pid, mref = MRef} = W) when is_pid(Pid) ->
-    exit(Pid, kill),
+stop_worker(#worker{pid = Pid} = W) when is_pid(Pid) ->
+    MRef = erlang:monitor(process, Pid),
+    ?LOG_DEBUG("Will stop worker ~p (MRef = ~p)", [Pid, MRef]),
+    exit(Pid, shutdown),
     receive
         {'EXIT', Pid, _} -> ok;
         {'DOWN', MRef, process, Pid, _} -> ok
@@ -362,7 +393,10 @@ spawn_worker(#worker{pid = undefined, nonce = Nonce, config = Cfg} = W, Cand) ->
                     init_worker(Data, Nonce, Target, Cfg1, Me)
             end),
     MRef = erlang:monitor(process, Pid),
-    W#worker{pid = Pid, mref = MRef, cand = Cand, nonce = Nonce}.
+    W#worker{pid = Pid, mref = MRef, cand = Cand, nonce = Nonce};
+spawn_worker(W, _) ->
+    % Worker already has work. Don't disturb it.
+    W.
 
 -spec init_worker(binary(), integer(), integer(), tuple(), pid()) -> no_return().
 init_worker(Data, Nonce, Target, Config, Parent) ->

src/gmhc_sup.erl

@@ -1,5 +1,6 @@
 %% -*- mode: erlang; erlang-indent-level: 4; indent-tabs-mode: nil -*-
 -module(gmhc_sup).
+-vsn("0.4.8").
 
 -behaviour(supervisor).
 

src/gmhc_workers.erl

@@ -8,6 +8,7 @@
 %%%-------------------------------------------------------------------
 
 -module(gmhc_workers).
+-vsn("0.4.8").
 
 -export([
           get_worker_configs/0
@@ -16,7 +17,7 @@
 
 -include_lib("kernel/include/logger.hrl").
 
--define(DEFAULT_EXECUTABLE_GROUP   , <<"gajumine">>).
+-define(DEFAULT_EXECUTABLE_GROUP   , <<"gmhive">>).
 -define(DEFAULT_EXTRA_ARGS         , <<>>).
 -define(DEFAULT_HEX_ENCODED_HEADER , false).
 -define(DEFAULT_REPEATS            , 1).
@@ -55,7 +56,7 @@ generate_from_hash(Hash, Target, Nonce, Config, WorkerInstance) ->
 
 build_worker_config(Config) when is_map(Config) ->
     Exec      = maps:get(<<"executable">>, Config),
-    ExecGroup = maps:get(<<"executable_group">>, Config, ?DEFAULT_EXECUTABLE_GROUP),
+    ExecGroup = ?DEFAULT_EXECUTABLE_GROUP,
     ExtraArgs = maps:get(<<"extra_args">>, Config, ?DEFAULT_EXTRA_ARGS),
     HexEncHdr = maps:get(<<"hex_encoded_header">>, Config,
                          hex_encoding_default(ExecGroup, Exec)),

src/gmhive_client.erl

@@ -1,4 +1,5 @@
 -module(gmhive_client).
+-vsn("0.4.8").
 
 -export([ connect/1
         , disconnect/1

zomp.meta

@@ -2,10 +2,14 @@
 {type,app}.
 {modules,[]}.
 {prefix,"gmhc"}.
-{desc,"Gajumaru Hive Client"}.
 {author,"Ulf Wiger, QPQ AG"}.
-{package_id,{"uwiger","gmhive_client",{0,2,1}}}.
-{deps,[{"uwiger","gmhive_worker",{0,1,1}},
+{desc,"Gajumaru Hive Client"}.
+{package_id,{"uwiger","gmhive_client",{0,4,8}}}.
+{deps,[{"uwiger","gmhive_worker",{0,4,0}},
+       {"uwiger","gmcuckoo",{1,2,3}},
+       {"otpr","eblake2",{1,0,1}},
+       {"otpr","base58",{0,1,1}},
+       {"otpr","gmserialization",{0,1,3}},
        {"uwiger","gmhive_protocol",{0,1,1}},
        {"uwiger","setup",{2,2,4}},
        {"uwiger","gproc",{1,0,1}},