diff --git a/src/gmhc_server.erl b/src/gmhc_server.erl
index 9ad83dc..68476bd 100644
--- a/src/gmhc_server.erl
+++ b/src/gmhc_server.erl
@@ -107,8 +107,10 @@ handle_cast({from_pool, #{via := Connector,
     %% We could check whether we have already received the candidate ...
     %% For now, stop all workers, restart with new candidate
     try
-        Workers1 = stop_workers(Workers),
-        {Workers2, Cand1} = assign_nonces(Workers1, Cand),
+        % Most of the time we don't want to stop the worker. If we do, though, then
+        % we need to do it more carefully than this, or memory usage will triple.
+        % Workers1 = stop_workers(Workers),
+        {Workers2, Cand1} = assign_nonces(Workers, Cand),
         #st{candidate = Cand2} = S1 = maybe_request_nonces(S#st{candidate = Cand1}),
         NewWorkers = [spawn_worker(W, Cand2) || W <- Workers2],
         {noreply, S1#st{workers = NewWorkers}}
@@ -371,7 +373,10 @@ spawn_worker(#worker{pid = undefined, nonce = Nonce, config = Cfg} = W, Cand) ->
                     init_worker(Data, Nonce, Target, Cfg1, Me)
             end),
     MRef = erlang:monitor(process, Pid),
-    W#worker{pid = Pid, mref = MRef, cand = Cand, nonce = Nonce}.
+    W#worker{pid = Pid, mref = MRef, cand = Cand, nonce = Nonce};
+spawn_worker(W, _) ->
+    % Worker already has work. Don't disturb it.
+    W.
 
 -spec init_worker(binary(), integer(), integer(), tuple(), pid()) -> no_return().
 init_worker(Data, Nonce, Target, Config, Parent) ->