Merge pull request #15 from aeternity/tb-add-error-store

Add caller error store to error handling

README.md

@@ -1,11 +1,13 @@
 # mnesia_rocksdb
-A RocksDb backend for Mnesia
+
+A RocksDB backend for Mnesia.
 
 This permits Erlang/OTP applications to use RocksDB as a backend for
 mnesia tables. It is based on Klarna's `mnesia_eleveldb`.
 
 ## Prerequisites
-- rocksdb
+
+- rocksdb (included as dependency)
 - Erlang/OTP 20.0 or newer (https://github.com/erlang/otp)
 
 ## Getting started
@@ -76,13 +78,16 @@ Contributions are welcome.
 
 The RocksDB update operations return either `ok` or `{error, any()}`.
 Since the actual updates are performed after the 'point-of-no-return',
-returning an `error` result will cause mnesia to behave unpredictably, since
+returning an `error` result will cause mnesia to behave unpredictably,
-the operations are expected to simply work.
+since the operations are expected to simply work.
+
+### Option 1: `on_write_error`
 
 An `on_write_error` option can be provided, per-table, in the `rocksdb_opts`
-user property (see [Customization](#customization) above). Supported values indicate at which level an error
+user property (see [Customization](#customization) above).
-indication should be reported. Mnesia may save reported events in RAM, and may
+Supported values indicate at which level an error indication should be reported.
-also print them, depending on the debug level (controlled with `mnesia:set_debug_level/1`).
+Mnesia may save reported events in RAM, and may also print them,
+depending on the debug level (controlled with `mnesia:set_debug_level/1`).
 
 Mnesia debug levels are, in increasing detail, `none | verbose | debug | trace`
 The supported values for `on_write_error` are:
@@ -95,6 +100,22 @@ The supported values for `on_write_error` are:
  | error   | always               | always                 | exception |
  | fatal   | always               | always                 | core dump |
 
+### Option 2: `on_write_error_store`
+
+An `on_write_error_store` option can be provided, per-table, in the `rocksdb_opts`
+user property (see [Customization](#customization) above).
+When set, the backend will use the value of the option as the name for an ETS table
+which is used as storage for runtime write errors. The table must be set up outside
+of the backend by the clients themselves.
+
+Entries to the table are in the form of a tuple `{{Table, Key}, Error, InsertedAt}`
+where `Table` refers to the Mnesia table name, `Key` is the primary key being used by Mnesia,
+`Error` is the error encountered by the backend, and `InsertedAt` refers to the time
+the error was encountered as system time in milliseconds.
+
+The backend will only insert entries and otherwise not manage the table. Thus, clients
+are expected to clean up the table during runtime to prevent memory leakage.
+
 ## Caveats
 
 Avoid placing `bag` tables in RocksDB. Although they work, each write

src/mnesia_rocksdb.erl

@@ -23,7 +23,6 @@
 
 -module(mnesia_rocksdb).
 
-
 %% ----------------------------------------------------------------------------
 %% BEHAVIOURS
 %% ----------------------------------------------------------------------------
@@ -31,7 +30,6 @@
 -behaviour(mnesia_backend_type).
 -behaviour(gen_server).
 
-
 %% ----------------------------------------------------------------------------
 %% EXPORTS
 %% ----------------------------------------------------------------------------
@@ -173,19 +171,24 @@
              , compiled_ms
              , limit
              , key_only = false                 % TODO: not used
-             , direction = forward}).           % TODO: not used
+             , direction = forward              % TODO: not used
+             }).
 
 -type on_write_error() :: debug | verbose | warning | error | fatal.
+-type on_write_error_store() :: atom() | undefined.
+
 -define(WRITE_ERR_DEFAULT, verbose).
+-define(WRITE_ERR_STORE_DEFAULT, undefined).
 
 -record(st, { ets
             , ref
             , alias
             , tab
             , type
-            , size_warnings			% integer()
+            , size_warnings :: integer()
-            , maintain_size			% boolean()
+            , maintain_size :: boolean()
             , on_write_error = ?WRITE_ERR_DEFAULT :: on_write_error()
+            , on_write_error_store = ?WRITE_ERR_STORE_DEFAULT :: on_write_error_store()
             }).
 
 %% ----------------------------------------------------------------------------
@@ -209,7 +212,6 @@ register(Alias) ->
 default_alias() ->
     rocksdb_copies.
 
-
 %% ----------------------------------------------------------------------------
 %% DEBUG API
 %% ----------------------------------------------------------------------------
@@ -342,33 +344,38 @@ prefixes(_) ->
 %% set is OK as ordered_set is a kind of set.
 check_definition(Alias, Tab, Nodes, Props) ->
     Id = {Alias, Nodes},
-    try lists:map(
+    try
-          fun({type, T} = P) ->
+        Props1 = lists:map(fun(E) -> check_definition_entry(Tab, Id, E) end, Props),
-                  if T==set; T==ordered_set; T==bag ->
-                          P;
-                     true ->
-                          mnesia:abort({combine_error,
-                                        Tab,
-                                        [Id, {type,T}]})
-                  end;
-             ({user_properties, UPs} = P) ->
-                  RdbOpts = proplists:get_value(rocksdb_opts, UPs, []),
-                  OWE = proplists:get_value(on_write_error, RdbOpts, ?WRITE_ERR_DEFAULT),
-                  case valid_mnesia_op(OWE) of
-                      true ->
-                          P;
-                      false ->
-                          throw({error, {invalid, {on_write_error, OWE}}})
-                  end;
-             (P) -> P
-          end, Props) of
-        Props1 ->
         {ok, Props1}
     catch
         throw:Error ->
             Error
     end.
 
+check_definition_entry(_Tab, _Id, {type, T} = P) when T==set; T==ordered_set; T==bag ->
+    P;
+check_definition_entry(Tab, Id, {type, T}) ->
+    mnesia:abort({combine_error, Tab, [Id, {type, T}]});
+check_definition_entry(_Tab, _Id, {user_properties, UPs} = P) ->
+    RdbOpts = proplists:get_value(rocksdb_opts, UPs, []),
+    OWE = proplists:get_value(on_write_error, RdbOpts, ?WRITE_ERR_DEFAULT),
+    OWEStore = proplists:get_value(on_write_error_store, RdbOpts, ?WRITE_ERR_STORE_DEFAULT),
+    case valid_mnesia_op(OWE) of
+        true ->
+            case OWEStore of
+                undefined ->
+                    P;
+                V when is_atom(V) ->
+                    P;
+                V ->
+                    throw({error, {invalid_configuration, {on_write_error_store, V}}})
+            end;
+        false ->
+            throw({error, {invalid_configuration, {on_write_error, OWE}}})
+    end;
+check_definition_entry(_Tab, _Id, P) ->
+    P.
+
 %% -> ok | {error, exists}
 create_table(_Alias, Tab, _Props) ->
     create_mountpoint(Tab).
@@ -666,7 +673,8 @@ lookup(Alias, Tab, Key) ->
     Enc = encode_key(Key),
     {Ref, Type} = call(Alias, Tab, get_ref),
     case Type of
-	bag -> lookup_bag(Ref, Key, Enc, keypos(Tab));
+        bag ->
+            lookup_bag(Ref, Key, Enc, keypos(Tab));
         _ ->
             case ?rocksdb:get(Ref, Enc, []) of
                 {ok, EncVal} ->
@@ -893,6 +901,8 @@ init({Alias, Tab, Type, RdbOpts}) ->
     process_flag(trap_exit, true),
     try
         {ok, Ref, Ets} = do_load_table(Tab, RdbOpts),
+        OWE = proplists:get_value(on_write_error, RdbOpts, ?WRITE_ERR_DEFAULT),
+        OWEStore = proplists:get_value(on_write_error_store, RdbOpts, ?WRITE_ERR_STORE_DEFAULT),
         St = #st{ ets = Ets
                 , ref = Ref
                 , alias = Alias
@@ -900,9 +910,10 @@ init({Alias, Tab, Type, RdbOpts}) ->
                 , type = Type
                 , size_warnings = 0
                 , maintain_size = should_maintain_size(Tab)
+                , on_write_error = OWE
+                , on_write_error_store = OWEStore
                 },
-        OnWriteError = proplists:get_value(on_write_error, RdbOpts, St#st.on_write_error),
+        {ok, recover_size_info(St)}
-        {ok, recover_size_info(St#st{on_write_error = OnWriteError})}
     catch
         throw:badarg ->
             {error, write_error}
@@ -1195,8 +1206,7 @@ do_insert(K, V, #st{ets = Ets, ref = Ref, type = bag, maintain_size = true} = St
 do_insert(K, V, #st{ref = Ref, maintain_size = false} = St) ->
     return_catch(fun() -> db_put(Ref, K, V, [], St) end);
 do_insert(K, V, #st{ets = Ets, ref = Ref, maintain_size = true} = St) ->
-    IsNew =
+    IsNew = case ?rocksdb:get(Ref, K, []) of
-	case ?rocksdb:get(Ref, K, []) of
                 {ok, _} ->
                     false;
                 _ ->
@@ -1279,8 +1289,7 @@ do_delete(Key, #st{ets = Ets, ref = Ref, maintain_size = true} = St) ->
     end.
 
 do_delete_bag(Sz, Key, Ref, TSz, St) ->
-    Found = 
+    Found = with_iterator(
-	with_iterator(
               Ref, fun(I) ->
                            do_delete_bag_(Sz, Key, ?rocksdb:iterator_move(I, Key),
                                           Ref, I)
@@ -1440,8 +1449,7 @@ do_select(Ref, Tab, _Type, MS, AccKeys, Limit) when is_boolean(AccKeys) ->
 i_do_select(I, #sel{keypat = Pfx,
                     compiled_ms = MS,
                     limit = Limit} = Sel, AccKeys, Acc) ->
-    StartKey =
+    StartKey = case Pfx of
-        case Pfx of
                    <<>> ->
                        <<?DATA_START>>;
                    _ ->
@@ -1612,8 +1620,9 @@ db_delete(Ref, K, Opts, St) ->
 
 write_result(ok, _, _, _) ->
     ok;
-write_result(Res, Op, Args, #st{on_write_error = Rpt}) ->
+write_result(Res, Op, Args, #st{tab = Tab, on_write_error = Rpt, on_write_error_store = OWEStore}) ->
     RptOp = rpt_op(Rpt),
+    maybe_store_error(OWEStore, Res, Tab, Op, Args, erlang:system_time(millisecond)),
     mnesia_lib:RptOp("FAILED rocksdb:~p(" ++ rpt_fmt(Args) ++ ") -> ~p~n",
                      [Op | Args] ++ [Res]),
     if Rpt == fatal; Rpt == error ->
@@ -1622,6 +1631,26 @@ write_result(Res, Op, Args, #st{on_write_error = Rpt}) ->
             ok
     end.
 
+maybe_store_error(undefined, _, _, _, _, _) ->
+    ok;
+maybe_store_error(Table, Err, IntTable, put, [_, K, _, _], Time) ->
+    insert_error(Table, IntTable, K, Err, Time);
+maybe_store_error(Table, Err, IntTable, delete, [_, K, _], Time) ->
+    insert_error(Table, IntTable, K, Err, Time);
+maybe_store_error(Table, Err, IntTable, write, [_, List, _], Time) ->
+    lists:map(fun
+                  ({put, K, _}) ->
+                      insert_error(Table, IntTable, K, Err, Time);
+                  ({delete, K}) ->
+                      insert_error(Table, IntTable, K, Err, Time)
+              end, List).
+
+insert_error(Table, {Type, _, _}, K, Err, Time) ->
+    {_, K1} = decode_key(K),
+    ets:insert(Table, {{Type, K1}, Err, Time});
+insert_error(Table, Type, K, Err, Time) when is_atom(Type) ->
+    ets:insert(Table, {{Type, K}, Err, Time}).
+
 rpt_fmt([_|T]) ->
     lists:append(["~p" | [", ~p" || _ <- T]]).
 

test/mnesia_rocksdb_SUITE.erl

@@ -29,7 +29,6 @@ groups() ->
 error_handling(_Config) ->
     mnesia_rocksdb_error_handling:run().
 
-
 init_per_suite(Config) ->
     Config.
 

test/mnesia_rocksdb_error_handling.erl

@@ -31,6 +31,14 @@ setup() ->
 
 create_tab(Type, Level, MaintainSz) ->
     TabName = tab_name(Type, Level, MaintainSz),
+    %% create error store before the table
+    case ets:info(?MODULE) of
+        undefined ->
+            ?MODULE = ets:new(?MODULE, [bag, public, named_table]),
+            ok;
+        _ ->
+            ok
+    end,
     UserProps = user_props(Level, MaintainSz),
     {atomic, ok} = mnesia:create_table(TabName, [{rdb, [node()]},
                                                  {user_properties, UserProps}]),
@@ -43,7 +51,8 @@ tab_name(Type, Level, MaintainSz) ->
 
 user_props(Level, MaintainSz) ->
     [{maintain_sz, MaintainSz},
-     {rocksdb_opts, [{on_write_error, Level}]}].
+     {rocksdb_opts, [ {on_write_error, Level}
+                    , {on_write_error_store, ?MODULE} ]}].
 
 start_mnesia() ->
     mnesia_rocksdb_tlib:start_mnesia(reset),
@@ -94,7 +103,11 @@ expect_error(Level, Tab) ->
             ok
     after 1000 ->
             error({expected_error, [Level, Tab]})
-    end.
+
+    end,
+    %% Also verify that an error entry has been written into the error store.
+    1 = ets:select_delete(?MODULE, [{{{Tab, '_'}, '_', '_'}, [], [true]}]),
+    ok.
 
 rpt_tag(fatal  ) -> mnesia_fatal;
 rpt_tag(error  ) -> mnesia_error;