Merge pull request #15 from aeternity/tb-add-error-store
Add caller error store to error handling
This commit is contained in:
Tino Breddin
GitHub
commit
8d3079ff25
35
README.md
35
README.md
@ -1,11 +1,13 @@
|
||||
# mnesia_rocksdb
|
||||
A RocksDb backend for Mnesia
|
||||
|
||||
A RocksDB backend for Mnesia.
|
||||
|
||||
This permits Erlang/OTP applications to use RocksDB as a backend for
|
||||
mnesia tables. It is based on Klarna's `mnesia_eleveldb`.
|
||||
|
||||
## Prerequisites
|
||||
- rocksdb
|
||||
|
||||
- rocksdb (included as dependency)
|
||||
- Erlang/OTP 20.0 or newer (https://github.com/erlang/otp)
|
||||
|
||||
## Getting started
|
||||
@ -76,13 +78,16 @@ Contributions are welcome.
|
||||
|
||||
The RocksDB update operations return either `ok` or `{error, any()}`.
|
||||
Since the actual updates are performed after the 'point-of-no-return',
|
||||
returning an `error` result will cause mnesia to behave unpredictably, since
|
||||
the operations are expected to simply work.
|
||||
returning an `error` result will cause mnesia to behave unpredictably,
|
||||
since the operations are expected to simply work.
|
||||
|
||||
### Option 1: `on_write_error`
|
||||
|
||||
An `on_write_error` option can be provided, per-table, in the `rocksdb_opts`
|
||||
user property (see [Customization](#customization) above). Supported values indicate at which level an error
|
||||
indication should be reported. Mnesia may save reported events in RAM, and may
|
||||
also print them, depending on the debug level (controlled with `mnesia:set_debug_level/1`).
|
||||
user property (see [Customization](#customization) above).
|
||||
Supported values indicate at which level an error indication should be reported.
|
||||
Mnesia may save reported events in RAM, and may also print them,
|
||||
depending on the debug level (controlled with `mnesia:set_debug_level/1`).
|
||||
|
||||
Mnesia debug levels are, in increasing detail, `none | verbose | debug | trace`
|
||||
The supported values for `on_write_error` are:
|
||||
@ -95,6 +100,22 @@ The supported values for `on_write_error` are:
|
||||
| error | always | always | exception |
|
||||
| fatal | always | always | core dump |
|
||||
|
||||
### Option 2: `on_write_error_store`
|
||||
|
||||
An `on_write_error_store` option can be provided, per-table, in the `rocksdb_opts`
|
||||
user property (see [Customization](#customization) above).
|
||||
When set, the backend will use the value of the option as the name for an ETS table
|
||||
which is used as storage for runtime write errors. The table must be set up outside
|
||||
of the backend by the clients themselves.
|
||||
|
||||
Entries to the table are in the form of a tuple `{{Table, Key}, Error, InsertedAt}`
|
||||
where `Table` refers to the Mnesia table name, `Key` is the primary key being used by Mnesia,
|
||||
`Error` is the error encountered by the backend, and `InsertedAt` refers to the time
|
||||
the error was encountered as system time in milliseconds.
|
||||
|
||||
The backend will only insert entries and otherwise not manage the table. Thus, clients
|
||||
are expected to clean up the table during runtime to prevent memory leakage.
|
||||
|
||||
## Caveats
|
||||
|
||||
Avoid placing `bag` tables in RocksDB. Although they work, each write
|
||||
|
||||
@ -23,7 +23,6 @@
|
||||
|
||||
-module(mnesia_rocksdb).
|
||||
|
||||
|
||||
%% ----------------------------------------------------------------------------
|
||||
%% BEHAVIOURS
|
||||
%% ----------------------------------------------------------------------------
|
||||
@ -31,7 +30,6 @@
|
||||
-behaviour(mnesia_backend_type).
|
||||
-behaviour(gen_server).
|
||||
|
||||
|
||||
%% ----------------------------------------------------------------------------
|
||||
%% EXPORTS
|
||||
%% ----------------------------------------------------------------------------
|
||||
@ -173,19 +171,24 @@
|
||||
, compiled_ms
|
||||
, limit
|
||||
, key_only = false % TODO: not used
|
||||
, direction = forward}). % TODO: not used
|
||||
, direction = forward % TODO: not used
|
||||
}).
|
||||
|
||||
-type on_write_error() :: debug | verbose | warning | error | fatal.
|
||||
-type on_write_error_store() :: atom() | undefined.
|
||||
|
||||
-define(WRITE_ERR_DEFAULT, verbose).
|
||||
-define(WRITE_ERR_STORE_DEFAULT, undefined).
|
||||
|
||||
-record(st, { ets
|
||||
, ref
|
||||
, alias
|
||||
, tab
|
||||
, type
|
||||
, size_warnings % integer()
|
||||
, maintain_size % boolean()
|
||||
, size_warnings :: integer()
|
||||
, maintain_size :: boolean()
|
||||
, on_write_error = ?WRITE_ERR_DEFAULT :: on_write_error()
|
||||
, on_write_error_store = ?WRITE_ERR_STORE_DEFAULT :: on_write_error_store()
|
||||
}).
|
||||
|
||||
%% ----------------------------------------------------------------------------
|
||||
@ -209,7 +212,6 @@ register(Alias) ->
|
||||
default_alias() ->
|
||||
rocksdb_copies.
|
||||
|
||||
|
||||
%% ----------------------------------------------------------------------------
|
||||
%% DEBUG API
|
||||
%% ----------------------------------------------------------------------------
|
||||
@ -342,33 +344,38 @@ prefixes(_) ->
|
||||
%% set is OK as ordered_set is a kind of set.
|
||||
check_definition(Alias, Tab, Nodes, Props) ->
|
||||
Id = {Alias, Nodes},
|
||||
try lists:map(
|
||||
fun({type, T} = P) ->
|
||||
if T==set; T==ordered_set; T==bag ->
|
||||
P;
|
||||
true ->
|
||||
mnesia:abort({combine_error,
|
||||
Tab,
|
||||
[Id, {type,T}]})
|
||||
end;
|
||||
({user_properties, UPs} = P) ->
|
||||
RdbOpts = proplists:get_value(rocksdb_opts, UPs, []),
|
||||
OWE = proplists:get_value(on_write_error, RdbOpts, ?WRITE_ERR_DEFAULT),
|
||||
case valid_mnesia_op(OWE) of
|
||||
true ->
|
||||
P;
|
||||
false ->
|
||||
throw({error, {invalid, {on_write_error, OWE}}})
|
||||
end;
|
||||
(P) -> P
|
||||
end, Props) of
|
||||
Props1 ->
|
||||
try
|
||||
Props1 = lists:map(fun(E) -> check_definition_entry(Tab, Id, E) end, Props),
|
||||
{ok, Props1}
|
||||
catch
|
||||
throw:Error ->
|
||||
Error
|
||||
end.
|
||||
|
||||
check_definition_entry(_Tab, _Id, {type, T} = P) when T==set; T==ordered_set; T==bag ->
|
||||
P;
|
||||
check_definition_entry(Tab, Id, {type, T}) ->
|
||||
mnesia:abort({combine_error, Tab, [Id, {type, T}]});
|
||||
check_definition_entry(_Tab, _Id, {user_properties, UPs} = P) ->
|
||||
RdbOpts = proplists:get_value(rocksdb_opts, UPs, []),
|
||||
OWE = proplists:get_value(on_write_error, RdbOpts, ?WRITE_ERR_DEFAULT),
|
||||
OWEStore = proplists:get_value(on_write_error_store, RdbOpts, ?WRITE_ERR_STORE_DEFAULT),
|
||||
case valid_mnesia_op(OWE) of
|
||||
true ->
|
||||
case OWEStore of
|
||||
undefined ->
|
||||
P;
|
||||
V when is_atom(V) ->
|
||||
P;
|
||||
V ->
|
||||
throw({error, {invalid_configuration, {on_write_error_store, V}}})
|
||||
end;
|
||||
false ->
|
||||
throw({error, {invalid_configuration, {on_write_error, OWE}}})
|
||||
end;
|
||||
check_definition_entry(_Tab, _Id, P) ->
|
||||
P.
|
||||
|
||||
%% -> ok | {error, exists}
|
||||
create_table(_Alias, Tab, _Props) ->
|
||||
create_mountpoint(Tab).
|
||||
@ -666,7 +673,8 @@ lookup(Alias, Tab, Key) ->
|
||||
Enc = encode_key(Key),
|
||||
{Ref, Type} = call(Alias, Tab, get_ref),
|
||||
case Type of
|
||||
bag -> lookup_bag(Ref, Key, Enc, keypos(Tab));
|
||||
bag ->
|
||||
lookup_bag(Ref, Key, Enc, keypos(Tab));
|
||||
_ ->
|
||||
case ?rocksdb:get(Ref, Enc, []) of
|
||||
{ok, EncVal} ->
|
||||
@ -893,6 +901,8 @@ init({Alias, Tab, Type, RdbOpts}) ->
|
||||
process_flag(trap_exit, true),
|
||||
try
|
||||
{ok, Ref, Ets} = do_load_table(Tab, RdbOpts),
|
||||
OWE = proplists:get_value(on_write_error, RdbOpts, ?WRITE_ERR_DEFAULT),
|
||||
OWEStore = proplists:get_value(on_write_error_store, RdbOpts, ?WRITE_ERR_STORE_DEFAULT),
|
||||
St = #st{ ets = Ets
|
||||
, ref = Ref
|
||||
, alias = Alias
|
||||
@ -900,9 +910,10 @@ init({Alias, Tab, Type, RdbOpts}) ->
|
||||
, type = Type
|
||||
, size_warnings = 0
|
||||
, maintain_size = should_maintain_size(Tab)
|
||||
, on_write_error = OWE
|
||||
, on_write_error_store = OWEStore
|
||||
},
|
||||
OnWriteError = proplists:get_value(on_write_error, RdbOpts, St#st.on_write_error),
|
||||
{ok, recover_size_info(St#st{on_write_error = OnWriteError})}
|
||||
{ok, recover_size_info(St)}
|
||||
catch
|
||||
throw:badarg ->
|
||||
{error, write_error}
|
||||
@ -1195,8 +1206,7 @@ do_insert(K, V, #st{ets = Ets, ref = Ref, type = bag, maintain_size = true} = St
|
||||
do_insert(K, V, #st{ref = Ref, maintain_size = false} = St) ->
|
||||
return_catch(fun() -> db_put(Ref, K, V, [], St) end);
|
||||
do_insert(K, V, #st{ets = Ets, ref = Ref, maintain_size = true} = St) ->
|
||||
IsNew =
|
||||
case ?rocksdb:get(Ref, K, []) of
|
||||
IsNew = case ?rocksdb:get(Ref, K, []) of
|
||||
{ok, _} ->
|
||||
false;
|
||||
_ ->
|
||||
@ -1279,8 +1289,7 @@ do_delete(Key, #st{ets = Ets, ref = Ref, maintain_size = true} = St) ->
|
||||
end.
|
||||
|
||||
do_delete_bag(Sz, Key, Ref, TSz, St) ->
|
||||
Found =
|
||||
with_iterator(
|
||||
Found = with_iterator(
|
||||
Ref, fun(I) ->
|
||||
do_delete_bag_(Sz, Key, ?rocksdb:iterator_move(I, Key),
|
||||
Ref, I)
|
||||
@ -1440,8 +1449,7 @@ do_select(Ref, Tab, _Type, MS, AccKeys, Limit) when is_boolean(AccKeys) ->
|
||||
i_do_select(I, #sel{keypat = Pfx,
|
||||
compiled_ms = MS,
|
||||
limit = Limit} = Sel, AccKeys, Acc) ->
|
||||
StartKey =
|
||||
case Pfx of
|
||||
StartKey = case Pfx of
|
||||
<<>> ->
|
||||
<<?DATA_START>>;
|
||||
_ ->
|
||||
@ -1612,8 +1620,9 @@ db_delete(Ref, K, Opts, St) ->
|
||||
|
||||
write_result(ok, _, _, _) ->
|
||||
ok;
|
||||
write_result(Res, Op, Args, #st{on_write_error = Rpt}) ->
|
||||
write_result(Res, Op, Args, #st{tab = Tab, on_write_error = Rpt, on_write_error_store = OWEStore}) ->
|
||||
RptOp = rpt_op(Rpt),
|
||||
maybe_store_error(OWEStore, Res, Tab, Op, Args, erlang:system_time(millisecond)),
|
||||
mnesia_lib:RptOp("FAILED rocksdb:~p(" ++ rpt_fmt(Args) ++ ") -> ~p~n",
|
||||
[Op | Args] ++ [Res]),
|
||||
if Rpt == fatal; Rpt == error ->
|
||||
@ -1622,6 +1631,26 @@ write_result(Res, Op, Args, #st{on_write_error = Rpt}) ->
|
||||
ok
|
||||
end.
|
||||
|
||||
maybe_store_error(undefined, _, _, _, _, _) ->
|
||||
ok;
|
||||
maybe_store_error(Table, Err, IntTable, put, [_, K, _, _], Time) ->
|
||||
insert_error(Table, IntTable, K, Err, Time);
|
||||
maybe_store_error(Table, Err, IntTable, delete, [_, K, _], Time) ->
|
||||
insert_error(Table, IntTable, K, Err, Time);
|
||||
maybe_store_error(Table, Err, IntTable, write, [_, List, _], Time) ->
|
||||
lists:map(fun
|
||||
({put, K, _}) ->
|
||||
insert_error(Table, IntTable, K, Err, Time);
|
||||
({delete, K}) ->
|
||||
insert_error(Table, IntTable, K, Err, Time)
|
||||
end, List).
|
||||
|
||||
insert_error(Table, {Type, _, _}, K, Err, Time) ->
|
||||
{_, K1} = decode_key(K),
|
||||
ets:insert(Table, {{Type, K1}, Err, Time});
|
||||
insert_error(Table, Type, K, Err, Time) when is_atom(Type) ->
|
||||
ets:insert(Table, {{Type, K}, Err, Time}).
|
||||
|
||||
rpt_fmt([_|T]) ->
|
||||
lists:append(["~p" | [", ~p" || _ <- T]]).
|
||||
|
||||
|
||||
@ -29,7 +29,6 @@ groups() ->
|
||||
error_handling(_Config) ->
|
||||
mnesia_rocksdb_error_handling:run().
|
||||
|
||||
|
||||
init_per_suite(Config) ->
|
||||
Config.
|
||||
|
||||
|
||||
@ -31,6 +31,14 @@ setup() ->
|
||||
|
||||
create_tab(Type, Level, MaintainSz) ->
|
||||
TabName = tab_name(Type, Level, MaintainSz),
|
||||
%% create error store before the table
|
||||
case ets:info(?MODULE) of
|
||||
undefined ->
|
||||
?MODULE = ets:new(?MODULE, [bag, public, named_table]),
|
||||
ok;
|
||||
_ ->
|
||||
ok
|
||||
end,
|
||||
UserProps = user_props(Level, MaintainSz),
|
||||
{atomic, ok} = mnesia:create_table(TabName, [{rdb, [node()]},
|
||||
{user_properties, UserProps}]),
|
||||
@ -43,7 +51,8 @@ tab_name(Type, Level, MaintainSz) ->
|
||||
|
||||
user_props(Level, MaintainSz) ->
|
||||
[{maintain_sz, MaintainSz},
|
||||
{rocksdb_opts, [{on_write_error, Level}]}].
|
||||
{rocksdb_opts, [ {on_write_error, Level}
|
||||
, {on_write_error_store, ?MODULE} ]}].
|
||||
|
||||
start_mnesia() ->
|
||||
mnesia_rocksdb_tlib:start_mnesia(reset),
|
||||
@ -94,7 +103,11 @@ expect_error(Level, Tab) ->
|
||||
ok
|
||||
after 1000 ->
|
||||
error({expected_error, [Level, Tab]})
|
||||
end.
|
||||
|
||||
end,
|
||||
%% Also verify that an error entry has been written into the error store.
|
||||
1 = ets:select_delete(?MODULE, [{{{Tab, '_'}, '_', '_'}, [], [true]}]),
|
||||
ok.
|
||||
|
||||
rpt_tag(fatal ) -> mnesia_fatal;
|
||||
rpt_tag(error ) -> mnesia_error;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user