From c4235be94aed2386698f7b1a894a085e68b6a2a5 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Fri, 15 Jul 2022 14:14:06 +0200 Subject: [PATCH 1/5] Be more lenient about mnesia asking more than once --- src/mnesia_rocksdb_admin.erl | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/mnesia_rocksdb_admin.erl b/src/mnesia_rocksdb_admin.erl index d99ef2a..1bb6421 100644 --- a/src/mnesia_rocksdb_admin.erl +++ b/src/mnesia_rocksdb_admin.erl @@ -510,12 +510,17 @@ intersection(A, B) -> -spec handle_req(alias(), req(), backend(), st()) -> gen_server_reply(). handle_req(Alias, {create_table, Name, Props}, Backend, St) -> - case create_trec(Alias, Name, Props, Backend, St) of - {ok, NewCf} -> - St1 = update_cf(Alias, Name, NewCf, St), - {reply, {ok, NewCf}, St1}; - {error, _} = Error -> - {reply, Error, St} + case find_cf(Alias, Name, Backend, St) of + {ok, TRec} -> + {reply, {ok, TRec}, St}; + error -> + case create_trec(Alias, Name, Props, Backend, St) of + {ok, NewCf} -> + St1 = update_cf(Alias, Name, NewCf, St), + {reply, {ok, NewCf}, St1}; + {error, _} = Error -> + {reply, Error, St} + end end; handle_req(Alias, {load_table, Name, Props}, Backend, St) -> try From 7057f4dcbda46a77bb9c7c756e5bcb5271544e67 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Fri, 22 Jul 2022 15:31:55 +0200 Subject: [PATCH 2/5] Remove rocksdb opts refault, safer index consistency check --- rebar.config.script | 7 ------- src/mnesia_rocksdb.erl | 9 +++------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/rebar.config.script b/rebar.config.script index 413b50d..01c1b67 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,11 +1,4 @@ %% -*- erlang-mode -*- -case os:getenv("ERLANG_ROCKSDB_OPTS") of - false -> - true = os:putenv("ERLANG_ROCKSDB_OPTS", "-DWITH_BUNDLE_LZ4=ON"); - _ -> - %% If manually set, we assume it's throught through - skip -end. case os:getenv("DEBUG") of "true" -> Opts = proplists:get_value(erl_opts, CONFIG, []), diff --git a/src/mnesia_rocksdb.erl b/src/mnesia_rocksdb.erl index 8cb4b6d..bdcfdc0 100644 --- a/src/mnesia_rocksdb.erl +++ b/src/mnesia_rocksdb.erl @@ -346,14 +346,11 @@ semantics(_Alias, index_fun) -> fun index_f/4; semantics(_Alias, _) -> undefined. is_index_consistent(Alias, {Tab, index, PosInfo}) -> - case info(Alias, Tab, {index_consistent, PosInfo}) of - true -> true; - _ -> false - end. + mnesia_rocksdb_admin:read_info(Alias, Tab, {index_consistent, PosInfo}, false). -index_is_consistent(_Alias, {Tab, index, PosInfo}, Bool) +index_is_consistent(Alias, {Tab, index, PosInfo}, Bool) when is_boolean(Bool) -> - mrdb:write_info(Tab, {index_consistent, PosInfo}, Bool). + mnesia_rocksdb_admin:write_info(Alias, Tab, {index_consistent, PosInfo}, Bool). %% PRIVATE FUN From 296abb23bb425831b57a767bfc574ea61604301b Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Fri, 22 Jul 2022 16:33:32 +0200 Subject: [PATCH 3/5] Fix double encoding of info data --- src/mnesia_rocksdb_admin.erl | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/mnesia_rocksdb_admin.erl b/src/mnesia_rocksdb_admin.erl index 1bb6421..5c1cd43 100644 --- a/src/mnesia_rocksdb_admin.erl +++ b/src/mnesia_rocksdb_admin.erl @@ -280,9 +280,12 @@ write_info(Alias, Tab, K, V) -> write_info_(get_ref({admin, Alias}), Tab, K, V). write_info_(Ref, Tab, K, V) -> + write_info_encv(Ref, Tab, K, term_to_binary(V)). + +write_info_encv(Ref, Tab, K, V) -> EncK = mnesia_rocksdb_lib:encode_key({info,Tab,K}, sext), maybe_write_standalone_info(Ref, K, V), - mrdb:rdb_put(Ref, EncK, term_to_binary(V), []). + mrdb:rdb_put(Ref, EncK, V, []). maybe_write_standalone_info(Ref, K, V) -> case Ref of @@ -1228,7 +1231,18 @@ load_info_(Res, I, ARef, Tab) -> DecK = mnesia_rocksdb_lib:decode_key(K), case read_info_(ARef, Tab, DecK, undefined) of undefined -> - write_info_(ARef, Tab, DecK, V); + write_info_encv(ARef, Tab, DecK, V); + <<131,_/binary>> = Value -> + %% Due to a previous bug, info values could be double-encoded with binary_to_term() + try binary_to_term(Value) of + _DecVal -> + %% We haven't been storing erlang-term encoded data as info, + %% so assume this is double-encoded and correct + write_info_encv(ARef, Tab, DecK, Value) + catch + error:_ -> + skip + end; _ -> skip end, From dfc012580073472ff717cba317536bd94b8029b3 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Mon, 25 Jul 2022 15:27:20 +0200 Subject: [PATCH 4/5] More dets-like load/close behavior --- src/mnesia_rocksdb.erl | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/mnesia_rocksdb.erl b/src/mnesia_rocksdb.erl index bdcfdc0..9665279 100644 --- a/src/mnesia_rocksdb.erl +++ b/src/mnesia_rocksdb.erl @@ -162,6 +162,7 @@ , type , status , on_error + , loaders = [] }). -type data_tab() :: atom(). @@ -795,10 +796,10 @@ handle_call({create_table, Tab, Props}, _From, exit:{aborted, Error} -> {reply, {aborted, Error}, St} end; -handle_call({load_table, _LoadReason, Props}, _From, +handle_call({load_table, _LoadReason, Props}, {Pid,_}, #st{alias = Alias, tab = Tab} = St) -> {ok, _Ref} = mnesia_rocksdb_admin:load_table(Alias, Tab, Props), - {reply, ok, St#st{status = active}}; + {reply, ok, St#st{status = active, loaders = [Pid|St#st.loaders]}}; handle_call({write_info, Key, Value}, _From, #st{tab = Tab} = St) -> mrdb:write_info(get_ref(Tab), Key, Value), {reply, ok, St}; @@ -823,9 +824,14 @@ handle_call({delete, Key}, _From, St) -> handle_call({match_delete, Pat}, _From, #st{tab = Tab} = St) -> Res = mrdb:match_delete(get_ref(Tab), Pat), {reply, Res, St}; -handle_call(close_table, _From, #st{alias = Alias, tab = Tab} = St) -> - _ = mnesia_rocksdb_admin:close_table(Alias, Tab), - {reply, ok, St#st{status = undefined}}; +handle_call(close_table, {Pid,_}, #st{alias = Alias, tab = Tab, loaders = Ls} = St) -> + case Ls -- [Pid] of + [] -> + _ = mnesia_rocksdb_admin:close_table(Alias, Tab), + {reply, ok, St#st{status = undefined, loaders = []}}; + Ls1 -> + {reply, ok, St#st{loaders = Ls1}} + end; handle_call(delete_table, _From, #st{alias = Alias, tab = Tab} = St) -> ok = mnesia_rocksdb_admin:delete_table(Alias, Tab), {stop, normal, ok, St#st{status = undefined}}. From b908998e6befed8b9a289d0da985872b9f632751 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Tue, 2 Aug 2022 17:07:53 +0200 Subject: [PATCH 5/5] Check pdict for dumper state at close_table --- src/mnesia_rocksdb.erl | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/mnesia_rocksdb.erl b/src/mnesia_rocksdb.erl index 9665279..e7b1e2f 100644 --- a/src/mnesia_rocksdb.erl +++ b/src/mnesia_rocksdb.erl @@ -162,7 +162,6 @@ , type , status , on_error - , loaders = [] }). -type data_tab() :: atom(). @@ -452,8 +451,13 @@ close_table(Alias, Tab) -> error -> ok; _ -> - ok = mnesia_rocksdb_admin:prep_close(Alias, Tab), - close_table_(Alias, Tab) + case get(mnesia_dumper_dets) of + undefined -> + ok = mnesia_rocksdb_admin:prep_close(Alias, Tab), + close_table_(Alias, Tab); + _ -> + ok + end end. close_table_(Alias, Tab) -> @@ -799,7 +803,7 @@ handle_call({create_table, Tab, Props}, _From, handle_call({load_table, _LoadReason, Props}, {Pid,_}, #st{alias = Alias, tab = Tab} = St) -> {ok, _Ref} = mnesia_rocksdb_admin:load_table(Alias, Tab, Props), - {reply, ok, St#st{status = active, loaders = [Pid|St#st.loaders]}}; + {reply, ok, St#st{status = active}}; handle_call({write_info, Key, Value}, _From, #st{tab = Tab} = St) -> mrdb:write_info(get_ref(Tab), Key, Value), {reply, ok, St}; @@ -824,14 +828,9 @@ handle_call({delete, Key}, _From, St) -> handle_call({match_delete, Pat}, _From, #st{tab = Tab} = St) -> Res = mrdb:match_delete(get_ref(Tab), Pat), {reply, Res, St}; -handle_call(close_table, {Pid,_}, #st{alias = Alias, tab = Tab, loaders = Ls} = St) -> - case Ls -- [Pid] of - [] -> - _ = mnesia_rocksdb_admin:close_table(Alias, Tab), - {reply, ok, St#st{status = undefined, loaders = []}}; - Ls1 -> - {reply, ok, St#st{loaders = Ls1}} - end; +handle_call(close_table, {Pid,_}, #st{alias = Alias, tab = Tab} = St) -> + _ = mnesia_rocksdb_admin:close_table(Alias, Tab), + {reply, ok, St#st{status = undefined}}; handle_call(delete_table, _From, #st{alias = Alias, tab = Tab} = St) -> ok = mnesia_rocksdb_admin:delete_table(Alias, Tab), {stop, normal, ok, St#st{status = undefined}}.