From 0e4382d5f722fd5d6848172d369db3b7237dc1f1 Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Fri, 23 Aug 2024 11:01:46 +0200 Subject: [PATCH 1/5] Use emqx fork of erlang-rocksdb --- rebar.config | 2 +- rebar.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index 4d94262..7f3ee48 100644 --- a/rebar.config +++ b/rebar.config @@ -4,7 +4,7 @@ {deps, [ {sext, "1.8.0"}, - {rocksdb, {git, "https://gitlab.com/barrel-db/erlang-rocksdb.git", {tag,"1.8.0"}}}, + {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb.git", {tag,"1.8.0-emqx-6"}}}, {hut, "1.4.0"} ]}. diff --git a/rebar.lock b/rebar.lock index 3b9e82e..39544d6 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,8 +1,8 @@ {"1.2.0", [{<<"hut">>,{pkg,<<"hut">>,<<"1.4.0">>},0}, {<<"rocksdb">>, - {git,"https://gitlab.com/barrel-db/erlang-rocksdb.git", - {ref,"fced5f637de7991c5948e28414ba3790b0476c4b"}}, + {git,"https://github.com/emqx/erlang-rocksdb.git", + {ref,"1f02f720b91fb53f9535f68548c31280b345d029"}}, 0}, {<<"sext">>,{pkg,<<"sext">>,<<"1.8.0">>},0}]}. [ -- 2.30.2 From 0ae7ecd41db8d2a2f6591988cc89f1e0bd1163ac Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Wed, 9 Oct 2024 13:53:01 +0200 Subject: [PATCH 2/5] Use our patched version of the EMQX fork --- rebar.config | 2 +- rebar.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index 7f3ee48..771b5f8 100644 --- a/rebar.config +++ b/rebar.config @@ -4,7 +4,7 @@ {deps, [ {sext, "1.8.0"}, - {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb.git", {tag,"1.8.0-emqx-6"}}}, + {rocksdb, {git, "https://github.com/aeternity/erlang-rocksdb.git", {ref,"79bab7c6b6"}}}, {hut, "1.4.0"} ]}. diff --git a/rebar.lock b/rebar.lock index 39544d6..ca035dc 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,8 +1,8 @@ {"1.2.0", [{<<"hut">>,{pkg,<<"hut">>,<<"1.4.0">>},0}, {<<"rocksdb">>, - {git,"https://github.com/emqx/erlang-rocksdb.git", - {ref,"1f02f720b91fb53f9535f68548c31280b345d029"}}, + {git,"https://github.com/aeternity/erlang-rocksdb.git", + {ref,"79bab7c6b693bf7016490c28987d0b12a6b0dae9"}}, 0}, {<<"sext">>,{pkg,<<"sext">>,<<"1.8.0">>},0}]}. [ -- 2.30.2 From d42055c7ac9144b39a1cfe91297f245ffbd5c75e Mon Sep 17 00:00:00 2001 From: Richard Carlsson Date: Sun, 13 Oct 2024 14:30:56 +0200 Subject: [PATCH 3/5] Update erlang-rocksdb to use latest patch fixing compilation issues --- rebar.config | 2 +- rebar.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index 771b5f8..5bd294c 100644 --- a/rebar.config +++ b/rebar.config @@ -4,7 +4,7 @@ {deps, [ {sext, "1.8.0"}, - {rocksdb, {git, "https://github.com/aeternity/erlang-rocksdb.git", {ref,"79bab7c6b6"}}}, + {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb.git", {ref,"d695c6e"}}}, {hut, "1.4.0"} ]}. diff --git a/rebar.lock b/rebar.lock index ca035dc..5149a02 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,8 +1,8 @@ {"1.2.0", [{<<"hut">>,{pkg,<<"hut">>,<<"1.4.0">>},0}, {<<"rocksdb">>, - {git,"https://github.com/aeternity/erlang-rocksdb.git", - {ref,"79bab7c6b693bf7016490c28987d0b12a6b0dae9"}}, + {git,"https://github.com/emqx/erlang-rocksdb.git", + {ref,"d695c6ee9dd27bfe492ed4e24c72ad20ab0d770b"}}, 0}, {<<"sext">>,{pkg,<<"sext">>,<<"1.8.0">>},0}]}. [ -- 2.30.2 From 56d78768d91ceaf05d6ff8dc66d7286b5cf1b0a6 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Sun, 23 Mar 2025 21:06:21 +0100 Subject: [PATCH 4/5] Produce stacktraces unless mnesia_compatible --- README.md | 12 ++++++++++++ src/mrdb.erl | 27 ++++++++++++++++++++++++++- test/mnesia_rocksdb_SUITE.erl | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3cfaee3..1ee59b8 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,18 @@ depend on having an up to date size count at all times, you need to maintain it yourself. If you only need the size occasionally, you may traverse the table to count the elements. +When `mrdb` transactions abort, they will return a stacktrace caught +from within the transaction fun, giving much better debugging info. +This is different from how mnesia does it. + +If behavior closer to mnesia's abort returns are needed, say, for backwards +compatibility, this can be controlled by setting the environment variable +`-mnesia_rocksdb mnesia_compatible_aborts true`, or by adding a transaction +option, e.g. `mrdb:activity({tx, #{mnesia_compatible => true}}, fun() ... end)`. +For really performance-critical transactions which may abort often, it might +make a difference to set this option to `true`, since there is a cost involved +in producing stacktraces. + ### Mnesia backend plugins ### diff --git a/src/mrdb.erl b/src/mrdb.erl index fb9d431..9be0a4a 100644 --- a/src/mrdb.erl +++ b/src/mrdb.erl @@ -297,14 +297,34 @@ do_activity(F, Alias, Ctxt) -> end. try_f(F, Ctxt) -> + try_f(mnesia_compatible_aborts(Ctxt), F, Ctxt). + +try_f(false, F, Ctxt) -> try run_f(F, Ctxt) of Res -> commit_and_pop(Res) catch + throw:Something -> + abort_and_pop(throw, Something); + Cat:Err:T -> + %% Without capturing the stacktract here, + %% debugging gets pretty difficult. Incompatible with mnesia, though. + abort_and_pop(Cat, {Err, T}) + end; +try_f(true, F, Ctxt) -> + try run_f(F, Ctxt) of + Res -> + commit_and_pop(Res) + catch + throw:Something -> + abort_and_pop(throw, Something); Cat:Err -> + %% Without capturing the stacktract here, + %% debugging gets pretty difficult abort_and_pop(Cat, Err) end. + run_f(F, Ctxt) -> push_ctxt(Ctxt), F(). @@ -411,7 +431,7 @@ apply_tx_opts(Opts0) when is_map(Opts0) -> check_tx_opts(maps:merge(default_tx_opts(), Opts0)). check_tx_opts(Opts) -> - case maps:without([no_snapshot, retries], Opts) of + case maps:without([no_snapshot, retries, mnesia_compatible], Opts) of Other when map_size(Other) > 0 -> abort({invalid_tx_opts, maps:keys(Other)}); _ -> @@ -523,6 +543,11 @@ re_throw(Cat, Err) -> mnesia_compatible_aborts() -> mnesia_rocksdb_admin:get_cached_env(mnesia_compatible_aborts, false). +mnesia_compatible_aborts(#{activity := #{mnesia_compatible := Bool}}) -> + Bool; +mnesia_compatible_aborts(_) -> + mnesia_compatible_aborts(). + fix_error({aborted, Err}) -> Err; fix_error(Err) -> diff --git a/test/mnesia_rocksdb_SUITE.erl b/test/mnesia_rocksdb_SUITE.erl index d0fa050..7bba6ab 100644 --- a/test/mnesia_rocksdb_SUITE.erl +++ b/test/mnesia_rocksdb_SUITE.erl @@ -269,7 +269,7 @@ mrdb_abort(Config) -> Pre = mrdb:read(tx_abort, a), D0 = get_dict(), TRes = try mrdb:activity( - tx, rdb, + {tx, #{mnesia_compatible => true}}, rdb, fun() -> [{tx_abort, a, N}] = mrdb:read(tx_abort, a), error(abort_here), -- 2.30.2 From 2ca7f36eb182b68a1840545501af528643eaaf18 Mon Sep 17 00:00:00 2001 From: Ulf Wiger Date: Mon, 2 Jun 2025 20:07:05 +0200 Subject: [PATCH 5/5] Consistent support for merge ops --- src/mnesia_rocksdb_admin.erl | 14 ++++++++------ src/mrdb.erl | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/mnesia_rocksdb_admin.erl b/src/mnesia_rocksdb_admin.erl index 3ae2c22..73953e3 100644 --- a/src/mnesia_rocksdb_admin.erl +++ b/src/mnesia_rocksdb_admin.erl @@ -1489,11 +1489,7 @@ open_db_(MP, Alias, Opts, CFs0, CreateIfMissing) -> %% not yet created CFs = cfs(CFs0), file:make_dir(MP), - OpenOpts = [ {create_if_missing, true} - , {create_missing_column_families, true} - , {merge_operator, erlang_merge_operator} - | Opts ], - OpenRes = mnesia_rocksdb_lib:open_rocksdb(MP, OpenOpts, CFs), + OpenRes = rocksdb_open(MP, Opts, CFs), map_cfs(OpenRes, CFs, Alias, Acc0); false -> {error, enoent}; @@ -1504,9 +1500,15 @@ open_db_(MP, Alias, Opts, CFs0, CreateIfMissing) -> map_cfs(rocksdb_open(MP, Opts, CFs1), CFs1, Alias, Acc0) end. +open_opts(Opts) -> + [ {create_if_missing, true} + , {create_missing_column_families, true} + , {merge_operator, erlang_merge_operator} + | Opts ]. + rocksdb_open(MP, Opts, CFs) -> %% rocksdb:open(MP, Opts, CFs), - mnesia_rocksdb_lib:open_rocksdb(MP, Opts, CFs). + mnesia_rocksdb_lib:open_rocksdb(MP, open_opts(Opts), CFs). is_open(Alias, #st{backends = Bs}) -> case maps:find(Alias, Bs) of diff --git a/src/mrdb.erl b/src/mrdb.erl index 9be0a4a..74025a0 100644 --- a/src/mrdb.erl +++ b/src/mrdb.erl @@ -53,6 +53,7 @@ , delete/2 , delete/3 , delete_object/2, delete_object/3 , match_delete/2 + , merge/3 , merge/4 , clear_table/1 , batch_write/2 , batch_write/3 , update_counter/3, update_counter/4 @@ -740,6 +741,21 @@ insert(Tab, Obj0, Opts) -> EncVal = encode_val(Obj, Ref), insert_(Ref, Key, encode_key(Key, Ref), EncVal, Obj, Opts). +merge(Tab, Key, MergeOp) -> + merge(Tab, Key, MergeOp, []). + +merge(Tab, Key, MergeOp, Opts) -> + #{encoding := Enc} = Ref = ensure_ref(Tab), + case Enc of + {_, {value, term}} -> + merge_(Ref, Key, MergeOp, Opts); + _ -> + abort(badarg) + end. + +merge_(Ref, Key, MergeOp, Opts) -> + rdb_merge(Ref, encode_key(Key), MergeOp, Opts). + validate_obj(Obj, #{mode := mnesia}) -> Obj; validate_obj(Obj, #{attr_pos := AP, -- 2.30.2