QPQ-AG/mnesia_rocksdb
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

first version, based on mnesia_eleveldb

Browse Source
This commit is contained in:
Ulf Wiger
2018-02-06 09:43:57 +01:00
commit 6c9f5b565f
30 changed files with 4835 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/Emakefile
+2
View File
@@ -0,0 +1,2 @@
mnesia_rocksdb_xform.
{'*', [{parse_transform, mnesia_rocksdb_xform},debug_info]}.
test/README
+17
View File
@@ -0,0 +1,17 @@
To run the mnesia test suite, replacing disc_only_copies references with
rocksdb_copies:
```
cd $ERL_TOP
make release_tests
cd release/tests/mnesia_test
cp $MNESIA_ROCKSDB/test/mnesia_rocksdb_backend_xform.erl .
cp $MNESIA_ROCKSDB/test/Emakefile .
```
You may use github.com/uwiger/parse_trans, and pretty-print the
debug_info in the transformed test suite modules using the following alias:
```
alias pp='escript $PARSE_TRANS_ROOT/ebin/parse_trans_pp.beam'
```
test/basho_bench_driver_mnesia_rocksdb.erl
+64
View File
@@ -0,0 +1,64 @@
%% -------------------------------------------------------------------
%%
%% basho_bench: Benchmarking Suite
%%
%% Copyright (c) 2009-2010 Basho Techonologies
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% -------------------------------------------------------------------
-module(basho_bench_driver_mnesia_rocksdb).
-export([new/1,
run/4]).
-include("mnesia_rocksdb_basho_bench.hrl").
%% ====================================================================
%% API
%% ====================================================================
new(_Id) ->
Type = basho_bench_config:get(backend, ram_copies),
Tab = basho_bench_config:get(mnesia_table, t),
ok = bootstrap_mnesia(Tab, Type),
{ok, Tab}.
bootstrap_mnesia(Tab, Type) ->
ok = mnesia:create_schema([node()],
[{backend_types,
[{rocksdb_copies, mnesia_rocksdb}]}]),
ok = mnesia:start(),
{atomic,ok} = mnesia:create_table(Tab, [{Type, [node()]}]),
mnesia:wait_for_tables([Tab], 10000).
run(get, KeyGen, _ValueGen, State) ->
Tab = State,
Key = KeyGen(),
case mnesia:dirty_read({Tab, Key}) of
[] ->
{ok, State};
[{_, Key, _}] ->
{ok, State}
end;
run(put, KeyGen, ValueGen, State) ->
Tab = State,
ok = mnesia:dirty_write({Tab, KeyGen(), ValueGen()}),
{ok, State};
run(delete, KeyGen, _ValueGen, State) ->
Tab = State,
ok = mnesia:dirty_delete({Tab, KeyGen()}),
{ok, State}.
test/mnesia_rocksdb_basho_bench.hrl
+15
View File
@@ -0,0 +1,15 @@
-define(FAIL_MSG(Str, Args), ?ERROR(Str, Args), basho_bench_app:halt_or_kill()).
-define(STD_ERR(Str, Args), io:format(standard_error, Str, Args)).
-define(CONSOLE(Str, Args), lager:info(Str, Args)).
-define(DEBUG(Str, Args), lager:debug(Str, Args)).
-define(INFO(Str, Args), lager:info(Str, Args)).
-define(WARN(Str, Args), lager:warning(Str, Args)).
-define(ERROR(Str, Args), lager:error(Str, Args)).
-define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))).
-define(VAL_GEN_BLOB_CFG, value_generator_blob_file).
-define(VAL_GEN_SRC_SIZE, value_generator_source_size).
test/mnesia_rocksdb_bench_disc_only.config
+18
View File
@@ -0,0 +1,18 @@
{mode, max}.
{duration, 10}.
{concurrent, 1}.
{driver, basho_bench_driver_mnesia_rocksdb}.
{key_generator, {int_to_bin,{uniform_int, 5000000}}}.
{value_generator, {fixed_bin, 10000}}.
{operations, [{get, 2}, {put, 2}, {delete, 1}]}.
{code_paths, []}.
{mnesia_table, doc}.
{backend, disc_only_copies}.
test/mnesia_rocksdb_bench_rocksdb_copies.config
+19
View File
@@ -0,0 +1,19 @@
{mode, max}.
{duration, 10}.
{concurrent, 1}.
{driver, basho_bench_driver_mnesia_rocksdb}.
{key_generator, {int_to_bin,{uniform_int, 5000000}}}.
{value_generator, {fixed_bin, 10000}}.
{operations, [{get, 2}, {put, 2}, {delete, 1}]}.
{code_paths, ["/Users/uwiger/git/rocksdb",
"/Users/uwiger/git/mnesia_rocksdb"]}.
{mnesia_table, rdb}.
{backend, rocksdb_copies}.
test/mnesia_rocksdb_chg_tbl_copy.erl
+131
View File
@@ -0,0 +1,131 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
%% @doc Run through all combinations of change_table_copy_type
%% @author Ulf Wiger <ulf.wiger@feuerlabs.com>
-module(mnesia_rocksdb_chg_tbl_copy).
%% This module implements a test (to be run manually) for iterating through
%% all table copy types on a mnesia table.
-export([full/0,
run/0,
run/1]).
-export([trace/0]).
full() ->
Perms = perms(copies()),
Res = [run(P) || P <- Perms],
Res = [ok || _ <- Perms].
run() ->
run([rdb,disc_copies,rdb,ram_copies,disc_only_copies,rdb]).
%% run([rdb,disc_only_copies]).
%% run([rdb,ram_copies]).
perms([]) ->
[[]];
perms(L) -> [[H|T] || H <- L, T <- perms(L--[H])].
copies() ->
[rdb,ram_copies,disc_copies,disc_only_copies].
run([T|Types]) ->
mnesia:stop(),
start_mnesia(),
ok = create_tab(T),
ok = change_type(Types, T).
create_tab(Type) ->
{atomic,ok} = mnesia:create_table(
t, [{Type, [node()]},
{attributes, [k,v]},
{index, [v]}]),
fill_tab(),
check_tab(),
ok.
change_type([To|Types], From) ->
io:fwrite("changing from ~p to ~p~n", [From, To]),
{atomic, ok} = mnesia:change_table_copy_type(t, node(), To),
ok = check_tab(),
io:fwrite("...ok~n", []),
change_type(Types, To);
change_type([], _) ->
ok.
fill_tab() ->
Res = [mnesia:dirty_write({t,K,V}) || {t,K,V} <- l()],
Res = [ok || _ <- Res],
ok.
l() -> [{t,a,1},
{t,b,2},
{t,c,3},
{t,d,4}].
check_tab() ->
L = l(),
L = lists:append([mnesia:dirty_read({t,K}) || K <- [a,b,c,d]]),
L = lists:append([mnesia:dirty_index_read(t,V,v) ||
V <- [1,2,3,4]]),
ok.
start_mnesia() -> mnesia_rocksdb_tlib:start_mnesia(reset).
trace() ->
dbg:tracer(),
[tp(M) || M <- mods()],
dbg:p(all,[c]),
try run()
after
[ctp(M) || M <- mods()],
dbg:stop()
end.
tp({l,M} ) -> dbg:tpl(M,x);
tp({g,M} ) -> dbg:tp(M,x);
tp({l,M,F}) -> dbg:tpl(M,F,x);
tp({g,M,F}) -> dbg:tp(M,F,x).
ctp({l,M} ) -> dbg:ctpl(M);
ctp({g,M} ) -> dbg:ctp(M);
ctp({l,M,F}) -> dbg:ctpl(M,F);
ctp({g,M,F}) -> dbg:ctp(M,F).
mods() ->
[
%% {l, mnesia_index},
%% {l, mnesia_lib, semantics}].
%% {g,mnesia_monitor},
%% {l,mnesia_dumper},
%% {g,mnesia_loader},
%% {g,mnesia_checkpoint},
%% {g,mnesia_lib},
{l,mnesia_schema,expand_index_attrs},
{l,mnesia_schema,list2cs},
{g,mnesia_schema,new_cs},
{g,mnesia_schema,make_change_table_copy_type},
{g,mnesia_schema,make_create_table},
{g,mnesia_lib,semantics},
{l,mnesia_dumper},
{g,mnesia_lib,exists},
{g,mnesia},
{l,mnesia_schema,intersect_types},
{g,ets,new}].
test/mnesia_rocksdb_conv_bigtab.erl
+59
View File
@@ -0,0 +1,59 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_conv_bigtab).
-export([init/0, mktab/2, run/1]).
-record(t, {k, i, v}).
run(Sz) ->
mnesia:stop(),
init(),
mktab(disc_copies, Sz),
mnesia:change_table_copy_type(t, node(), rdb).
init() ->
mnesia_rocksdb_tlib:start_mnesia(reset).
mktab(Backend, Sz) ->
mnesia_rocksdb_tlib:create_table(Backend, [k, i, v], [i]),
fill_table(Sz).
fill_table(Sz) when is_integer(Sz), Sz > 0 ->
fill_table(1, Sz).
fill_table(N, Max) when N =< Max ->
mnesia:dirty_write(#t{k = N, i = N, v = val()}),
fill_table(N+1, Max);
fill_table(N, _) when is_integer(N) ->
ok.
val() ->
{1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0,
1,2,3,4,5,6,7,8,9,0}.
test/mnesia_rocksdb_fallback.erl
+98
View File
@@ -0,0 +1,98 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_fallback).
-export([run/0]).
-define(m(A,B), fun() -> L = ?LINE,
case {A,B} of
{__X, __X} ->
B;
Other ->
error({badmatch, [Other,
{line, L}]})
end
end()).
run() ->
cleanup(),
mnesia_rocksdb_tlib:start_mnesia(reset),
mnesia_rocksdb_tlib:create_table(rdb),
ok = mnesia:backup("bup0.BUP"),
[mnesia:dirty_write({t,K,V}) || {K,V} <- [{a,1},
{b,2},
{c,3}]],
ok = mnesia:backup("bup1.BUP"),
[mnesia:dirty_write({t,K,V}) || {K,V} <- [{d,4},
{e,5},
{f,6}]],
ok = mnesia:backup("bup2.BUP"),
io:fwrite("*****************************************~n", []),
load_backup("bup0.BUP"),
?m([], mnesia:dirty_match_object(t, {t,'_','_'})),
?m([], mnesia:dirty_index_read(t,2,v)),
io:fwrite("*****************************************~n", []),
load_backup("bup1.BUP"),
?m([{t,a,1},{t,b,2},{t,c,3}], mnesia:dirty_match_object(t, {t,'_','_'})),
?m([{t,b,2}], mnesia:dirty_index_read(t,2,v)),
io:fwrite("*****************************************~n", []),
load_backup("bup2.BUP"),
?m([{t,a,1},{t,b,2},{t,c,3},
{t,d,4},{t,e,5},{t,f,6}], mnesia:dirty_match_object(t, {t,'_','_'})),
?m([{t,b,2}], mnesia:dirty_index_read(t,2,v)),
?m([{t,e,5}], mnesia:dirty_index_read(t,5,v)),
ok.
load_backup(BUP) ->
mnesia_rocksdb_tlib:trace(
fun() ->
io:fwrite("loading backup ~s~n", [BUP]),
ok = mnesia:install_fallback(BUP),
io:fwrite("stopping~n", []),
mnesia:stop(),
timer:sleep(3000),
io:fwrite("starting~n", []),
mnesia:start(),
WaitRes = mnesia:wait_for_tables([t], 5000),
io:fwrite("WaitRes = ~p~n", [WaitRes])
end,
mods(0)
).
cleanup() ->
os:cmd("rm *.BUP").
mods(0) ->
[];
mods(1) ->
[
{l, mnesia_rocksdb},
{g, rocksdb}
];
mods(2) ->
[
%% {l, mnesia_monitor},
{g, mnesia_rocksdb},
{l, mnesia_bup},
{g, mnesia_lib},
{g, mnesia_schema},
%% {g, mnesia_loader},
{g, mnesia_index},
{l, mnesia_tm}
].
test/mnesia_rocksdb_indexes.erl
+174
View File
@@ -0,0 +1,174 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_indexes).
-export([run/0,
r1/0]).
run() ->
mnesia:stop(),
ok = mnesia_rocksdb_tlib:start_mnesia(reset),
test(1, ram_copies, r1),
test(1, disc_copies, d1),
fail(test, [1, disc_only_copies, do1]), % doesn't support ordered
test(2, disc_only_copies, do1),
fail(test, [1, rdb, l1]), % doesn't support bag
test(3, rdb, l1),
add_del_indexes(),
{atomic,ok} = mnesia_schema:add_index_plugin(
{pfx},mnesia_rocksdb, ix_prefixes),
test_index_plugin(pr1, ram_copies, ordered),
test_index_plugin(pr2, ram_copies, bag),
test_index_plugin(pd1, disc_copies, ordered),
fail(test_index_plugin, [pd2, disc_only_copies, ordered]),
test_index_plugin(pd2, disc_copies, bag),
test_index_plugin(pl2, rdb, ordered),
test_index_plugin_mgmt(),
ok.
r1() ->
mnesia:stop(),
ok = mnesia_rocksdb_tlib:start_mnesia(reset),
{atomic,ok} = mnesia_schema:add_index_plugin(
{pfx},mnesia_rocksdb, ix_prefixes),
dbg:tracer(),
dbg:tpl(mnesia_schema,x),
dbg:tpl(mnesia_index,x),
dbg:p(all,[c]),
test_index_plugin(pd2, disc_only_copies, ordered).
fail(F, Args) ->
try apply(?MODULE, F, Args),
error(should_fail)
catch
error:_ ->
io:fwrite("apply(~p, ~p, ~p) -> fails as expected~n",
[?MODULE, F, Args])
end.
test(N, Type, T) ->
{atomic, ok} = mnesia:create_table(T, [{Type,[node()]},
{attributes,[k,a,b,c]},
{index, indexes(N)}]),
ok = test_index(N, T).
add_del_indexes() ->
{atomic, ok} = mnesia:del_table_index(r1, a),
{aborted, _} = mnesia:del_table_index(r1, a),
{atomic, ok} = mnesia:add_table_index(r1, a),
{aborted, _} = mnesia:add_table_index(r1, a),
{atomic, ok} = mnesia:del_table_index(d1, a),
{atomic, ok} = mnesia:add_table_index(d1, a),
{atomic, ok} = mnesia:del_table_index(do1, a),
{atomic, ok} = mnesia:add_table_index(do1, a),
{atomic, ok} = mnesia:del_table_index(l1, a),
{atomic, ok} = mnesia:add_table_index(l1, a),
io:fwrite("add_del_indexes() -> ok~n", []).
test_index_plugin(Tab, Type, IxType) ->
{atomic, ok} = mnesia:create_table(Tab, [{Type, [node()]},
{index, [{{pfx}, IxType}]}]),
mnesia:dirty_write({Tab, "foobar", "sentence"}),
mnesia:dirty_write({Tab, "yellow", "sensor"}),
mnesia:dirty_write({Tab, "truth", "white"}),
mnesia:dirty_write({Tab, "fulcrum", "white"}),
Res1 = [{Tab, "foobar", "sentence"}, {Tab, "yellow", "sensor"}],
Res2 = [{Tab, "fulcrum", "white"}, {Tab, "truth", "white"}],
if IxType == bag ->
Res1 = lists:sort(mnesia:dirty_index_read(Tab,<<"sen">>, {pfx})),
Res2 = lists:sort(mnesia:dirty_index_read(Tab,<<"whi">>, {pfx})),
[{Tab,"foobar","sentence"}] = mnesia:dirty_index_read(
Tab, <<"foo">>, {pfx});
IxType == ordered ->
Res1 = lists:sort(mnesia:dirty_index_read(Tab,<<"sen">>, {pfx})),
Res2 = lists:sort(mnesia:dirty_index_read(Tab,<<"whi">>, {pfx})),
[{Tab,"foobar","sentence"}] = mnesia:dirty_index_read(
Tab, <<"foo">>, {pfx})
end,
io:fwrite("test_index_plugin(~p, ~p, ~p) -> ok~n", [Tab,Type,IxType]).
test_index_plugin_mgmt() ->
{aborted,_} = mnesia:create_table(x, [{index,[{unknown}]}]),
{aborted,_} = mnesia:create_table(x, [{index,[{{unknown},bag}]}]),
{aborted,_} = mnesia:create_table(x, [{index,[{{unknown},ordered}]}]),
{atomic,ok} = mnesia_schema:add_index_plugin(
{t}, mnesia_rocksdb,ix_prefixes),
{atomic,ok} = mnesia_schema:delete_index_plugin({t}),
{aborted,{bad_type,x,_}} =
mnesia:create_table(x, [{index,[{{t},ordered}]}]),
%% re-add plugin
{atomic,ok} = mnesia_schema:add_index_plugin(
{t}, mnesia_rocksdb,ix_prefixes),
{atomic,ok} =
mnesia:create_table(x, [{index,[{{t},ordered}]}]),
{aborted,{plugin_in_use,{t}}} =
mnesia_schema:delete_index_plugin({t}).
test_index(1, T) ->
L2 = [{T,K,x,y,z} || K <- lists:seq(4,6)],
L1 = [{T,K,a,b,c} || K <- lists:seq(1,3)],
true = lists:all(fun(X) -> X == ok end,
[mnesia:dirty_write(Obj) || Obj <- L1 ++ L2]),
L1 = lists:sort(mnesia:dirty_index_read(T,a,a)),
L1 = lists:sort(mnesia:dirty_index_read(T,a,3)),
L1 = mnesia:dirty_index_read(T,b,b),
L1 = lists:sort(mnesia:dirty_index_read(T,c,c)),
L2 = lists:sort(mnesia:dirty_index_read(T,x,a)),
L2 = lists:sort(mnesia:dirty_index_read(T,x,3)),
L2 = mnesia:dirty_index_read(T,y,b),
L2 = lists:sort(mnesia:dirty_index_read(T,z,c)),
io:fwrite("test_index(1, ~p) -> ok~n", [T]),
ok;
test_index(2, T) ->
L1 = [{T,K,a,b,c} || K <- lists:seq(1,3)],
L2 = [{T,K,x,y,z} || K <- lists:seq(4,6)],
true = lists:all(fun(X) -> X == ok end,
[mnesia:dirty_write(Obj) || Obj <- L1 ++ L2]),
L1 = lists:sort(mnesia:dirty_index_read(T,a,a)),
L1 = lists:sort(mnesia:dirty_index_read(T,a,3)),
L1 = lists:sort(mnesia:dirty_index_read(T,b,b)),
L1 = lists:sort(mnesia:dirty_index_read(T,c,c)),
L2 = lists:sort(mnesia:dirty_index_read(T,x,a)),
L2 = lists:sort(mnesia:dirty_index_read(T,x,3)),
L2 = lists:sort(mnesia:dirty_index_read(T,y,b)),
L2 = lists:sort(mnesia:dirty_index_read(T,z,c)),
io:fwrite("test_index(1, ~p) -> ok~n", [T]),
ok;
test_index(3, T) ->
L2 = [{T,K,x,y,z} || K <- lists:seq(4,6)],
L1 = [{T,K,a,b,c} || K <- lists:seq(1,3)],
true = lists:all(fun(X) -> X == ok end,
[mnesia:dirty_write(Obj) || Obj <- L1 ++ L2]),
L1 = mnesia:dirty_index_read(T,a,a),
L1 = mnesia:dirty_index_read(T,a,3),
L1 = mnesia:dirty_index_read(T,b,b),
L1 = mnesia:dirty_index_read(T,c,c),
L2 = mnesia:dirty_index_read(T,x,a),
L2 = mnesia:dirty_index_read(T,x,3),
L2 = mnesia:dirty_index_read(T,y,b),
L2 = mnesia:dirty_index_read(T,z,c),
io:fwrite("test_index(1, ~p) -> ok~n", [T]),
ok.
indexes(1) ->
[a,{b,ordered},{c,bag}];
indexes(2) ->
[a,b,{c,bag}];
indexes(3) ->
[a,{b,ordered},{c,ordered}].
test/mnesia_rocksdb_proper_semantics_test.erl
+161
View File
@@ -0,0 +1,161 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
%% @doc Verify dirty vs transaction semantics against rocksdb mnesia backend
%% @author Ulf Wiger <ulf.wiger@feuerlabs.com>
-module(mnesia_rocksdb_proper_semantics_test).
%% This module uses the proper_statem pattern to generate random
%% sequences of commands, mixing dirty and transaction operations
%% (including dirty ops from within transactions). Each sequence is run
%% against a disc_copies table and a rocksdb_copies table, after
%% which the result of each operation in the sequence is compared between
%% the two runs. The postcondition is that every command in every sequence
%% should yield the same value against both backends.
-export([test/1,
prop_seq/0]).
%% statem callbacks
-export([initial_state/0,
command/1,
precondition/2,
postcondition/3,
next_state/3]).
%% command callbacks
-export([activity/2]).
-include_lib("proper/include/proper.hrl").
-include_lib("eunit/include/eunit.hrl").
-record(st, {}).
-define(KEYS, [a,b,c]).
basic_test_() ->
{timeout, 60000, [fun() -> test(100) end]}.
test(N) ->
setup_mnesia(),
true = proper:quickcheck(?MODULE:prop_seq(), N),
ok.
prop_seq() ->
?FORALL(Cmds, proper_statem:commands(?MODULE),
begin
setup(),
{H, S, Res} =
proper_statem:run_commands(?MODULE, Cmds),
cleanup(),
?WHENFAIL(
io:fwrite("History: ~w~n"
"State : ~w~n"
"Result : ~w~n", [H, S, Res]),
proper:aggregate(
proper_statem:command_names(Cmds), Res =:= ok))
end).
%% Note that this requires the rocksdb application to be in the path,
%% and obviously an OTP patched with the backend plugin behavior.
setup_mnesia() ->
stopped = mnesia:stop(),
ok = mnesia:delete_schema([node()]),
ok = mnesia:create_schema([node()]),
ok = mnesia:start(),
{ok, rocksdb_copies} = mnesia_rocksdb:register().
setup() ->
{atomic,ok} = mnesia:create_table(d, [{disc_copies, [node()]},
{record_name, x}]),
{atomic,ok} = mnesia:create_table(l, [{rocksdb_copies, [node()]},
{record_name, x}]),
ok = mnesia:wait_for_tables([d, l], 30000),
ok.
cleanup() ->
{atomic, ok} = mnesia:delete_table(d),
{atomic, ok} = mnesia:delete_table(l),
ok.
initial_state() ->
#st{}.
command(#st{}) ->
?LET(Type, type(),
{call, ?MODULE, activity, [Type, sequence()]}).
type() ->
proper_types:oneof([async_dirty, transaction]).
precondition(_, _) ->
true.
postcondition(_, {call,?MODULE,activity,_}, {A, B}) ->
A == B;
postcondition(_, _, _) ->
false.
next_state(St, _, _) ->
St.
sequence() ->
proper_types:list(db_cmd()).
db_cmd() ->
?LET(Type, type(),
proper_types:oneof([{Type, read, key()},
{Type, write, key(), value()},
{Type, delete, key()}])).
key() ->
proper_types:oneof([a,b,c]).
value() ->
proper_types:oneof([1,2,3]).
activity(Type, Seq) ->
{mnesia:activity(Type, fun() ->
apply_seq(Type, d, Seq)
end),
mnesia:activity(Type, fun() ->
apply_seq(Type, l, Seq)
end)}.
apply_seq(Type, Tab, Seq) ->
apply_seq(Type, Tab, Seq, []).
apply_seq(transaction=X, Tab, [H|T], Acc) ->
Res = case H of
{X,read, K} -> mnesia:read(Tab, K, read);
{_,read, K} -> mnesia:dirty_read(Tab,K);
{X,write,K,V} -> mnesia:write(Tab, {x, K, V}, write);
{_,write,K,V} -> mnesia:dirty_write(Tab, {x,K,V});
{X,delete,K} -> mnesia:delete(Tab, K, write);
{_,delete,K} -> mnesia:dirty_delete(Tab,K)
end,
apply_seq(X, Tab, T, [Res|Acc]);
apply_seq(X, Tab, [H|T], Acc) ->
Res = case H of
{_,read, K} -> mnesia:read(Tab, K, read);
{_,write,K,V} -> mnesia:write(Tab, {x, K, V}, write);
{_,delete,K} -> mnesia:delete(Tab, K, write)
end,
apply_seq(X, Tab, T, [Res|Acc]);
apply_seq(_, _, [], Acc) ->
lists:reverse(Acc).
test/mnesia_rocksdb_size_info.erl
+84
View File
@@ -0,0 +1,84 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_size_info).
-export([run/0]).
-define(m(A, B), (fun(L) -> {L,A} = {L,B} end)(?LINE)).
run() ->
initialize_mnesia(),
test_set(),
test_bag().
initialize_mnesia() ->
mnesia:stop(),
mnesia:delete_schema([node()]),
mnesia:create_schema([node()], [{backend_types,
[{rocksdb_copies, mnesia_rocksdb}]}]),
mnesia:start(),
{atomic,ok} = mnesia:create_table(s, [{type, set},
{record_name, x},
{rocksdb_copies, [node()]}]),
{atomic,ok} = mnesia:create_table(b, [{type, bag},
{record_name, x},
{rocksdb_copies, [node()]}]),
ok.
test_set() ->
?m(0, mnesia:table_info(s, size)),
?m(1, w(s, 1, a)),
?m(1, w(s, 1, b)),
?m(2, w(s, 2, c)),
?m(3, w(s, 3, d)),
?m(2, d(s, 3)),
mnesia:stop(),
mnesia:start(),
await(s),
?m(2, mnesia:table_info(s, size)).
test_bag() ->
?m(0, mnesia:table_info(b, size)),
?m(1, w(b, 1, a)),
?m(2, w(b, 1, b)),
?m(3, w(b, 2, a)),
?m(4, w(b, 2, d)),
?m(5, w(b, 2, c)),
?m(4, do(b, 2, c)),
?m(2, d(b, 2)),
mnesia:stop(),
mnesia:start(),
await(b),
?m(2, mnesia:table_info(b, size)).
w(T, K, V) ->
ok = mnesia:dirty_write(T, {x, K, V}),
mnesia:table_info(T, size).
d(T, K) ->
mnesia:dirty_delete({T, K}),
mnesia:table_info(T, size).
do(T, K, V) ->
mnesia:dirty_delete_object(T, {x, K, V}),
mnesia:table_info(T, size).
await(T) ->
?m(ok, mnesia:wait_for_tables([T], 10000)).
test/mnesia_rocksdb_tlib.erl
+66
View File
@@ -0,0 +1,66 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_tlib).
-export([start_mnesia/0,
start_mnesia/1,
create_table/1,
create_table/3,
trace/2]).
start_mnesia() ->
start_mnesia(false).
start_mnesia(Mode) ->
if Mode==reset ->
mnesia:delete_schema([node()]),
mnesia:create_schema([node()],
[{backend_types,
[{rdb,mnesia_rocksdb}]}]);
true -> ok
end,
mnesia:start().
create_table(Backend) ->
create_table(Backend, [k,v], [v]).
create_table(Backend, Attrs, Indexes) ->
mnesia:create_table(t, [{index,Indexes}, {attributes,Attrs},
{Backend, [node()]}]).
trace(F, Ms) ->
dbg:tracer(),
[tp(M) || M <- Ms],
dbg:p(all,[c]),
try F()
after
[ctp(M) || M <- Ms],
dbg:stop()
end.
tp({l,M} ) -> dbg:tpl(M,x);
tp({g,M} ) -> dbg:tp(M,x);
tp({l,M,F}) -> dbg:tpl(M,F,x);
tp({g,M,F}) -> dbg:tp(M,F,x).
ctp({l,M} ) -> dbg:ctpl(M);
ctp({g,M} ) -> dbg:ctp(M);
ctp({l,M,F}) -> dbg:ctpl(M,F);
ctp({g,M,F}) -> dbg:ctp(M,F).
test/mnesia_rocksdb_xform.erl
+334
View File
@@ -0,0 +1,334 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
%% This module is used to test backend plugin extensions to the mnesia
%% backend. It also indirectly tests the mnesia backend plugin
%% extension machinery
%%
%% Usage: mnesia_ext_rocksdb_test:recompile(Extension).
%% Usage: mnesia_ext_rocksdb_test:recompile().
%% This command is executed in the release/tests/test_server directory
%% before running the normal tests. The command patches the test code,
%% via a parse_transform, to replace disc_only_copies with the Alias.
-module(mnesia_rocksdb_xform).
-author("roland.karlsson@erlang-solutions.com").
-author("ulf.wiger@klarna.com").
%% EXPORTS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Exporting API
-export([recompile/0, recompile/1]).
%% Exporting parse_transform callback
-export([parse_transform/2]).
%% Exporting replacement for mnesia:create_table/2
-export([create_table/1, create_table/2, rpc/4]).
%% API %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Recompiling the test code, replacing disc_only_copies with
%% Extension.
recompile() ->
[{Module,Alias}|_] = extensions(),
recompile(Module, Alias).
recompile(MorA) ->
case { lists:keyfind(MorA, 1, extensions()),
lists:keyfind(MorA, 2, extensions())
} of
{{Module,Alias}, _} ->
recompile(Module, Alias);
{false, {Module,Alias}} ->
recompile(Module, Alias);
{false,false} ->
{error, cannot_find_module_or_alias}
end.
recompile(Module, Alias) ->
io:format("recompile(~p,~p)~n",[Module, Alias]),
put_ext(module, Module),
put_ext(alias, Alias),
Modules = [ begin {M,_} = lists:split(length(F)-4, F),
list_to_atom(M) end ||
F <- begin {ok,L} = file:list_dir("."), L end,
lists:suffix(".erl", F),
F=/= atom_to_list(?MODULE) ++ ".erl" ],
io:format("Modules = ~p~n",[Modules]),
lists:foreach(fun(M) ->
c:c(M, [{parse_transform, ?MODULE}])
end, Modules).
%% TEST REPLACEMENT CALLBACKS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% replacement for mnesia:create_table that ensures that
create_table(Name, Parameters) ->
create_table([{name,Name} | Parameters]).
create_table(Parameters) when is_list(Parameters) ->
case lists:keymember(rocksdb_copies, 1, Parameters) of
true ->
%% case lists:member({type, bag}, Parameters) of
%% true ->
%% ct:comment("ERROR: Contains rocksdb table with bag"),
%% {aborted, {rocksdb_does_not_support_bag, Parameters}};
%% false ->
ct:comment("INFO: Contains rocksdb table"),
io:format("INFO: create_table(~p)~n", [Parameters]),
mnesia:create_table(Parameters);
%% end;
false ->
mnesia:create_table(Parameters)
end;
create_table(Param) ->
%% Probably bad input, e.g. from mnesia_evil_coverage_SUITE.erl
mnesia:create_table(Param).
rpc(N, mnesia, start, [Opts]) ->
case lists:keymember(schema, 1, Opts) of
true -> rpc:call(N, mnesia, call, [Opts]);
false ->
Opts1 = [{schema, [{backend_types, backends()}]}|Opts],
rpc:call(N, mnesia, start, [Opts1])
end;
rpc(N, M, F, A) ->
rpc:call(N, M, F, A).
%% PARSE_TRANSFORM CALLBACK %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% The callback for c:c(Module, [{parse_transform,?MODULE}])
parse_transform(Forms, _Options) ->
plain_transform(fun do_transform/1, Forms).
do_transform({'attribute', _, module, Module}) ->
io:format("~n~nMODULE: ~p~n", [Module]),
continue;
do_transform({'atom', Line, disc_only_copies}) ->
io:format("replacing disc_only_copies with ~p~n", [get_ext(alias)]),
{'atom', Line, get_ext(alias)};
do_transform(Form = { call, L1,
{ remote, L2,
{atom, L3, mnesia},
{atom, L4, create_table}},
Arguments}) ->
NewForm = { call, L1,
{ remote, L2,
{atom, L3, ?MODULE},
{atom, L4, create_table}},
plain_transform(fun do_transform/1, Arguments)},
io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]),
NewForm;
do_transform(Form = { call, L1,
{ remote, L2,
{atom, L3, rpc},
{atom, L4, call}},
[{var, _, _} = N, {atom, _, mnesia} = Mnesia,
{atom, _, start} = Start, Args]}) ->
NewForm = { call, L1, { remote, L2,
{atom, L3, ?MODULE},
{atom, L4, rpc}},
[N, Mnesia, Start, Args]},
io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]),
NewForm;
do_transform(Form = { call, L1,
{ remote, L2,
{atom, L3, mnesia},
{atom, L4, create_schema}},
[Nodes]}) ->
P = element(2, Nodes),
NewForm = { call, L1,
{ remote, L2,
{atom, L3, mnesia},
{atom, L4, create_schema}},
[Nodes, erl_parse:abstract([{backend_types, backends()}], P)]},
io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]),
NewForm;
do_transform(Form = { call, L1,
{ remote, L2,
{atom, L3, mnesia},
{atom, L4, start}},
[]}) ->
NewForm = { call, L1,
{ remote, L2,
{atom, L3, mnesia},
{atom, L4, start}},
[erl_parse:abstract(
[{schema, [{backend_types, backends()}]}], L4)]},
io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]),
NewForm;
do_transform(Form = { call, L1,
{ remote, L2,
{atom, L3, mnesia},
{atom, L4, start}},
[Opts]}) ->
P = element(2, Opts),
NewForm = { call, L1,
{ remote, L2,
{atom, L3, mnesia},
{atom, L4, start}},
[{cons, P,
erl_parse:abstract(
{schema, [{backend_types, backends()}]}, L4), Opts}]},
io:format("~nConvert Form:~n~s~n~s~n", [pp_form(Form), pp_form(NewForm)]),
NewForm;
%% 1354:unsupp_user_props(doc) ->
%% 1355: ["Simple test of adding user props in a schema_transaction"];
%% 1356:unsupp_user_props(suite) -> [];
%% 1357:unsupp_user_props(Config) when is_list(Config) ->
do_transform(Form = { function, L1, F, 1, [C1, C2, C3] })
when F == unsupp_user_props ->
L3 = element(2, C3),
NewForm = { function, L1, F, 1,
[C1, C2, {clause, L3, [{var, L3, '_'}], [],
[{tuple, L3, [{atom, L3, skip},
erl_parse:abstract(
"Skipped for rocksdb test", L3)]}
]} ] },
io:format("~nConvert Form:"
"~n=============~n~s"
"==== To: ====~n~s"
"=============~n",
[cut(20, pp_form(Form)), cut(20, pp_form(NewForm))]),
NewForm;
do_transform(Form = { function, L1, F, 1, [C1, C2] })
when F == storage_options ->
L2 = element(2, C2),
NewForm = { function, L1, F, 1,
[C1, {clause, L2, [{var, L2, '_'}], [],
[{tuple, L2, [{atom, L2, skip},
erl_parse:abstract(
"Skipped for rocksdb test", L2)]}
]} ] },
io:format("~nConvert Form:"
"~n=============~n~s"
"==== To: ====~n~s"
"=============~n",
[cut(20, pp_form(Form)), cut(20, pp_form(NewForm))]),
NewForm;
do_transform(_Form) ->
continue.
pp_form(F) when element(1,F) == attribute; element(1,F) == function ->
erl_pp:form(F);
pp_form(F) ->
erl_pp:expr(F).
cut(Lines, S) ->
case re:split(S, "\\v", [{return,list}]) of
Lns when length(Lns) =< Lines ->
S;
Lns ->
lists:flatten(
add_lf(lists:sublist(Lns, 1, Lines) ++ ["...\n"]))
end.
add_lf([H|T]) ->
[H | ["\n" ++ L || L <- T]].
%% INTERNAL %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% A trick for doing parse transforms easier
plain_transform(Fun, Forms) when is_function(Fun, 1), is_list(Forms) ->
plain_transform1(Fun, Forms).
plain_transform1(_, []) ->
[];
plain_transform1(Fun, [F|Fs]) when is_atom(element(1,F)) ->
case Fun(F) of
continue ->
[list_to_tuple(plain_transform1(Fun, tuple_to_list(F))) |
plain_transform1(Fun, Fs)];
{done, NewF} ->
[NewF | Fs];
{error, Reason} ->
io:format("Error: ~p (~p)~n", [F,Reason]);
NewF when is_tuple(NewF) ->
[NewF | plain_transform1(Fun, Fs)]
end;
plain_transform1(Fun, [L|Fs]) when is_list(L) ->
[plain_transform1(Fun, L) | plain_transform1(Fun, Fs)];
plain_transform1(Fun, [F|Fs]) ->
[F | plain_transform1(Fun, Fs)];
plain_transform1(_, F) ->
F.
%% Existing extensions.
%% NOTE: The first is default.
extensions() ->
[ {mnesia_rocksdb, rocksdb_copies}
].
%% {mnesia_ext_filesystem, fs_copies},
%% {mnesia_ext_filesystem, fstab_copies},
%% {mnesia_ext_filesystem, raw_fs_copies}
%% ].
backends() ->
[{T,M} || {M,T} <- extensions()].
%% Process global storage
put_ext(Key, Value) ->
ets:insert(global_storage(), {Key, Value}).
global_storage() ->
case whereis(?MODULE) of
undefined ->
Me = self(),
P = spawn(fun() ->
T = ets:new(?MODULE, [public,named_table]),
init_ext(T),
register(?MODULE, self()),
Me ! {self(), done},
wait()
end),
receive {P, done} ->
ok
end;
_ ->
ok
end,
?MODULE.
init_ext(T) ->
[{Mod,Alias}|_] = extensions(),
ets:insert(T, {alias, Alias}),
ets:insert(T, {module, Mod}).
wait() ->
receive stop ->
ok
end.
get_ext(Key) ->
case catch ets:lookup(global_storage(), Key) of
[] ->
io:format("Data for ~p not stored~n", [Key]),
undefined;
{'EXIT', Reason} ->
io:format("Get value for ~p failed (~p)~n", [Key, Reason]),
undefined;
[{Key,Value}] ->
Value
end.