QPQ-AG/mnesia_rocksdb
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More renaming, ldb -> rdb

Browse Source
This commit is contained in:
Ulf Wiger 2018-02-06 20:03:01 +01:00
parent d326daf251
commit af4c6e8fd2
3 changed files with 53 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/mnesia_rocksdb.erl
View File

@ -360,26 +360,26 @@ create_table(_Alias, Tab, _Props) ->
load_table(Alias, Tab, _LoadReason, Opts) -> load_table(Alias, Tab, _LoadReason, Opts) ->
Type = proplists:get_value(type, Opts), Type = proplists:get_value(type, Opts),
LdbUserProps = proplists:get_value( RdbUserProps = proplists:get_value(
rocksdb_opts, proplists:get_value( rocksdb_opts, proplists:get_value(
user_properties, Opts, []), []), user_properties, Opts, []), []),
StorageProps = proplists:get_value( StorageProps = proplists:get_value(
rocksdb, proplists:get_value( rocksdb, proplists:get_value(
storage_properties, Opts, []), LdbUserProps), storage_properties, Opts, []), RdbUserProps),
LdbOpts = mnesia_rocksdb_params:lookup(Tab, StorageProps), RdbOpts = mnesia_rocksdb_params:lookup(Tab, StorageProps),
ProcName = proc_name(Alias, Tab), ProcName = proc_name(Alias, Tab),
case whereis(ProcName) of case whereis(ProcName) of
undefined -> undefined ->
load_table_(Alias, Tab, Type, LdbOpts); load_table_(Alias, Tab, Type, RdbOpts);
Pid -> Pid ->
gen_server:call(Pid, {load, Alias, Tab, Type, LdbOpts}, infinity) gen_server:call(Pid, {load, Alias, Tab, Type, RdbOpts}, infinity)
end. end.
load_table_(Alias, Tab, Type, LdbOpts) -> load_table_(Alias, Tab, Type, RdbOpts) ->
ShutdownTime = proplists:get_value( ShutdownTime = proplists:get_value(
owner_shutdown_time, LdbOpts, 120000), owner_shutdown_time, RdbOpts, 120000),
case mnesia_ext_sup:start_proc( case mnesia_ext_sup:start_proc(
Tab, ?MODULE, start_proc, [Alias,Tab,Type, LdbOpts], Tab, ?MODULE, start_proc, [Alias,Tab,Type, RdbOpts],
[{shutdown, ShutdownTime}]) of [{shutdown, ShutdownTime}]) of
{ok, _Pid} -> {ok, _Pid} ->
ok; ok;
@ -858,7 +858,7 @@ validate_record(_Alias, _Tab, RecName, Arity, Type, _Obj) ->
%% Extensions for files that are permanent. Needs to be cleaned up %% Extensions for files that are permanent. Needs to be cleaned up
%% e.g. at deleting the schema. %% e.g. at deleting the schema.
real_suffixes() -> real_suffixes() ->
[".extldb"]. [".extrdb"].
%% Extensions for temporary files. Can be cleaned up when mnesia %% Extensions for temporary files. Can be cleaned up when mnesia
%% cleans up other temporary files. %% cleans up other temporary files.
@ -870,14 +870,14 @@ tmp_suffixes() ->
%% GEN SERVER CALLBACKS AND CALLS %% GEN SERVER CALLBACKS AND CALLS
%% ---------------------------------------------------------------------------- %% ----------------------------------------------------------------------------
start_proc(Alias, Tab, Type, LdbOpts) -> start_proc(Alias, Tab, Type, RdbOpts) ->
ProcName = proc_name(Alias, Tab), ProcName = proc_name(Alias, Tab),
gen_server:start_link({local, ProcName}, ?MODULE, gen_server:start_link({local, ProcName}, ?MODULE,
{Alias, Tab, Type, LdbOpts}, []). {Alias, Tab, Type, RdbOpts}, []).
init({Alias, Tab, Type, LdbOpts}) -> init({Alias, Tab, Type, RdbOpts}) ->
process_flag(trap_exit, true), process_flag(trap_exit, true),
{ok, Ref, Ets} = do_load_table(Tab, LdbOpts), {ok, Ref, Ets} = do_load_table(Tab, RdbOpts),
St = #st{ ets = Ets St = #st{ ets = Ets
, ref = Ref , ref = Ref
, alias = Alias , alias = Alias
@ -888,17 +888,17 @@ init({Alias, Tab, Type, LdbOpts}) ->
}, },
{ok, recover_size_info(St)}. {ok, recover_size_info(St)}.
do_load_table(Tab, LdbOpts) -> do_load_table(Tab, RdbOpts) ->
MPd = data_mountpoint(Tab), MPd = data_mountpoint(Tab),
?dbg("** Mountpoint: ~p~n ~s~n", [MPd, os:cmd("ls " ++ MPd)]), ?dbg("** Mountpoint: ~p~n ~s~n", [MPd, os:cmd("ls " ++ MPd)]),
Ets = ets:new(tab_name(icache,Tab), [set, protected, named_table]), Ets = ets:new(tab_name(icache,Tab), [set, protected, named_table]),
{ok, Ref} = open_rocksdb(MPd, LdbOpts), {ok, Ref} = open_rocksdb(MPd, RdbOpts),
rocksdb_to_ets(Ref, Ets), rocksdb_to_ets(Ref, Ets),
{ok, Ref, Ets}. {ok, Ref, Ets}.
handle_call({load, Alias, Tab, Type, LdbOpts}, _From, handle_call({load, Alias, Tab, Type, RdbOpts}, _From,
#st{type = Type, alias = Alias, tab = Tab} = St) -> #st{type = Type, alias = Alias, tab = Tab} = St) ->
{ok, Ref, Ets} = do_load_table(Tab, LdbOpts), {ok, Ref, Ets} = do_load_table(Tab, RdbOpts),
{reply, ok, St#st{ref = Ref, ets = Ets}}; {reply, ok, St#st{ref = Ref, ets = Ets}};
handle_call(get_ref, _From, #st{ref = Ref, type = Type} = St) -> handle_call(get_ref, _From, #st{ref = Ref, type = Type} = St) ->
{reply, {Ref, Type}, St}; {reply, {Ref, Type}, St};
@ -993,14 +993,14 @@ rocksdb_open_opts({Tab, index, {Pos,_}}) ->
rocksdb_open_opts_(PosOpts); rocksdb_open_opts_(PosOpts);
rocksdb_open_opts(Tab) -> rocksdb_open_opts(Tab) ->
UserProps = mnesia_lib:val({Tab, user_properties}), UserProps = mnesia_lib:val({Tab, user_properties}),
LdbOpts = proplists:get_value(rocksdb_opts, UserProps, []), RdbOpts = proplists:get_value(rocksdb_opts, UserProps, []),
rocksdb_open_opts_(LdbOpts). rocksdb_open_opts_(RdbOpts).
rocksdb_open_opts_(LdbOpts) -> rocksdb_open_opts_(RdbOpts) ->
lists:foldl( lists:foldl(
fun({K,_} = Item, Acc) -> fun({K,_} = Item, Acc) ->
lists:keystore(K, 1, Acc, Item) lists:keystore(K, 1, Acc, Item)
end, default_open_opts(), LdbOpts). end, default_open_opts(), RdbOpts).
default_open_opts() -> default_open_opts() ->
[ {create_if_missing, true} [ {create_if_missing, true}
@ -1016,12 +1016,12 @@ default_open_opts() ->
, {use_bloomfilter, true} , {use_bloomfilter, true}
]. ].
destroy_recreate(MPd, LdbOpts) -> destroy_recreate(MPd, RdbOpts) ->
ok = destroy_db(MPd, []), ok = destroy_db(MPd, []),
open_rocksdb(MPd, LdbOpts). open_rocksdb(MPd, RdbOpts).
open_rocksdb(MPd, LdbOpts) -> open_rocksdb(MPd, RdbOpts) ->
open_rocksdb(MPd, rocksdb_open_opts_(LdbOpts), get_retries()). open_rocksdb(MPd, rocksdb_open_opts_(RdbOpts), get_retries()).
%% Code adapted from basho/riak_kv_eleveldb_backend.erl %% Code adapted from basho/riak_kv_eleveldb_backend.erl
open_rocksdb(MPd, Opts, Retries) -> open_rocksdb(MPd, Opts, Retries) ->
@ -1618,7 +1618,7 @@ assert_proper_mountpoint(_Tab, _MPd) ->
data_mountpoint(Tab) -> data_mountpoint(Tab) ->
Dir = mnesia_monitor:get_env(dir), Dir = mnesia_monitor:get_env(dir),
filename:join(Dir, tabname(Tab) ++ ".extldb"). filename:join(Dir, tabname(Tab) ++ ".extrdb").
tabname({Tab, index, {{Pos},_}}) -> tabname({Tab, index, {{Pos},_}}) ->
atom_to_list(Tab) ++ "-=" ++ atom_to_list(Pos) ++ "=-_ix"; atom_to_list(Tab) ++ "-=" ++ atom_to_list(Pos) ++ "=-_ix";
@ -1665,7 +1665,7 @@ related_resources(Tab) ->
end. end.
is_index_dir(F, TabS) -> is_index_dir(F, TabS) ->
case re:run(F, TabS ++ "-([0-9]+)-_ix.extldb", [{capture, [1], list}]) of case re:run(F, TabS ++ "-([0-9]+)-_ix.extrdb", [{capture, [1], list}]) of
nomatch -> nomatch ->
false; false;
{match, [P]} -> {match, [P]} ->

4
src/mnesia_rocksdb_params.erl
View File

@ -132,10 +132,10 @@ store_params(Params) ->
fun({Tab, Sz}) when is_atom(Tab); fun({Tab, Sz}) when is_atom(Tab);
is_atom(element(1,Tab)), is_atom(element(1,Tab)),
is_integer(element(2,Tab)) -> is_integer(element(2,Tab)) ->
ets:insert(?MODULE, {Tab, ldb_params(Sz, Env1, Tab)}) ets:insert(?MODULE, {Tab, rdb_params(Sz, Env1, Tab)})
end, Params). end, Params).
ldb_params(Sz, Env, _Tab) -> rdb_params(Sz, Env, _Tab) ->
MaxFiles = mnesia_rocksdb_tuning:max_files(Sz) * Env#tuning.files_factor, MaxFiles = mnesia_rocksdb_tuning:max_files(Sz) * Env#tuning.files_factor,
Opts = if Env#tuning.avail_ram > 100 -> % Gigabytes Opts = if Env#tuning.avail_ram > 100 -> % Gigabytes
[{write_buffer_size, mnesia_rocksdb_tuning:write_buffer(Sz)}, [{write_buffer_size, mnesia_rocksdb_tuning:write_buffer(Sz)},

48
src/mnesia_rocksdb_tuning.erl
View File

@ -21,9 +21,9 @@
-export([describe_env/0, -export([describe_env/0,
get_maxfiles/0, get_maxfiles/1, get_maxfiles/0, get_maxfiles/1,
get_avail_ram/0, get_avail_ram/0,
ldb_tabs/0, ldb_tabs/1, rdb_tabs/0, rdb_tabs/1,
ldb_indexes/0, ldb_indexes/1, rdb_indexes/0, rdb_indexes/1,
count_ldb_tabs/0, count_ldb_tabs/1, count_rdb_tabs/0, count_rdb_tabs/1,
calc_sizes/0, calc_sizes/1, calc_sizes/0, calc_sizes/1,
ideal_max_files/0, ideal_max_files/1, ideal_max_files/0, ideal_max_files/1,
max_files/1, max_files/1,
@ -66,16 +66,16 @@ get_avail_ram({unix,linux}) ->
re:run(os:cmd("free -g"), "Mem:[ ]+([0-9]+) ",[{capture,[1],list}]), re:run(os:cmd("free -g"), "Mem:[ ]+([0-9]+) ",[{capture,[1],list}]),
list_to_integer(S). list_to_integer(S).
ldb_tabs() -> rdb_tabs() ->
ldb_tabs(mnesia_lib:dir()). rdb_tabs(mnesia_lib:dir()).
ldb_tabs(Db) -> rdb_tabs(Db) ->
ldb_tabs(list_dir(Db), Db). rdb_tabs(list_dir(Db), Db).
ldb_tabs(Fs, _Db) -> rdb_tabs(Fs, _Db) ->
lists:flatmap( lists:flatmap(
fun(F) -> fun(F) ->
case re:run(F, "(.+)-_tab\\.extldb", case re:run(F, "(.+)-_tab\\.extrdb",
[global,{capture,[1],list}]) of [global,{capture,[1],list}]) of
{match, [Match]} -> {match, [Match]} ->
Match; Match;
@ -84,16 +84,16 @@ ldb_tabs(Fs, _Db) ->
end end
end, Fs). end, Fs).
ldb_indexes() -> rdb_indexes() ->
ldb_indexes(mnesia_lib:dir()). rdb_indexes(mnesia_lib:dir()).
ldb_indexes(Db) -> rdb_indexes(Db) ->
ldb_indexes(list_dir(Db), Db). rdb_indexes(list_dir(Db), Db).
ldb_indexes(Fs, _Db) -> rdb_indexes(Fs, _Db) ->
lists:flatmap( lists:flatmap(
fun(F) -> fun(F) ->
case re:run(F, "(.+)-([0-9]+)-_ix\\.extldb", case re:run(F, "(.+)-([0-9]+)-_ix\\.extrdb",
[global,{capture,[1,2],list}]) of [global,{capture,[1,2],list}]) of
{match, [[T,P]]} -> {match, [[T,P]]} ->
[{T,P}]; [{T,P}];
@ -109,17 +109,17 @@ list_dir(D) ->
end. end.
fname({Tab,IxPos}, Dir) -> fname({Tab,IxPos}, Dir) ->
filename:join(Dir, Tab ++ "-" ++ IxPos ++ "-_ix.extldb"); filename:join(Dir, Tab ++ "-" ++ IxPos ++ "-_ix.extrdb");
fname(Tab, Dir) when is_list(Tab) -> fname(Tab, Dir) when is_list(Tab) ->
filename:join(Dir, Tab ++ "-_tab.extldb"). filename:join(Dir, Tab ++ "-_tab.extrdb").
%% Number of leveldb tables + indexes %% Number of rocksdb tables + indexes
count_ldb_tabs() -> count_rdb_tabs() ->
count_ldb_tabs(mnesia_lib:dir()). count_rdb_tabs(mnesia_lib:dir()).
count_ldb_tabs(Db) -> count_rdb_tabs(Db) ->
Fs = list_dir(Db), Fs = list_dir(Db),
length(ldb_tabs(Fs, Db)) + length(ldb_indexes(Fs, Db)). length(rdb_tabs(Fs, Db)) + length(rdb_indexes(Fs, Db)).
calc_sizes() -> calc_sizes() ->
calc_sizes(mnesia_lib:dir()). calc_sizes(mnesia_lib:dir()).
@ -127,8 +127,8 @@ calc_sizes() ->
calc_sizes(D) -> calc_sizes(D) ->
lists:sort( lists:sort(
fun(A,B) -> sort_size(B,A) end, % rev sort fun(A,B) -> sort_size(B,A) end, % rev sort
[{T, dir_size(fname(T, D))} || T <- ldb_tabs(D)] [{T, dir_size(fname(T, D))} || T <- rdb_tabs(D)]
++ [{I, dir_size(fname(I, D))} || I <- ldb_indexes(D)]). ++ [{I, dir_size(fname(I, D))} || I <- rdb_indexes(D)]).
ideal_max_files() -> ideal_max_files() ->
ideal_max_files(mnesia_lib:dir()). ideal_max_files(mnesia_lib:dir()).