QPQ-AG/mnesia_rocksdb
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

first version, based on mnesia_eleveldb

Browse Source
This commit is contained in:
Ulf Wiger
2018-02-06 09:43:57 +01:00
commit 6c9f5b565f
30 changed files with 4835 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mnesia_rocksdb.app.src
+12
View File
@@ -0,0 +1,12 @@
{application, mnesia_rocksdb,
[
{description, "RocksDB backend plugin for Mnesia"},
{vsn, "1.0"},
{modules, [mnesia_rocksdb, mnesia_rocksdb_app,
mnesia_rocksdb_params, mnesia_rocksdb_sext,
mnesia_rocksdb_sup, mnesia_rocksdb_tuning]},
{registered, []},
{mod, {mnesia_rocksdb_app, []}},
{env, []},
{applications, [kernel, stdlib]}
]}.
src/mnesia_rocksdb.erl
+1725
View File
File diff suppressed because it is too large Load Diff
src/mnesia_rocksdb_app.erl
+34
View File
@@ -0,0 +1,34 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_app).
-behaviour(application).
%% Application callbacks
-export([start/2, stop/1]).
%% ===================================================================
%% Application callbacks
%% ===================================================================
start(_StartType, _StartArgs) ->
mnesia_rocksdb_sup:start_link().
stop(_State) ->
ok.
src/mnesia_rocksdb_params.erl
+149
View File
@@ -0,0 +1,149 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_params).
-behaviour(gen_server).
-export([lookup/2,
store/2,
delete/1]).
-export([start_link/0,
init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3]).
-include("mnesia_rocksdb_tuning.hrl").
-define(KB, 1024).
-define(MB, 1024 * 1024).
-define(GB, 1024 * 1024 * 1024).
-ifdef(DEBUG).
-define(dbg(Fmt, Args), io:fwrite(user,"~p:~p: "++(Fmt),[?MODULE,?LINE|Args])).
-else.
-define(dbg(Fmt, Args), ok).
-endif.
lookup(Tab, Default) ->
try ets:lookup(?MODULE, Tab) of
[{_, Params}] ->
Params;
[] ->
Default
catch error:badarg ->
Default
end.
store(Tab, Params) ->
ets:insert(?MODULE, {Tab, Params}).
delete(Tab) ->
ets:delete(?MODULE, Tab).
start_link() ->
case ets:info(?MODULE, name) of
undefined ->
ets:new(?MODULE, [ordered_set, public, named_table]),
load_tuning_parameters();
_ ->
ok
end,
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
init(_) ->
{ok, []}.
handle_call(_, _, S) -> {reply, error, S}.
handle_cast(_, S) -> {noreply, S}.
handle_info(_, S) -> {noreply, S}.
terminate(_, _) -> ok.
code_change(_, S, _) -> {ok, S}.
load_tuning_parameters() ->
case application:get_env(mnesia_rocksdb, tuning_params) of
{ok, Ps} ->
case Ps of
{consult, F} -> consult(F);
{script, F} -> script(F);
_ when is_list(Ps) ->
store_params(Ps)
end;
_ ->
ok
end.
consult(F) ->
case file:consult(F) of
{ok, Terms} ->
store_params(Terms);
{error, Reason} ->
{error, {Reason, F}}
end.
script(F) ->
case file:script(F) of
{ok, Terms} ->
store_params(Terms);
{error, Reason} ->
{error, {Reason, F}}
end.
store_params(Params) ->
_ = lists:foreach(fun({_,S}) -> valid_size(S) end, Params),
NTabs = length(Params),
Env0= mnesia_rocksdb_tuning:describe_env(),
Env = Env0#tuning{n_tabs = NTabs},
?dbg("Env = ~p~n", [Env]),
TotalFiles = lists:sum([mnesia_rocksdb_tuning:max_files(Sz) ||
{_, Sz} <- Params]),
?dbg("TotalFiles = ~p~n", [TotalFiles]),
MaxFs = Env#tuning.max_files,
?dbg("MaxFs = ~p~n", [MaxFs]),
FsHeadroom = MaxFs * 0.6,
?dbg("FsHeadroom = ~p~n", [FsHeadroom]),
FilesFactor = if TotalFiles =< FsHeadroom ->
1; % don't have to scale down
true ->
FsHeadroom / TotalFiles
end,
Env1 = Env#tuning{files_factor = FilesFactor},
?dbg("Env1 = ~p~n", [Env1]),
lists:foreach(
fun({Tab, Sz}) when is_atom(Tab);
is_atom(element(1,Tab)),
is_integer(element(2,Tab)) ->
ets:insert(?MODULE, {Tab, ldb_params(Sz, Env1, Tab)})
end, Params).
ldb_params(Sz, Env, _Tab) ->
MaxFiles = mnesia_rocksdb_tuning:max_files(Sz) * Env#tuning.files_factor,
Opts = if Env#tuning.avail_ram > 100 -> % Gigabytes
[{write_buffer_size, mnesia_rocksdb_tuning:write_buffer(Sz)},
{cache_size, mnesia_rocksdb_tuning:cache(Sz)}];
true ->
[]
end,
[{max_open_files, MaxFiles} | Opts].
valid_size({I,U}) when is_number(I) ->
true = lists:member(U, [k,m,g]).
src/mnesia_rocksdb_sext.erl
+1100
View File
File diff suppressed because it is too large Load Diff
src/mnesia_rocksdb_sup.erl
+44
View File
@@ -0,0 +1,44 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_sup).
-behaviour(supervisor).
%% API
-export([start_link/0]).
%% Supervisor callbacks
-export([init/1]).
%% Helper macro for declaring children of supervisor
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
%% ===================================================================
%% API functions
%% ===================================================================
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
%% ===================================================================
%% Supervisor callbacks
%% ===================================================================
init([]) ->
{ok, { {one_for_one, 5, 10}, [?CHILD(mnesia_rocksdb_params, worker)]} }.
src/mnesia_rocksdb_tuning.erl
+198
View File
@@ -0,0 +1,198 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-module(mnesia_rocksdb_tuning).
-export([describe_env/0,
get_maxfiles/0, get_maxfiles/1,
get_avail_ram/0,
ldb_tabs/0, ldb_tabs/1,
ldb_indexes/0, ldb_indexes/1,
count_ldb_tabs/0, count_ldb_tabs/1,
calc_sizes/0, calc_sizes/1,
ideal_max_files/0, ideal_max_files/1,
max_files/1,
write_buffer/1,
cache/1,
default/1]).
-include("mnesia_rocksdb_tuning.hrl").
-define(KB, 1024).
-define(MB, 1024 * 1024).
-define(GB, 1024 * 1024 * 1024).
describe_env() ->
#tuning{max_files = get_maxfiles(),
avail_ram = get_avail_ram()}.
get_maxfiles() ->
get_maxfiles(os:type()).
get_maxfiles({unix,darwin}) ->
Limit = os:cmd("launchctl limit maxfiles"),
[_, Soft, _] = string:tokens(Limit, " \t\n"),
list_to_integer(Soft);
get_maxfiles({unix,linux}) ->
[L|_] = string:tokens(os:cmd("ulimit -n"), "\n"),
list_to_integer(L).
%% Returns installed RAM in Gigabytes
get_avail_ram() ->
get_avail_ram(os:type()).
get_avail_ram({unix,darwin}) ->
{match, [S]} =
re:run(os:cmd("/usr/sbin/system_profiler SPHardwareDataType"),
"Memory: (.+) GB", [{capture, [1], list}]),
list_to_integer(S);
get_avail_ram({unix,linux}) ->
{match, [S]} =
re:run(os:cmd("free -g"), "Mem:[ ]+([0-9]+) ",[{capture,[1],list}]),
list_to_integer(S).
ldb_tabs() ->
ldb_tabs(mnesia_lib:dir()).
ldb_tabs(Db) ->
ldb_tabs(list_dir(Db), Db).
ldb_tabs(Fs, _Db) ->
lists:flatmap(
fun(F) ->
case re:run(F, "(.+)-_tab\\.extldb",
[global,{capture,[1],list}]) of
{match, [Match]} ->
Match;
_ ->
[]
end
end, Fs).
ldb_indexes() ->
ldb_indexes(mnesia_lib:dir()).
ldb_indexes(Db) ->
ldb_indexes(list_dir(Db), Db).
ldb_indexes(Fs, _Db) ->
lists:flatmap(
fun(F) ->
case re:run(F, "(.+)-([0-9]+)-_ix\\.extldb",
[global,{capture,[1,2],list}]) of
{match, [[T,P]]} ->
[{T,P}];
_ ->
[]
end
end, Fs).
list_dir(D) ->
case file:list_dir(D) of
{ok, Fs} -> Fs;
{error, Reason} -> erlang:error({Reason,D})
end.
fname({Tab,IxPos}, Dir) ->
filename:join(Dir, Tab ++ "-" ++ IxPos ++ "-_ix.extldb");
fname(Tab, Dir) when is_list(Tab) ->
filename:join(Dir, Tab ++ "-_tab.extldb").
%% Number of leveldb tables + indexes
count_ldb_tabs() ->
count_ldb_tabs(mnesia_lib:dir()).
count_ldb_tabs(Db) ->
Fs = list_dir(Db),
length(ldb_tabs(Fs, Db)) + length(ldb_indexes(Fs, Db)).
calc_sizes() ->
calc_sizes(mnesia_lib:dir()).
calc_sizes(D) ->
lists:sort(
fun(A,B) -> sort_size(B,A) end, % rev sort
[{T, dir_size(fname(T, D))} || T <- ldb_tabs(D)]
++ [{I, dir_size(fname(I, D))} || I <- ldb_indexes(D)]).
ideal_max_files() ->
ideal_max_files(mnesia_lib:dir()).
ideal_max_files(D) ->
[{T,Sz,max_files(Sz)} || {T, Sz} <- calc_sizes(D)].
max_files({I,g}) ->
round(I * 1000) div 20;
max_files({I,m}) when I > 400 ->
round(I) div 20;
max_files(_) ->
default(max_open_files).
write_buffer({_,g}) ->
120 * ?MB;
write_buffer({I,m}) when I > 400 ->
120 * ?MB;
write_buffer(_) ->
default(write_buffer).
cache({_,g}) ->
8 * ?MB;
cache({I,m}) when I > 400 ->
6 * ?MB;
cache(_) ->
default(cache).
default(write_buffer) -> 2 * ?MB;
default(max_open_files) -> 20;
default(cache) -> 2 * ?MB.
%% open_file_memory() ->
%% (max_open_files-10) *
%% (184 + (average_sst_filesize/2048) *
%% (8 + ((average_key_size+average_value_size)/2048 +1) * 0.6).
dir_size(D) ->
R = os:cmd("du -hc " ++ D ++ " | grep total"),
parse_sz(hd(string:tokens(R," \t\n"))).
parse_sz(S) ->
{match,[I,U]} = re:run(S, "([\\.0-9]+)([BKMG])", [{capture,[1,2],list}]),
{scan_num(I), unit(U)}.
scan_num(S) ->
case erl_scan:string(S) of
{ok, [{integer,_,I}],_} ->
I;
{ok, [{float,_,F}],_} ->
F
end.
unit("B") -> b;
unit("K") -> k;
unit("M") -> m;
unit("G") -> g.
%% Custom sort: b < k < m < g
sort_size({_,{A,U}},{_,{B,U}}) -> A < B;
sort_size({_,{_,U1}},{_,{_,U2}}) ->
case {U1,U2} of
{b,_} -> true;
{k,_} when U2 =/= b -> true;
{m,g} -> true;
_ -> false
end.
src/mnesia_rocksdb_tuning.hrl
+22
View File
@@ -0,0 +1,22 @@
%%----------------------------------------------------------------
%% Copyright (c) 2013-2016 Klarna AB
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License. You may obtain
%% a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied. See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%----------------------------------------------------------------
-record(tuning, {max_files,
n_tabs,
avail_ram,
files_factor = 1}).