diff --git a/README.md b/README.md index 1ee59b8..a7d1472 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ is provided. * sext (included as dependency) -* Erlang/OTP 21.0 or newer (https://github.com/erlang/otp) +* Erlang/OTP 27.0 or newer (https://github.com/erlang/otp) @@ -115,6 +115,65 @@ source code in `mnesia_rocksdb_tuning.erl` and `mnesia_rocksdb_params.erl`. Contributions are welcome. +#### Backup #### + +`mnesia_rocksdb:backup(Dir)` creates a **local hybrid backup** of Mnesia +metadata/tables and RocksDB data. Unlike `mnesia:backup/1`, RocksDB-backed +tables are not dumped record-by-record through Mnesia; they are captured with +the [RocksDB backup engine](https://rocksdb.hexdocs.pm/1.9.0/how_to_backup_rocksdb.html). + +Layout of `Dir` after a successful backup: + +``` +Dir/ + MANIFEST % restore metadata (Erlang term) + mnesia.BUP % schema + non-RocksDB tables + rocksdb//admin/ % backup engine dir for each alias admin DB + rocksdb/standalone/... % one dir per open standalone RocksDB table +``` + +Algorithm (local node only): + +1. Activate a Mnesia checkpoint on **schema and non-RocksDB tables only** + (so large RocksDB tables do not get expensive checkpoint retainers). +2. For each open RocksDB instance (admin DB per alias, and standalone tables), + run `rocksdb:create_new_backup/2`. +3. Dump the checkpoint with `mnesia:backup_checkpoint/3`, excluding RocksDB + table data. Schema still describes RocksDB tables. +4. Write `MANIFEST`. + +Options map (second argument to `backup/2`): + +* `{verify, true}` — call `rocksdb:verify_backup/2` after each RocksDB backup +* `{mnesia_backup_module, Mod}` — override the Mnesia backup media module + +#### Restore + +`mnesia_rocksdb:restore(Dir)` restores a hybrid backup offline (local node). + +Mnesia applies fallbacks by **purging** the Mnesia directory. RocksDB data is +therefore restored first into a **sibling staging directory** and mountpoints +are redirected until the fallback has finished: + +1. Stop Mnesia (closes RocksDB handles). +2. Restore each RocksDB DB into + `".rocksdb_restore/"` (sibling of the Mnesia dir). +3. Set application env `mnesia_rocksdb.restore_mountpoint_root` so + `data_mountpoint/1` opens DBs from staging. +4. Install `Dir/mnesia.BUP` and start Mnesia (fallback purges the real Mnesia + dir; RocksDB tables load from staging with full data). During this window, + `delete_table` does not destroy staged RocksDB files. +5. Stop Mnesia, clear the redirect, move staged `*.extrdb` directories into the + real Mnesia directory. +6. Optionally start Mnesia (`#{start => true}`). + +Common options for `restore/2`: + +* `{stop, true|false}` — stop Mnesia before restore (default `true`) +* `{start, true|false}` — start Mnesia after finalize (default `false`) +* `{mnesia_dir, Dir}` — override target Mnesia directory +* `{skip_rocksdb, true}` / `{skip_mnesia, true}` — partial restore + #### Caveats #### Avoid placing `bag` tables in RocksDB. Although they work, each write diff --git a/doc/README.md b/doc/README.md index 28c149d..0aa362f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -43,7 +43,7 @@ is provided. * sext (included as dependency) -* Erlang/OTP 21.0 or newer (https://github.com/erlang/otp) +* Erlang/OTP 27.0 or newer (https://github.com/erlang/otp) diff --git a/doc/plugin-userguide.md b/doc/plugin-userguide.md index d9712d1..9307484 100644 --- a/doc/plugin-userguide.md +++ b/doc/plugin-userguide.md @@ -45,7 +45,7 @@ backs up data. - rocksdb (included as dependency) - sext (included as dependency) -- Erlang/OTP 22.0 or newer (https://github.com/erlang/otp) +- Erlang/OTP 27.0 or newer (https://github.com/erlang/otp) ### Getting started diff --git a/rebar.config b/rebar.config index e480c14..80ce9a4 100644 --- a/rebar.config +++ b/rebar.config @@ -1,5 +1,5 @@ %% -*- erlang -*- -{minimum_otp_vsn, "21.0"}. +{minimum_otp_vsn, "27.0"}. {deps, [ diff --git a/src/mnesia_rocksdb.erl b/src/mnesia_rocksdb.erl index 82b50a9..871f2f8 100644 --- a/src/mnesia_rocksdb.erl +++ b/src/mnesia_rocksdb.erl @@ -49,7 +49,11 @@ -export([register/0, register/1, - default_alias/0]). + default_alias/0, + backup/1, + backup/2, + restore/1, + restore/2]). %% %% DEBUG API @@ -218,6 +222,44 @@ get_ref(Tab) -> default_alias() -> rocksdb_copies. +%% @doc Hybrid local backup of Mnesia + RocksDB tables. +%% +%% Similar intent to `mnesia:backup/1`, but RocksDB-backed tables are backed up +%% with the RocksDB backup engine instead of record-level iteration. +%% +%% `Dir` must be a non-existent or empty directory. On success it contains: +%% - `MANIFEST` — restore metadata +%% - `mnesia.BUP` — schema and non-RocksDB tables +%% - `rocksdb/...` — native RocksDB backups for each open DB +%% +%% Local node only. See `mnesia_rocksdb_admin:backup/2` for details and options. +%% @end +-spec backup(file:filename_all()) -> ok | {error, term()}. +backup(Dir) -> + mnesia_rocksdb_admin:backup(Dir, #{}). + +%% @equiv mnesia_rocksdb_admin:backup/2 +-spec backup(file:filename_all(), map()) -> ok | {error, term()}. +backup(Dir, Opts) when is_map(Opts) -> + mnesia_rocksdb_admin:backup(Dir, Opts). + +%% @doc Restore a hybrid backup created by {@link backup/1}. +%% +%% Offline restore: RocksDB data is restored into a sibling staging directory +%% first; Mnesia fallback runs with mountpoints redirected there (so the +%% fallback purge does not wipe RocksDB data); then staged dirs are moved into +%% the real Mnesia directory. Optionally starts Mnesia (`#{start => true}`). +%% +%% See `mnesia_rocksdb_admin:restore/2` for options and the full sequence. +%% @end +-spec restore(file:filename_all()) -> ok | {error, term()}. +restore(Dir) -> + mnesia_rocksdb_admin:restore(Dir, #{}). + +%% @equiv mnesia_rocksdb_admin:restore/2 +-spec restore(file:filename_all(), map()) -> ok | {error, term()}. +restore(Dir, Opts) when is_map(Opts) -> + mnesia_rocksdb_admin:restore(Dir, Opts). encode_key(Key) -> mnesia_rocksdb_lib:encode_key(Key, sext). diff --git a/src/mnesia_rocksdb_admin.erl b/src/mnesia_rocksdb_admin.erl index 9287f90..f7ed9b0 100644 --- a/src/mnesia_rocksdb_admin.erl +++ b/src/mnesia_rocksdb_admin.erl @@ -17,6 +17,11 @@ , request_ref/2 %% (Alias, Name) -> {ok, Ref} | {error, _} , close_table/2 , clear_table/1 + , backup/1 %% (Dir) -> ok | {error, _} + , backup/2 %% (Dir, Opts) -> ok | {error, _} + , restore/1 %% (Dir) -> ok | {error, _} + , restore/2 %% (Dir, Opts) -> ok | {error, _} + , list_open_dbs/0 %% () -> [db_backup_spec()] ]). -export([ migrate_standalone/2 @@ -109,7 +114,35 @@ | {abort_migration, tabname()} | {prep_close, table()} | {close_table, table()} - | {clear_table, table() | cf() }. + | {clear_table, table() | cf() } + | list_open_dbs. + +%% Spec describing an open RocksDB instance eligible for backup. +-type db_backup_spec() :: #{ kind := admin | standalone + , alias := alias() + , name := table() + , db_ref := db_ref() + , mountpoint := file:filename() + }. + +-type backup_opts() :: #{ verify => boolean() + , mnesia_backup_module => module() + }. + +%% Restore options: +%% - `mnesia_dir': target Mnesia directory (default: configured Mnesia dir) +%% - `stop': stop Mnesia if running before restore (default true) +%% - `start': start Mnesia after restore (default false) +%% - `mnesia_backup_module': module for reading mnesia.BUP +%% - `skip_rocksdb': only install Mnesia fallback (default false) +%% - `skip_mnesia': only restore RocksDB DBs (default false) +-type restore_opts() :: #{ mnesia_dir => file:filename_all() + , stop => boolean() + , start => boolean() + , mnesia_backup_module => module() + , skip_rocksdb => boolean() + , skip_mnesia => boolean() + }. -type reason() :: any(). -type reply() :: any(). @@ -164,7 +197,13 @@ erase_pt_list(Names) -> check_application_defaults(Meta) -> Value = application:get_env(mnesia_rocksdb, mnesia_compatible_aborts, false), - Meta#{ {mnesia_compatible_aborts} => Value }. + Meta1 = Meta#{ {mnesia_compatible_aborts} => Value }, + case application:get_env(mnesia_rocksdb, restore_mountpoint_root) of + {ok, Root} when is_list(Root); is_binary(Root) -> + Meta1#{ {restore_mountpoint_root} => filename:absname(Root) }; + _ -> + Meta1 + end. get_cached_env(Key, Default) -> maps:get({Key}, meta(), Default). @@ -200,6 +239,96 @@ get_pt(Name, Default) -> set_and_cache_env(Key, Value) -> gen_server:call(?MODULE, {set_and_cache_env, Key, Value}). +%% @doc Hybrid backup of Mnesia metadata/non-RocksDB tables and RocksDB DBs. +%% +%% Creates a directory layout: +%% ``` +%% Dir/ +%% MANIFEST +%% mnesia.BUP +%% rocksdb//admin/ +%% rocksdb/standalone// +%% ''' +%% +%% Steps: +%% 1. Activate a Mnesia checkpoint covering schema and non-RocksDB tables only +%% (avoids heavy RocksDB retainer tables). +%% 2. For each open RocksDB instance (admin DBs and standalone tables), create a +%% RocksDB-native backup via the backup engine. +%% 3. Dump the checkpoint with `mnesia:backup_checkpoint/3`, excluding RocksDB +%% table data (schema still describes those tables). +%% 4. Write a MANIFEST describing the layout for a future restore. +%% +%% Local-node only. Cross-store consistency is best-effort within the checkpoint +%% window. +%% +%% Options: +%% - `verify` (boolean, default false): call `rocksdb:verify_backup/2` +%% - `mnesia_backup_module` (module): override Mnesia backup media module +-spec backup(file:filename_all()) -> ok | {error, term()}. +backup(Dir) -> + backup(Dir, #{}). + +-spec backup(file:filename_all(), backup_opts()) -> ok | {error, term()}. +backup(Dir, Opts) when is_map(Opts) -> + ensure_started(), + case mnesia:system_info(is_running) of + yes -> + case prepare_backup_dir(Dir) of + ok -> + try do_backup(filename:absname(Dir), Opts) + catch + error:Reason:Stack -> + {error, {Reason, Stack}}; + throw:{error, _} = Err -> + Err + end; + {error, _} = E -> + E + end; + Other -> + {error, {mnesia_not_running, Other}} + end. + +%% @doc Restore a hybrid backup previously created by {@link backup/1}. +%% +%% Offline restore (local node). Mnesia fallback application **purges** the +%% Mnesia directory, so RocksDB data is restored first into a **sibling** +%% staging directory and mountpoints are redirected while the fallback runs: +%% +%% 1. Stop Mnesia (closes RocksDB handles). +%% 2. Restore each RocksDB DB into +%% `".rocksdb_restore/"` (sibling of the Mnesia dir). +%% 3. Set `restore_mountpoint_root` so `data_mountpoint/1` uses the staging +%% location. +%% 4. Install `mnesia.BUP` and start Mnesia (fallback purges the real Mnesia +%% dir; RocksDB tables open from staging with full data). +%% 5. Stop Mnesia, clear the redirect, move staged `*.extrdb` into the real +%% Mnesia directory. +%% 6. Optionally start Mnesia (`#{start => true}`). +-spec restore(file:filename_all()) -> ok | {error, term()}. +restore(Dir) -> + restore(Dir, #{}). + +-spec restore(file:filename_all(), restore_opts()) -> ok | {error, term()}. +restore(Dir, Opts) when is_map(Opts) -> + try do_restore(filename:absname(Dir), Opts) + catch + error:Reason:Stack -> + {error, {Reason, Stack}}; + throw:{error, _} = Err -> + Err + end. + +%% @doc Snapshot of open RocksDB instances (admin + standalone). +-spec list_open_dbs() -> [db_backup_spec()]. +list_open_dbs() -> + ensure_started(), + case gen_server:call(?MODULE, {[], list_open_dbs}, infinity) of + {ok, Dbs} -> Dbs; + {error, _} = E -> error(E) + end. + create_table(Alias, Name, Props) -> call(Alias, {create_table, Name, Props}). @@ -616,6 +745,8 @@ handle_call({[], {get_ref, Name}}, _From, St) -> _ -> {reply, {error, not_found}, St} end; +handle_call({[], list_open_dbs}, _From, St) -> + {reply, {ok, list_open_dbs_from_st(St)}, St}; handle_call({Alias, Req}, _From, St) -> handle_call_for_alias(Alias, Req, St); handle_call(_Req, _From, St) -> @@ -1604,23 +1735,35 @@ do_close_table(Alias, Name, Backend, #st{standalone = Ts} = St) -> end. do_delete_table(Alias, Name, Backend, #st{} = St) -> - case find_cf(Alias, Name, Backend, St) of - {ok, Where} -> - erase_pt(Name), - case Where of - #{db_ref := DbRef, cf_handle := CfH, type := column_family} -> - rocksdb:drop_column_family(DbRef, CfH), - drop_cached_cf(tab_to_cf_name(Name), CfH), - rocksdb:destroy_column_family(DbRef, CfH), - {ok, delete_cf(Alias, Name, St)}; - #{type := standalone} = R -> - St1 = close_and_delete_standalone(R, St), - {ok, St1} - end; - error -> - {error, not_found} + case restore_staging_active() of + true -> + %% Hybrid restore: RocksDB lives in a staging sibling. Mnesia + %% fallback calls delete_table before reopening; must not destroy + %% staged files or drop CFs that still hold restored data. + {ok, St}; + false -> + case find_cf(Alias, Name, Backend, St) of + {ok, Where} -> + erase_pt(Name), + case Where of + #{db_ref := DbRef, cf_handle := CfH, + type := column_family} -> + rocksdb:drop_column_family(DbRef, CfH), + drop_cached_cf(tab_to_cf_name(Name), CfH), + rocksdb:destroy_column_family(DbRef, CfH), + {ok, delete_cf(Alias, Name, St)}; + #{type := standalone} = R -> + St1 = close_and_delete_standalone(R, St), + {ok, St1} + end; + error -> + {error, not_found} + end end. +restore_staging_active() -> + mnesia_rocksdb_lib:restore_mountpoint_root() =/= undefined. + load_admin_db(Alias, Opts) -> DbName = {admin, Alias}, open_db(DbName, Alias, Opts, [DbName], true). @@ -2377,3 +2520,669 @@ destroy_db(MPd, Opts, RetriesLeft, _) -> get_retries() -> 30. get_retry_delay() -> 10000. + +%% =================================================================== +%% Hybrid backup (Mnesia checkpoint + RocksDB backup engine) +%% =================================================================== + +list_open_dbs_from_st(#st{backends = Bs, standalone = Ts}) -> + Admin = + maps:fold( + fun(Alias, #{db_ref := DbRef, mountpoint := MP}, Acc) -> + [#{ kind => admin + , alias => Alias + , name => {admin, Alias} + , db_ref => DbRef + , mountpoint => MP } | Acc]; + (_, _, Acc) -> + Acc + end, [], Bs), + Standalone = + maps:fold( + fun({Alias, Name}, #{db_ref := DbRef, mountpoint := MP} = R, Acc) -> + case maps:get(status, R, open) of + closed -> + Acc; + _ -> + [#{ kind => standalone + , alias => Alias + , name => Name + , db_ref => DbRef + , mountpoint => MP } | Acc] + end; + (_, _, Acc) -> + Acc + end, [], Ts), + Admin ++ Standalone. + +prepare_backup_dir(Dir) -> + case filelib:is_dir(Dir) of + true -> + case file:list_dir(Dir) of + {ok, []} -> + ok; + {ok, _} -> + {error, {not_empty, Dir}}; + {error, _} = E -> + E + end; + false -> + case filelib:ensure_dir(filename:join(Dir, "dummy")) of + ok -> + case file:make_dir(Dir) of + ok -> ok; + {error, eexist} -> + %% Race: created between is_dir and make_dir + prepare_backup_dir(Dir); + {error, _} = E -> + E + end; + {error, _} = E -> + E + end + end. + +do_backup(Dir, Opts) -> + {RdbTabs, NonRdbTabs} = classify_tables(), + CpArgs = [ {ram_overrides_dump, false} + , {max, NonRdbTabs} + , {allow_remote, false} ], + case mnesia:activate_checkpoint(CpArgs) of + {ok, CpName, _Nodes} -> + try + Dbs = list_open_dbs(), + case backup_rocksdbs(Dir, Dbs, Opts) of + {ok, DbInfos} -> + Bup = filename:join(Dir, "mnesia.BUP"), + case backup_mnesia_checkpoint(CpName, Bup, NonRdbTabs, Opts) of + ok -> + write_manifest(Dir, NonRdbTabs, RdbTabs, DbInfos); + {error, _} = E -> + E + end; + {error, _} = E -> + E + end + after + _ = mnesia:deactivate_checkpoint(CpName) + end; + {error, _} = E -> + E + end. + +classify_tables() -> + All = mnesia:system_info(tables), + Rdb = [T || T <- All, is_local_rdb_table(T)], + NonRdb = All -- Rdb, + {lists:sort(Rdb), lists:sort(NonRdb)}. + +is_local_rdb_table(schema) -> + false; +is_local_rdb_table(Tab) -> + try mnesia:table_info(Tab, storage_type) of + {ext, _Alias, mnesia_rocksdb} -> true; + _ -> false + catch + _:_ -> false + end. + +backup_mnesia_checkpoint(CpName, BupPath, NonRdbTabs, Opts) -> + Mod = maps:get(mnesia_backup_module, Opts, + mnesia:system_info(backup_module)), + %% mnesia:backup_checkpoint/3 is typed as module()-only; the implementation + %% (and mnesia_log:backup_checkpoint/3) accepts a proplist with table filter. + Args = [ {module, Mod} + , {tables, NonRdbTabs} + , {scope, local} ], + case mnesia_log:backup_checkpoint(CpName, BupPath, Args) of + ok -> ok; + {error, _} = E -> E + end. + +backup_rocksdbs(Dir, Dbs, Opts) -> + backup_rocksdbs(Dir, Dbs, Opts, []). + +backup_rocksdbs(_Dir, [], _Opts, Acc) -> + {ok, lists:reverse(Acc)}; +backup_rocksdbs(Dir, [Spec | Rest], Opts, Acc) -> + case backup_one_db(Dir, Spec, Opts) of + {ok, Info} -> + backup_rocksdbs(Dir, Rest, Opts, [Info | Acc]); + {error, _} = E -> + E + end. + +backup_one_db(Dir, #{ db_ref := DbRef + , mountpoint := MP + , kind := Kind + , alias := Alias + , name := Name } = Spec, Opts) -> + Rel = backup_relpath(Spec), + Abs = filename:join(Dir, Rel), + case ensure_dir_tree(Abs) of + ok -> + case rocksdb:open_backup_engine(Abs) of + {ok, BE} -> + try + case rocksdb:create_new_backup(BE, DbRef) of + ok -> + case latest_backup_id(BE) of + {ok, BackupId} -> + case maybe_verify_backup(BE, BackupId, Opts) of + ok -> + {ok, #{ kind => Kind + , alias => Alias + , name => Name + , mountpoint => MP + , backup_relpath => Rel + , backup_id => BackupId }}; + {error, _} = E -> + E + end; + {error, _} = E -> + E + end; + {error, _} = E -> + E + end + after + _ = rocksdb:close_backup_engine(BE) + end; + {error, _} = E -> + E + end; + {error, _} = E -> + E + end. + +backup_relpath(#{kind := admin, alias := Alias}) -> + filename:join(["rocksdb", atom_to_list(Alias), "admin"]); +backup_relpath(#{kind := standalone, name := Name}) -> + filename:join(["rocksdb", "standalone", sanitize_backup_name(Name)]). + +sanitize_backup_name(Name) when is_atom(Name) -> + atom_to_list(Name); +sanitize_backup_name(Name) -> + %% Index/retainer or other compound names: reuse mountpoint basename. + filename:basename(mnesia_rocksdb_lib:data_mountpoint(Name), ".extrdb"). + +ensure_dir_tree(Dir) -> + case filelib:is_dir(Dir) of + true -> + ok; + false -> + case filelib:ensure_dir(filename:join(Dir, "dummy")) of + ok -> + case file:make_dir(Dir) of + ok -> ok; + {error, eexist} -> ok; + {error, _} = E -> E + end; + {error, _} = E -> + E + end + end. + +latest_backup_id(BE) -> + %% Erase rocksdb's incomplete -spec (list of #{id}) so we can handle the + %% real NIF shape ({ok, [#{backup_id => ...}]}). + Raw = binary_to_term(term_to_binary(rocksdb:get_backup_info(BE))), + case normalize_backup_infos(Raw) of + [] -> + {error, no_backup_info}; + Infos -> + {ok, lists:max([backup_info_id(I) || I <- Infos])} + end. + +-spec normalize_backup_infos(term()) -> [term()]. +normalize_backup_infos({ok, Infos}) when is_list(Infos) -> + Infos; +normalize_backup_infos(Infos) when is_list(Infos) -> + Infos; +normalize_backup_infos(_) -> + []. + +-spec backup_info_id(term()) -> non_neg_integer(). +backup_info_id(Info) when is_map(Info) -> + case maps:find(id, Info) of + {ok, Id} when is_integer(Id), Id >= 0 -> + Id; + error -> + case maps:find(backup_id, Info) of + {ok, Id} when is_integer(Id), Id >= 0 -> + Id; + _ -> + error({missing_backup_id, Info}) + end; + _ -> + error({invalid_backup_id, Info}) + end; +backup_info_id(Other) -> + error({invalid_backup_info, Other}). + +maybe_verify_backup(BE, BackupId, Opts) -> + case maps:get(verify, Opts, false) of + true -> + rocksdb:verify_backup(BE, BackupId); + false -> + ok + end. + +write_manifest(Dir, NonRdbTabs, RdbTabs, DbInfos) -> + Manifest = + #{ version => 1 + , created => erlang:system_time(second) + , node => node() + , mnesia_directory => mnesia:system_info(directory) + , mnesia_backup => "mnesia.BUP" + , mnesia_tables => NonRdbTabs + , rocksdb_tables => RdbTabs + , dbs => [maps:with([kind, alias, name, mountpoint, + backup_relpath, backup_id], I) + || I <- DbInfos] + }, + Path = filename:join(Dir, "MANIFEST"), + Bin = unicode:characters_to_binary(io_lib:format("~p.~n", [Manifest])), + case file:write_file(Path, Bin) of + ok -> ok; + {error, _} = E -> E + end. + +%% =================================================================== +%% Hybrid restore +%% =================================================================== + +do_restore(Dir, Opts) -> + case read_manifest(Dir) of + {ok, Manifest} -> + case maybe_stop_mnesia(Opts) of + ok -> + restore_body(Dir, Manifest, Opts); + {error, _} = E -> + E + end; + {error, _} = E -> + E + end. + +restore_body(Dir, Manifest, Opts) -> + SkipMnesia = maps:get(skip_mnesia, Opts, false), + SkipRocks = maps:get(skip_rocksdb, Opts, false), + MnesiaDir = target_mnesia_dir(Opts, Manifest), + case wait_admin_down(100, 50) of + ok -> + restore_body1(Dir, Manifest, Opts, MnesiaDir, + SkipMnesia, SkipRocks); + {error, _} = E -> + E + end. + +restore_body1(Dir, Manifest, Opts, MnesiaDir, SkipMnesia, SkipRocks) -> + case {SkipMnesia, SkipRocks} of + {true, true} -> + ok; + {true, false} -> + %% RocksDB only: write straight into final mountpoints. + case restore_rocksdbs(Dir, Manifest, MnesiaDir) of + ok -> maybe_start_mnesia(Opts); + {error, _} = E -> E + end; + {false, true} -> + %% Mnesia fallback only (no RocksDB data restore). + case install_and_apply_fallback(Dir, Manifest, Opts) of + ok -> maybe_start_mnesia(Opts); + {error, _} = E -> E + end; + {false, false} -> + staged_full_restore(Dir, Manifest, Opts, MnesiaDir) + end. + +%% Full hybrid restore via staging sibling (survives fallback purge). +staged_full_restore(Dir, Manifest, Opts, MnesiaDir) -> + Staging = mnesia_rocksdb_lib:restore_staging_root(MnesiaDir), + try + case prepare_staging_root(Staging) of + ok -> + case restore_rocksdbs(Dir, Manifest, Staging) of + ok -> + set_restore_mountpoint_root(Staging), + case install_and_apply_fallback(Dir, Manifest, Opts) of + ok -> + finalize_staging(Staging, Manifest, MnesiaDir, + Opts); + {error, _} = E -> + clear_restore_mountpoint_root(), + E + end; + {error, _} = E -> + E + end; + {error, _} = E -> + E + end + after + clear_restore_mountpoint_root() + end. + +finalize_staging(Staging, Manifest, MnesiaDir, Opts) -> + clear_restore_mountpoint_root(), + case relocate_staging_to_mnesia(Staging, Manifest, MnesiaDir) of + ok -> + _ = clear_dir(Staging), + maybe_start_mnesia(Opts); + {error, _} = E -> + E + end. + +prepare_staging_root(Staging) -> + case clear_dir(Staging) of + ok -> + ensure_dir_tree(Staging); + {error, _} = E -> + E + end. + +set_restore_mountpoint_root(Root) -> + Abs = filename:absname(Root), + application:set_env(mnesia_rocksdb, restore_mountpoint_root, Abs), + %% Keep PT in sync when present (admin may not be running yet). + case persistent_term:get(?PT_KEY, undefined) of + undefined -> + ok; + Meta when is_map(Meta) -> + persistent_term:put( + ?PT_KEY, Meta#{ {restore_mountpoint_root} => Abs }) + end, + ok. + +clear_restore_mountpoint_root() -> + application:unset_env(mnesia_rocksdb, restore_mountpoint_root), + case persistent_term:get(?PT_KEY, undefined) of + undefined -> + ok; + Meta when is_map(Meta) -> + persistent_term:put( + ?PT_KEY, maps:remove({restore_mountpoint_root}, Meta)) + end, + ok. + +install_and_apply_fallback(Dir, Manifest, Opts) -> + case install_mnesia_fallback(Dir, Manifest, Opts) of + ok -> + apply_mnesia_fallback(); + {error, _} = E -> + E + end. + +%% Start Mnesia so FALLBACK.BUP is applied (RocksDB opens from staging if set), +%% wait for tables, then stop so files can be relocated to the real Mnesia dir. +apply_mnesia_fallback() -> + StartRes = + case mnesia:start() of + ok -> ok; + {error, {already_started, mnesia}} -> ok; + {error, _} = StartErr -> StartErr + end, + case StartRes of + ok -> + case mnesia:system_info(is_running) of + yes -> + Tabs = mnesia:system_info(tables), + case mnesia:wait_for_tables(Tabs, 120000) of + ok -> + stopped = mnesia:stop(), + wait_admin_down(100, 50); + {timeout, _} = T -> + _ = mnesia:stop(), + _ = wait_admin_down(100, 50), + {error, T}; + {error, _} = WaitErr -> + _ = mnesia:stop(), + _ = wait_admin_down(100, 50), + WaitErr + end; + _ -> + {error, fallback_apply_failed} + end; + {error, _} = Err -> + Err + end. + +relocate_staging_to_mnesia(Staging, #{dbs := Dbs}, MnesiaDir) -> + relocate_loop(Dbs, Staging, MnesiaDir). + +relocate_loop([], _Staging, _MnesiaDir) -> + ok; +relocate_loop([#{mountpoint := OrigMP} | Rest], Staging, MnesiaDir) -> + Base = filename:basename(OrigMP), + From = filename:join(Staging, Base), + To = filename:join(MnesiaDir, Base), + case relocate_one_mountpoint(From, To, MnesiaDir) of + ok -> + relocate_loop(Rest, Staging, MnesiaDir); + {error, _} = E -> + E + end. + +relocate_one_mountpoint(From, To, MnesiaDir) -> + case filelib:is_dir(From) of + false -> + {error, {missing_staged_mountpoint, From}}; + true -> + case is_under_dir(To, MnesiaDir) of + false -> + {error, {mountpoint_outside_mnesia_dir, To, MnesiaDir}}; + true -> + case clear_dir(To) of + ok -> + case ensure_parent_dir(To) of + ok -> + case file:rename(From, To) of + ok -> ok; + {error, _} = E -> E + end; + {error, _} = E -> + E + end; + {error, _} = E -> + E + end + end + end. + +ensure_parent_dir(Path) -> + filelib:ensure_dir(filename:join(Path, "dummy")). + +read_manifest(Dir) -> + Path = filename:join(Dir, "MANIFEST"), + case file:consult(Path) of + {ok, [#{version := 1} = M]} -> + {ok, M}; + {ok, [Other]} -> + {error, {invalid_manifest, Other}}; + {ok, Other} -> + {error, {invalid_manifest, Other}}; + {error, enoent} -> + {error, {missing_manifest, Path}}; + {error, _} = E -> + E + end. + +target_mnesia_dir(Opts, Manifest) -> + case maps:get(mnesia_dir, Opts, undefined) of + undefined -> + case mnesia_directory() of + undefined -> + maps:get(mnesia_directory, Manifest); + Dir -> + Dir + end; + Dir -> + filename:absname(Dir) + end. + +mnesia_directory() -> + try mnesia:system_info(directory) of + Dir when is_list(Dir); is_binary(Dir) -> filename:absname(Dir); + _ -> undefined + catch + _:_ -> + case application:get_env(mnesia, dir) of + {ok, Dir} -> filename:absname(Dir); + undefined -> undefined + end + end. + +maybe_stop_mnesia(Opts) -> + case maps:get(stop, Opts, true) of + false -> + case mnesia:system_info(is_running) of + yes -> + {error, {mnesia_running, + "stop Mnesia before restore, or pass #{stop => true}"}}; + _ -> + ok + end; + true -> + case mnesia:system_info(is_running) of + yes -> + stopped = mnesia:stop(), + wait_admin_down(100, 50); + _ -> + %% Best-effort: admin may still hold open DBs if left over. + stop_admin_if_alive(), + wait_admin_down(100, 50) + end + end. + +stop_admin_if_alive() -> + case whereis(?MODULE) of + undefined -> + ok; + Pid when is_pid(Pid) -> + try gen_server:stop(Pid, shutdown, 30000) + catch + _:_ -> ok + end + end. + +wait_admin_down(0, _Sleep) -> + case whereis(?MODULE) of + undefined -> ok; + Pid -> {error, {admin_still_running, Pid}} + end; +wait_admin_down(N, Sleep) -> + case whereis(?MODULE) of + undefined -> + ok; + _ -> + timer:sleep(Sleep), + wait_admin_down(N - 1, Sleep) + end. + +maybe_start_mnesia(Opts) -> + case maps:get(start, Opts, false) of + true -> + case mnesia:start() of + ok -> ok; + {error, {already_started, mnesia}} -> ok; + {error, _} = E -> E + end; + false -> + ok + end. + +%% Restore RocksDB engines under Root (staging sibling or final Mnesia dir). +restore_rocksdbs(Dir, #{dbs := Dbs}, Root) -> + restore_rocksdbs_loop(Dir, Dbs, Root). + +restore_rocksdbs_loop(_Dir, [], _Root) -> + ok; +restore_rocksdbs_loop(Dir, [Spec | Rest], Root) -> + case restore_one_db(Dir, Spec, Root) of + ok -> + restore_rocksdbs_loop(Dir, Rest, Root); + {error, _} = E -> + E + end. + +restore_one_db(Dir, #{ backup_relpath := Rel + , backup_id := BackupId + , mountpoint := OrigMP }, Root) -> + AbsBackup = filename:join(Dir, Rel), + TargetMP = remap_mountpoint(OrigMP, Root), + case prepare_restore_mountpoint(TargetMP, Root) of + ok -> + case rocksdb:open_backup_engine(AbsBackup) of + {ok, BE} -> + try + case rocksdb:restore_db_from_backup( + BE, BackupId, TargetMP, TargetMP) of + ok -> ok; + {error, _} = E -> E + end + after + _ = rocksdb:close_backup_engine(BE) + end; + {error, _} = E -> + E + end; + {error, _} = E -> + E + end. + +%% Keep only the basename under Root so restores work when the absolute path +%% differs from the backup host (and for staging siblings). +remap_mountpoint(OrigMP, Root) -> + filename:join(Root, filename:basename(OrigMP)). + +prepare_restore_mountpoint(TargetMP, Root) -> + case is_under_dir(TargetMP, Root) of + true -> + case clear_dir(TargetMP) of + ok -> + ensure_dir_tree(TargetMP); + {error, _} = E -> + E + end; + false -> + {error, {mountpoint_outside_root, TargetMP, Root}} + end. + +is_under_dir(Path, Root) -> + AbsPath = filename:absname(Path), + AbsRoot = filename:absname(Root), + lists:prefix(AbsRoot ++ "/", AbsPath ++ "/") + orelse AbsPath =:= AbsRoot. + +clear_dir(Dir) -> + case filelib:is_dir(Dir) of + true -> + file:del_dir_r(Dir); + false -> + ok + end. + +install_mnesia_fallback(Dir, Manifest, Opts) -> + Rel = maps:get(mnesia_backup, Manifest, "mnesia.BUP"), + Bup = filename:join(Dir, Rel), + case filelib:is_regular(Bup) of + false -> + {error, {missing_mnesia_backup, Bup}}; + true -> + Mod = maps:get(mnesia_backup_module, Opts, + default_backup_module()), + Args = [ {module, Mod} + , {scope, local} + ], + case mnesia:install_fallback(Bup, Args) of + ok -> ok; + {error, _} = E -> E + end + end. + +default_backup_module() -> + try mnesia:system_info(backup_module) + catch + _:_ -> mnesia_backup + end. diff --git a/src/mnesia_rocksdb_lib.erl b/src/mnesia_rocksdb_lib.erl index d42b340..6f5ba28 100644 --- a/src/mnesia_rocksdb_lib.erl +++ b/src/mnesia_rocksdb_lib.erl @@ -11,6 +11,8 @@ , data_mountpoint/1 , create_mountpoint/1 , tabname/1 + , restore_mountpoint_root/0 + , restore_staging_root/1 ]). -export([ default_encoding/3 @@ -59,8 +61,16 @@ create_mountpoint(Tab) -> file:make_dir(MPd), ok; true -> - Dir = mnesia_lib:dir(), - case lists:prefix(Dir, MPd) of + AllowedRoots = + case restore_mountpoint_root() of + undefined -> [mnesia_lib:dir()]; + Staging -> [mnesia_lib:dir(), Staging] + end, + case lists:any( + fun(Root) -> + lists:prefix(filename:absname(Root), + filename:absname(MPd)) + end, AllowedRoots) of true -> ok; false -> @@ -68,9 +78,37 @@ create_mountpoint(Tab) -> end end. +%% @doc Absolute RocksDB directory for a logical table / admin DB. +%% +%% During hybrid restore, `application:get_env(mnesia_rocksdb, +%% restore_mountpoint_root)` may point at a sibling staging directory so +%% RocksDB data survives Mnesia's fallback purge of the real Mnesia dir. +-spec data_mountpoint(term()) -> file:filename(). data_mountpoint(Tab) -> - Dir = mnesia_monitor:get_env(dir), - filename:join(Dir, tabname(Tab) ++ ".extrdb"). + Base = tabname(Tab) ++ ".extrdb", + case restore_mountpoint_root() of + undefined -> + filename:join(mnesia_monitor:get_env(dir), Base); + Root -> + filename:join(Root, Base) + end. + +%% @doc Staging root used while applying a hybrid restore, if any. +-spec restore_mountpoint_root() -> file:filename() | undefined. +restore_mountpoint_root() -> + case application:get_env(mnesia_rocksdb, restore_mountpoint_root) of + {ok, Root} when is_list(Root); is_binary(Root) -> + filename:absname(Root); + _ -> + undefined + end. + +%% @doc Sibling of the Mnesia directory for staged RocksDB restore data. +-spec restore_staging_root(file:filename()) -> file:filename(). +restore_staging_root(MnesiaDir) -> + Abs = filename:absname(MnesiaDir), + filename:join(filename:dirname(Abs), + filename:basename(Abs) ++ ".rocksdb_restore"). tabname({admin, Alias}) -> "mnesia_rocksdb-" ++ atom_to_list(Alias) ++ "-_db"; diff --git a/test/mnesia_rocksdb_backup_SUITE.erl b/test/mnesia_rocksdb_backup_SUITE.erl new file mode 100644 index 0000000..1cd7d12 --- /dev/null +++ b/test/mnesia_rocksdb_backup_SUITE.erl @@ -0,0 +1,288 @@ +%% -*- mode: erlang; erlang-indent-level: 4; indent-tabs-mode: nil -*- +-module(mnesia_rocksdb_backup_SUITE). + +-export([ all/0 + , suite/0 + , groups/0 + , init_per_suite/1 + , end_per_suite/1 + , init_per_group/2 + , end_per_group/2 + , init_per_testcase/2 + , end_per_testcase/2 + ]). + +-export([ hybrid_backup_cf_and_disc/1 + , hybrid_backup_with_standalone/1 + , backup_rejects_nonempty_dir/1 + , backup_with_verify/1 + , restore_roundtrip_cf_and_disc/1 + , restore_roundtrip_with_standalone/1 + ]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +suite() -> + []. + +all() -> + [{group, all_tests}]. + +groups() -> + [{all_tests, [sequence], + [ hybrid_backup_cf_and_disc + , hybrid_backup_with_standalone + , backup_rejects_nonempty_dir + , backup_with_verify + , restore_roundtrip_cf_and_disc + , restore_roundtrip_with_standalone + ]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_, Config) -> + Config. + +end_per_group(_, _Config) -> + ok. + +init_per_testcase(_TC, Config) -> + mnesia:stop(), + ok = mnesia_rocksdb_tlib:start_mnesia(reset), + Config. + +end_per_testcase(_TC, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +%% disc_copies (not ram_copies): with ram_overrides_dump=false, Mnesia +%% checkpoints prefer the dumped image for ram_copies, so pure RAM writes +%% are a poor stand-in for "persistent non-RocksDB tables". +hybrid_backup_cf_and_disc(Config) -> + create_rdb_table(t, []), + {atomic, ok} = + mnesia:create_table(d, [{disc_copies, [node()]}, + {attributes, [k, v]}]), + ok = mnesia:wait_for_tables([t, d], 5000), + + [ok = mnesia:dirty_write({t, K, V}) || {K, V} <- [{a, 1}, {b, 2}, {c, 3}]], + [ok = mnesia:dirty_write({d, K, V}) || {K, V} <- [{x, 10}, {y, 20}]], + + BackupDir = backup_dir(Config, "cf_disc"), + ?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)), + + assert_layout(BackupDir), + Manifest = read_manifest(BackupDir), + ct:log("MANIFEST = ~p", [Manifest]), + + ?assertEqual(1, maps:get(version, Manifest)), + ?assert(lists:member(t, maps:get(rocksdb_tables, Manifest))), + ?assertNot(lists:member(t, maps:get(mnesia_tables, Manifest))), + ?assert(lists:member(schema, maps:get(mnesia_tables, Manifest))), + ?assert(lists:member(d, maps:get(mnesia_tables, Manifest))), + + Dbs = maps:get(dbs, Manifest), + ?assert(lists:any(fun(#{kind := admin}) -> true; (_) -> false end, Dbs)), + + assert_admin_backup_ok(BackupDir, Dbs), + assert_mnesia_bup_has_table(BackupDir, d), + assert_mnesia_bup_lacks_table_data(BackupDir, t), + ok. + +hybrid_backup_with_standalone(Config) -> + create_rdb_table(t, []), + create_rdb_table(ts, [{user_properties, [{rocksdb_standalone, true}]}]), + {atomic, ok} = + mnesia:create_table(d, [{disc_copies, [node()]}, + {attributes, [k, v]}]), + ok = mnesia:wait_for_tables([t, ts, d], 5000), + + ok = mnesia:dirty_write({t, 1, one}), + ok = mnesia:dirty_write({ts, 2, two}), + ok = mnesia:dirty_write({d, 3, three}), + + #{type := standalone} = mrdb:get_ref(ts), + + BackupDir = backup_dir(Config, "standalone"), + ?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)), + + Manifest = read_manifest(BackupDir), + RdbTabs = maps:get(rocksdb_tables, Manifest), + ?assert(lists:member(t, RdbTabs)), + ?assert(lists:member(ts, RdbTabs)), + ?assert(lists:member(d, maps:get(mnesia_tables, Manifest))), + + Dbs = maps:get(dbs, Manifest), + ?assert(lists:any(fun(#{kind := standalone, name := ts}) -> true; + (_) -> false end, Dbs)), + ?assert(lists:any(fun(#{kind := admin}) -> true; (_) -> false end, Dbs)), + + lists:foreach( + fun(#{backup_relpath := Rel, backup_id := Id}) -> + Abs = filename:join(BackupDir, Rel), + {ok, BE} = rocksdb:open_backup_engine(Abs), + try + ?assertEqual(ok, rocksdb:verify_backup(BE, Id)) + after + rocksdb:close_backup_engine(BE) + end + end, Dbs), + ok. + +backup_rejects_nonempty_dir(Config) -> + create_rdb_table(t, []), + ok = mnesia:wait_for_tables([t], 5000), + ok = mnesia:dirty_write({t, 1, a}), + + BackupDir = backup_dir(Config, "nonempty"), + ok = filelib:ensure_dir(filename:join(BackupDir, "noise")), + case file:make_dir(BackupDir) of + ok -> ok; + {error, eexist} -> ok + end, + ok = file:write_file(filename:join(BackupDir, "noise"), <<"x">>), + + ?assertMatch({error, {not_empty, _}}, mnesia_rocksdb:backup(BackupDir)), + ok. + +backup_with_verify(Config) -> + create_rdb_table(t, []), + ok = mnesia:wait_for_tables([t], 5000), + ok = mnesia:dirty_write({t, 1, a}), + + BackupDir = backup_dir(Config, "verify"), + ?assertEqual(ok, mnesia_rocksdb:backup(BackupDir, #{verify => true})), + assert_layout(BackupDir), + ok. + +restore_roundtrip_cf_and_disc(Config) -> + create_rdb_table(t, []), + {atomic, ok} = + mnesia:create_table(d, [{disc_copies, [node()]}, + {attributes, [k, v]}]), + ok = mnesia:wait_for_tables([t, d], 5000), + + [ok = mnesia:dirty_write({t, K, V}) || {K, V} <- [{a, 1}, {b, 2}]], + [ok = mnesia:dirty_write({d, K, V}) || {K, V} <- [{x, 10}, {y, 20}]], + + BackupDir = backup_dir(Config, "restore_cf"), + ?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)), + + %% Mutate after backup — restore must discard these changes. + ok = mnesia:dirty_write({t, a, 999}), + ok = mnesia:dirty_write({t, c, 3}), + ok = mnesia:dirty_delete({d, x}), + ok = mnesia:dirty_write({d, z, 30}), + + ?assertEqual(ok, mnesia_rocksdb:restore(BackupDir, #{start => true})), + ?assertEqual(ok, mnesia:wait_for_tables([t, d], 10000)), + + ?assertEqual([{t, a, 1}, {t, b, 2}], + lists:sort(mnesia:dirty_match_object({t, '_', '_'}))), + ?assertEqual([{d, x, 10}, {d, y, 20}], + lists:sort(mnesia:dirty_match_object({d, '_', '_'}))), + ok. + +restore_roundtrip_with_standalone(Config) -> + create_rdb_table(t, []), + create_rdb_table(ts, [{user_properties, [{rocksdb_standalone, true}]}]), + {atomic, ok} = + mnesia:create_table(d, [{disc_copies, [node()]}, + {attributes, [k, v]}]), + ok = mnesia:wait_for_tables([t, ts, d], 5000), + + ok = mnesia:dirty_write({t, 1, one}), + ok = mnesia:dirty_write({ts, 2, two}), + ok = mnesia:dirty_write({d, 3, three}), + #{type := standalone} = mrdb:get_ref(ts), + + BackupDir = backup_dir(Config, "restore_sa"), + ?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)), + + ok = mnesia:dirty_write({t, 1, changed}), + ok = mnesia:dirty_write({ts, 2, changed}), + ok = mnesia:dirty_write({d, 3, changed}), + + ?assertEqual(ok, mnesia_rocksdb:restore(BackupDir, #{start => true})), + ?assertEqual(ok, mnesia:wait_for_tables([t, ts, d], 10000)), + + ?assertEqual([{t, 1, one}], mnesia:dirty_read(t, 1)), + ?assertEqual([{ts, 2, two}], mnesia:dirty_read(ts, 2)), + ?assertEqual([{d, 3, three}], mnesia:dirty_read(d, 3)), + #{type := standalone} = mrdb:get_ref(ts), + ok. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +create_rdb_table(Name, Extra) -> + {atomic, ok} = + mnesia:create_table(Name, [{rdb, [node()]}, + {attributes, [k, v]} | Extra]), + ok. + +backup_dir(Config, Name) -> + filename:join(?config(priv_dir, Config), "backup_" ++ Name). + +assert_layout(BackupDir) -> + ?assert(filelib:is_regular(filename:join(BackupDir, "MANIFEST"))), + ?assert(filelib:is_regular(filename:join(BackupDir, "mnesia.BUP"))), + ?assert(filelib:is_dir(filename:join(BackupDir, "rocksdb"))). + +read_manifest(BackupDir) -> + Path = filename:join(BackupDir, "MANIFEST"), + {ok, [Manifest]} = file:consult(Path), + Manifest. + +assert_admin_backup_ok(BackupDir, Dbs) -> + [Admin | _] = [D || #{kind := admin} = D <- Dbs], + #{backup_relpath := Rel, backup_id := Id} = Admin, + Abs = filename:join(BackupDir, Rel), + {ok, BE} = rocksdb:open_backup_engine(Abs), + try + Infos = case rocksdb:get_backup_info(BE) of + {ok, L} -> L; + L when is_list(L) -> L + end, + ?assertNotEqual([], Infos), + ?assertEqual(ok, rocksdb:verify_backup(BE, Id)) + after + rocksdb:close_backup_engine(BE) + end. + +assert_mnesia_bup_has_table(BackupDir, Tab) -> + Bup = filename:join(BackupDir, "mnesia.BUP"), + %% Fun is applied per backup item (not per batch). + {ok, Found} = + mnesia:traverse_backup( + Bup, mnesia_backup, read_only, read_only, + fun(Rec, _Acc) when is_tuple(Rec), element(1, Rec) =:= Tab -> + {[Rec], true}; + (Rec, Acc) -> + {[Rec], Acc} + end, false), + ?assertEqual(true, Found). + +assert_mnesia_bup_lacks_table_data(BackupDir, Tab) -> + Bup = filename:join(BackupDir, "mnesia.BUP"), + {ok, Found} = + mnesia:traverse_backup( + Bup, mnesia_backup, read_only, read_only, + fun({schema, _, _} = Rec, Acc) -> + {[Rec], Acc}; + (Rec, _Acc) when is_tuple(Rec), element(1, Rec) =:= Tab -> + {[Rec], true}; + (Rec, Acc) -> + {[Rec], Acc} + end, false), + %% Data records for Tab should not appear; schema may still mention Tab. + ?assertEqual(false, Found).