QPQ-AG/mnesia_rocksdb
11
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Add hybrid mnesia_rocksdb backup and restore #12

Open
uwiger wants to merge 1 commits from uw-backup-restore into master
pull from: uw-backup-restore
merge into: QPQ-AG:master
Conversation 0 Commits 1 Files Changed 8
+1262 -26
8 changed files with 1262 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 32384f76fe - Show all commits
README.md
+60 -1
View File
@@ -43,7 +43,7 @@ is provided.
* sext (included as dependency) * sext (included as dependency)
* Erlang/OTP 21.0 or newer (https://github.com/erlang/otp) * Erlang/OTP 27.0 or newer (https://github.com/erlang/otp)
@@ -115,6 +115,65 @@ source code in `mnesia_rocksdb_tuning.erl` and `mnesia_rocksdb_params.erl`.
Contributions are welcome. Contributions are welcome.
#### <a name="Backup">Backup</a> ####
`mnesia_rocksdb:backup(Dir)` creates a **local hybrid backup** of Mnesia
metadata/tables and RocksDB data. Unlike `mnesia:backup/1`, RocksDB-backed
tables are not dumped record-by-record through Mnesia; they are captured with
the [RocksDB backup engine](https://rocksdb.hexdocs.pm/1.9.0/how_to_backup_rocksdb.html).
Layout of `Dir` after a successful backup:
```
Dir/
MANIFEST % restore metadata (Erlang term)
mnesia.BUP % schema + non-RocksDB tables
rocksdb/<alias>/admin/ % backup engine dir for each alias admin DB
rocksdb/standalone/... % one dir per open standalone RocksDB table
```
Algorithm (local node only):
1. Activate a Mnesia checkpoint on **schema and non-RocksDB tables only**
(so large RocksDB tables do not get expensive checkpoint retainers).
2. For each open RocksDB instance (admin DB per alias, and standalone tables),
run `rocksdb:create_new_backup/2`.
3. Dump the checkpoint with `mnesia:backup_checkpoint/3`, excluding RocksDB
table data. Schema still describes RocksDB tables.
4. Write `MANIFEST`.
Options map (second argument to `backup/2`):
* `{verify, true}` — call `rocksdb:verify_backup/2` after each RocksDB backup
* `{mnesia_backup_module, Mod}` — override the Mnesia backup media module
#### Restore
`mnesia_rocksdb:restore(Dir)` restores a hybrid backup offline (local node).
Mnesia applies fallbacks by **purging** the Mnesia directory. RocksDB data is
therefore restored first into a **sibling staging directory** and mountpoints
are redirected until the fallback has finished:
1. Stop Mnesia (closes RocksDB handles).
2. Restore each RocksDB DB into
`"<basename(mnesia_dir)>.rocksdb_restore/"` (sibling of the Mnesia dir).
3. Set application env `mnesia_rocksdb.restore_mountpoint_root` so
`data_mountpoint/1` opens DBs from staging.
4. Install `Dir/mnesia.BUP` and start Mnesia (fallback purges the real Mnesia
dir; RocksDB tables load from staging with full data). During this window,
`delete_table` does not destroy staged RocksDB files.
5. Stop Mnesia, clear the redirect, move staged `*.extrdb` directories into the
real Mnesia directory.
6. Optionally start Mnesia (`#{start => true}`).
Common options for `restore/2`:
* `{stop, true|false}` — stop Mnesia before restore (default `true`)
* `{start, true|false}` — start Mnesia after finalize (default `false`)
* `{mnesia_dir, Dir}` — override target Mnesia directory
* `{skip_rocksdb, true}` / `{skip_mnesia, true}` — partial restore
#### <a name="Caveats">Caveats</a> #### #### <a name="Caveats">Caveats</a> ####
Avoid placing `bag` tables in RocksDB. Although they work, each write Avoid placing `bag` tables in RocksDB. Although they work, each write
doc/README.md
+1 -1
View File
@@ -43,7 +43,7 @@ is provided.
* sext (included as dependency) * sext (included as dependency)
* Erlang/OTP 21.0 or newer (https://github.com/erlang/otp) * Erlang/OTP 27.0 or newer (https://github.com/erlang/otp)
doc/plugin-userguide.md
+1 -1
View File
@@ -45,7 +45,7 @@ backs up data.
- rocksdb (included as dependency) - rocksdb (included as dependency)
- sext (included as dependency) - sext (included as dependency)
- Erlang/OTP 22.0 or newer (https://github.com/erlang/otp) - Erlang/OTP 27.0 or newer (https://github.com/erlang/otp)
### Getting started ### Getting started
rebar.config
+1 -1
View File
@@ -1,5 +1,5 @@
%% -*- erlang -*- %% -*- erlang -*-
{minimum_otp_vsn, "21.0"}. {minimum_otp_vsn, "27.0"}.
{deps, {deps,
[ [
src/mnesia_rocksdb.erl
+43 -1
View File
@@ -49,7 +49,11 @@
-export([register/0, -export([register/0,
register/1, register/1,
default_alias/0]). default_alias/0,
backup/1,
backup/2,
restore/1,
restore/2]).
%% %%
%% DEBUG API %% DEBUG API
@@ -218,6 +222,44 @@ get_ref(Tab) ->
default_alias() -> default_alias() ->
rocksdb_copies. rocksdb_copies.
%% @doc Hybrid local backup of Mnesia + RocksDB tables.
%%
%% Similar intent to `mnesia:backup/1`, but RocksDB-backed tables are backed up
%% with the RocksDB backup engine instead of record-level iteration.
%%
%% `Dir` must be a non-existent or empty directory. On success it contains:
%% - `MANIFEST` — restore metadata
%% - `mnesia.BUP` — schema and non-RocksDB tables
%% - `rocksdb/...` — native RocksDB backups for each open DB
%%
%% Local node only. See `mnesia_rocksdb_admin:backup/2` for details and options.
%% @end
-spec backup(file:filename_all()) -> ok | {error, term()}.
backup(Dir) ->
mnesia_rocksdb_admin:backup(Dir, #{}).
%% @equiv mnesia_rocksdb_admin:backup/2
-spec backup(file:filename_all(), map()) -> ok | {error, term()}.
backup(Dir, Opts) when is_map(Opts) ->
mnesia_rocksdb_admin:backup(Dir, Opts).
%% @doc Restore a hybrid backup created by {@link backup/1}.
%%
%% Offline restore: RocksDB data is restored into a sibling staging directory
%% first; Mnesia fallback runs with mountpoints redirected there (so the
%% fallback purge does not wipe RocksDB data); then staged dirs are moved into
%% the real Mnesia directory. Optionally starts Mnesia (`#{start => true}`).
%%
%% See `mnesia_rocksdb_admin:restore/2` for options and the full sequence.
%% @end
-spec restore(file:filename_all()) -> ok | {error, term()}.
restore(Dir) ->
mnesia_rocksdb_admin:restore(Dir, #{}).
%% @equiv mnesia_rocksdb_admin:restore/2
-spec restore(file:filename_all(), map()) -> ok | {error, term()}.
restore(Dir, Opts) when is_map(Opts) ->
mnesia_rocksdb_admin:restore(Dir, Opts).
encode_key(Key) -> encode_key(Key) ->
mnesia_rocksdb_lib:encode_key(Key, sext). mnesia_rocksdb_lib:encode_key(Key, sext).
src/mnesia_rocksdb_admin.erl
+812 -3
View File
@@ -17,6 +17,11 @@
, request_ref/2 %% (Alias, Name) -> {ok, Ref} | {error, _} , request_ref/2 %% (Alias, Name) -> {ok, Ref} | {error, _}
, close_table/2 , close_table/2
, clear_table/1 , clear_table/1
, backup/1 %% (Dir) -> ok | {error, _}
, backup/2 %% (Dir, Opts) -> ok | {error, _}
, restore/1 %% (Dir) -> ok | {error, _}
, restore/2 %% (Dir, Opts) -> ok | {error, _}
, list_open_dbs/0 %% () -> [db_backup_spec()]
]). ]).
-export([ migrate_standalone/2 -export([ migrate_standalone/2
@@ -109,7 +114,35 @@
| {abort_migration, tabname()} | {abort_migration, tabname()}
| {prep_close, table()} | {prep_close, table()}
| {close_table, table()} | {close_table, table()}
| {clear_table, table() | cf() }. | {clear_table, table() | cf() }
| list_open_dbs.
%% Spec describing an open RocksDB instance eligible for backup.
-type db_backup_spec() :: #{ kind := admin | standalone
, alias := alias()
, name := table()
, db_ref := db_ref()
, mountpoint := file:filename()
}.
-type backup_opts() :: #{ verify => boolean()
, mnesia_backup_module => module()
}.
%% Restore options:
%% - `mnesia_dir': target Mnesia directory (default: configured Mnesia dir)
%% - `stop': stop Mnesia if running before restore (default true)
%% - `start': start Mnesia after restore (default false)
%% - `mnesia_backup_module': module for reading mnesia.BUP
%% - `skip_rocksdb': only install Mnesia fallback (default false)
%% - `skip_mnesia': only restore RocksDB DBs (default false)
-type restore_opts() :: #{ mnesia_dir => file:filename_all()
, stop => boolean()
, start => boolean()
, mnesia_backup_module => module()
, skip_rocksdb => boolean()
, skip_mnesia => boolean()
}.
-type reason() :: any(). -type reason() :: any().
-type reply() :: any(). -type reply() :: any().
@@ -164,7 +197,13 @@ erase_pt_list(Names) ->
check_application_defaults(Meta) -> check_application_defaults(Meta) ->
Value = application:get_env(mnesia_rocksdb, mnesia_compatible_aborts, false), Value = application:get_env(mnesia_rocksdb, mnesia_compatible_aborts, false),
Meta#{ {mnesia_compatible_aborts} => Value }. Meta1 = Meta#{ {mnesia_compatible_aborts} => Value },
case application:get_env(mnesia_rocksdb, restore_mountpoint_root) of
{ok, Root} when is_list(Root); is_binary(Root) ->
Meta1#{ {restore_mountpoint_root} => filename:absname(Root) };
_ ->
Meta1
end.
get_cached_env(Key, Default) -> get_cached_env(Key, Default) ->
maps:get({Key}, meta(), Default). maps:get({Key}, meta(), Default).
@@ -200,6 +239,96 @@ get_pt(Name, Default) ->
set_and_cache_env(Key, Value) -> set_and_cache_env(Key, Value) ->
gen_server:call(?MODULE, {set_and_cache_env, Key, Value}). gen_server:call(?MODULE, {set_and_cache_env, Key, Value}).
%% @doc Hybrid backup of Mnesia metadata/non-RocksDB tables and RocksDB DBs.
%%
%% Creates a directory layout:
%% ```
%% Dir/
%% MANIFEST
%% mnesia.BUP
%% rocksdb/<alias>/admin/
%% rocksdb/standalone/<name>/
%% '''
%%
%% Steps:
%% 1. Activate a Mnesia checkpoint covering schema and non-RocksDB tables only
%% (avoids heavy RocksDB retainer tables).
%% 2. For each open RocksDB instance (admin DBs and standalone tables), create a
%% RocksDB-native backup via the backup engine.
%% 3. Dump the checkpoint with `mnesia:backup_checkpoint/3`, excluding RocksDB
%% table data (schema still describes those tables).
%% 4. Write a MANIFEST describing the layout for a future restore.
%%
%% Local-node only. Cross-store consistency is best-effort within the checkpoint
%% window.
%%
%% Options:
%% - `verify` (boolean, default false): call `rocksdb:verify_backup/2`
%% - `mnesia_backup_module` (module): override Mnesia backup media module
-spec backup(file:filename_all()) -> ok | {error, term()}.
backup(Dir) ->
backup(Dir, #{}).
-spec backup(file:filename_all(), backup_opts()) -> ok | {error, term()}.
backup(Dir, Opts) when is_map(Opts) ->
ensure_started(),
case mnesia:system_info(is_running) of
yes ->
case prepare_backup_dir(Dir) of
ok ->
try do_backup(filename:absname(Dir), Opts)
catch
error:Reason:Stack ->
{error, {Reason, Stack}};
throw:{error, _} = Err ->
Err
end;
{error, _} = E ->
E
end;
Other ->
{error, {mnesia_not_running, Other}}
end.
%% @doc Restore a hybrid backup previously created by {@link backup/1}.
%%
%% Offline restore (local node). Mnesia fallback application **purges** the
%% Mnesia directory, so RocksDB data is restored first into a **sibling**
%% staging directory and mountpoints are redirected while the fallback runs:
%%
%% 1. Stop Mnesia (closes RocksDB handles).
%% 2. Restore each RocksDB DB into
%% `"<mnesia_dir_basename>.rocksdb_restore/"` (sibling of the Mnesia dir).
%% 3. Set `restore_mountpoint_root` so `data_mountpoint/1` uses the staging
%% location.
%% 4. Install `mnesia.BUP` and start Mnesia (fallback purges the real Mnesia
%% dir; RocksDB tables open from staging with full data).
%% 5. Stop Mnesia, clear the redirect, move staged `*.extrdb` into the real
%% Mnesia directory.
%% 6. Optionally start Mnesia (`#{start => true}`).
-spec restore(file:filename_all()) -> ok | {error, term()}.
restore(Dir) ->
restore(Dir, #{}).
-spec restore(file:filename_all(), restore_opts()) -> ok | {error, term()}.
restore(Dir, Opts) when is_map(Opts) ->
try do_restore(filename:absname(Dir), Opts)
catch
error:Reason:Stack ->
{error, {Reason, Stack}};
throw:{error, _} = Err ->
Err
end.
%% @doc Snapshot of open RocksDB instances (admin + standalone).
-spec list_open_dbs() -> [db_backup_spec()].
list_open_dbs() ->
ensure_started(),
case gen_server:call(?MODULE, {[], list_open_dbs}, infinity) of
{ok, Dbs} -> Dbs;
{error, _} = E -> error(E)
end.
create_table(Alias, Name, Props) -> create_table(Alias, Name, Props) ->
call(Alias, {create_table, Name, Props}). call(Alias, {create_table, Name, Props}).
@@ -616,6 +745,8 @@ handle_call({[], {get_ref, Name}}, _From, St) ->
_ -> _ ->
{reply, {error, not_found}, St} {reply, {error, not_found}, St}
end; end;
handle_call({[], list_open_dbs}, _From, St) ->
{reply, {ok, list_open_dbs_from_st(St)}, St};
handle_call({Alias, Req}, _From, St) -> handle_call({Alias, Req}, _From, St) ->
handle_call_for_alias(Alias, Req, St); handle_call_for_alias(Alias, Req, St);
handle_call(_Req, _From, St) -> handle_call(_Req, _From, St) ->
@@ -1604,11 +1735,19 @@ do_close_table(Alias, Name, Backend, #st{standalone = Ts} = St) ->
end. end.
do_delete_table(Alias, Name, Backend, #st{} = St) -> do_delete_table(Alias, Name, Backend, #st{} = St) ->
case restore_staging_active() of
true ->
%% Hybrid restore: RocksDB lives in a staging sibling. Mnesia
%% fallback calls delete_table before reopening; must not destroy
%% staged files or drop CFs that still hold restored data.
{ok, St};
false ->
case find_cf(Alias, Name, Backend, St) of case find_cf(Alias, Name, Backend, St) of
{ok, Where} -> {ok, Where} ->
erase_pt(Name), erase_pt(Name),
case Where of case Where of
#{db_ref := DbRef, cf_handle := CfH, type := column_family} -> #{db_ref := DbRef, cf_handle := CfH,
type := column_family} ->
rocksdb:drop_column_family(DbRef, CfH), rocksdb:drop_column_family(DbRef, CfH),
drop_cached_cf(tab_to_cf_name(Name), CfH), drop_cached_cf(tab_to_cf_name(Name), CfH),
rocksdb:destroy_column_family(DbRef, CfH), rocksdb:destroy_column_family(DbRef, CfH),
@@ -1619,8 +1758,12 @@ do_delete_table(Alias, Name, Backend, #st{} = St) ->
end; end;
error -> error ->
{error, not_found} {error, not_found}
end
end. end.
restore_staging_active() ->
mnesia_rocksdb_lib:restore_mountpoint_root() =/= undefined.
load_admin_db(Alias, Opts) -> load_admin_db(Alias, Opts) ->
DbName = {admin, Alias}, DbName = {admin, Alias},
open_db(DbName, Alias, Opts, [DbName], true). open_db(DbName, Alias, Opts, [DbName], true).
@@ -2377,3 +2520,669 @@ destroy_db(MPd, Opts, RetriesLeft, _) ->
get_retries() -> 30. get_retries() -> 30.
get_retry_delay() -> 10000. get_retry_delay() -> 10000.
%% ===================================================================
%% Hybrid backup (Mnesia checkpoint + RocksDB backup engine)
%% ===================================================================
list_open_dbs_from_st(#st{backends = Bs, standalone = Ts}) ->
Admin =
maps:fold(
fun(Alias, #{db_ref := DbRef, mountpoint := MP}, Acc) ->
[#{ kind => admin
, alias => Alias
, name => {admin, Alias}
, db_ref => DbRef
, mountpoint => MP } | Acc];
(_, _, Acc) ->
Acc
end, [], Bs),
Standalone =
maps:fold(
fun({Alias, Name}, #{db_ref := DbRef, mountpoint := MP} = R, Acc) ->
case maps:get(status, R, open) of
closed ->
Acc;
_ ->
[#{ kind => standalone
, alias => Alias
, name => Name
, db_ref => DbRef
, mountpoint => MP } | Acc]
end;
(_, _, Acc) ->
Acc
end, [], Ts),
Admin ++ Standalone.
prepare_backup_dir(Dir) ->
case filelib:is_dir(Dir) of
true ->
case file:list_dir(Dir) of
{ok, []} ->
ok;
{ok, _} ->
{error, {not_empty, Dir}};
{error, _} = E ->
E
end;
false ->
case filelib:ensure_dir(filename:join(Dir, "dummy")) of
ok ->
case file:make_dir(Dir) of
ok -> ok;
{error, eexist} ->
%% Race: created between is_dir and make_dir
prepare_backup_dir(Dir);
{error, _} = E ->
E
end;
{error, _} = E ->
E
end
end.
do_backup(Dir, Opts) ->
{RdbTabs, NonRdbTabs} = classify_tables(),
CpArgs = [ {ram_overrides_dump, false}
, {max, NonRdbTabs}
, {allow_remote, false} ],
case mnesia:activate_checkpoint(CpArgs) of
{ok, CpName, _Nodes} ->
try
Dbs = list_open_dbs(),
case backup_rocksdbs(Dir, Dbs, Opts) of
{ok, DbInfos} ->
Bup = filename:join(Dir, "mnesia.BUP"),
case backup_mnesia_checkpoint(CpName, Bup, NonRdbTabs, Opts) of
ok ->
write_manifest(Dir, NonRdbTabs, RdbTabs, DbInfos);
{error, _} = E ->
E
end;
{error, _} = E ->
E
end
after
_ = mnesia:deactivate_checkpoint(CpName)
end;
{error, _} = E ->
E
end.
classify_tables() ->
All = mnesia:system_info(tables),
Rdb = [T || T <- All, is_local_rdb_table(T)],
NonRdb = All -- Rdb,
{lists:sort(Rdb), lists:sort(NonRdb)}.
is_local_rdb_table(schema) ->
false;
is_local_rdb_table(Tab) ->
try mnesia:table_info(Tab, storage_type) of
{ext, _Alias, mnesia_rocksdb} -> true;
_ -> false
catch
_:_ -> false
end.
backup_mnesia_checkpoint(CpName, BupPath, NonRdbTabs, Opts) ->
Mod = maps:get(mnesia_backup_module, Opts,
mnesia:system_info(backup_module)),
%% mnesia:backup_checkpoint/3 is typed as module()-only; the implementation
%% (and mnesia_log:backup_checkpoint/3) accepts a proplist with table filter.
Args = [ {module, Mod}
, {tables, NonRdbTabs}
, {scope, local} ],
case mnesia_log:backup_checkpoint(CpName, BupPath, Args) of
ok -> ok;
{error, _} = E -> E
end.
backup_rocksdbs(Dir, Dbs, Opts) ->
backup_rocksdbs(Dir, Dbs, Opts, []).
backup_rocksdbs(_Dir, [], _Opts, Acc) ->
{ok, lists:reverse(Acc)};
backup_rocksdbs(Dir, [Spec | Rest], Opts, Acc) ->
case backup_one_db(Dir, Spec, Opts) of
{ok, Info} ->
backup_rocksdbs(Dir, Rest, Opts, [Info | Acc]);
{error, _} = E ->
E
end.
backup_one_db(Dir, #{ db_ref := DbRef
, mountpoint := MP
, kind := Kind
, alias := Alias
, name := Name } = Spec, Opts) ->
Rel = backup_relpath(Spec),
Abs = filename:join(Dir, Rel),
case ensure_dir_tree(Abs) of
ok ->
case rocksdb:open_backup_engine(Abs) of
{ok, BE} ->
try
case rocksdb:create_new_backup(BE, DbRef) of
ok ->
case latest_backup_id(BE) of
{ok, BackupId} ->
case maybe_verify_backup(BE, BackupId, Opts) of
ok ->
{ok, #{ kind => Kind
, alias => Alias
, name => Name
, mountpoint => MP
, backup_relpath => Rel
, backup_id => BackupId }};
{error, _} = E ->
E
end;
{error, _} = E ->
E
end;
{error, _} = E ->
E
end
after
_ = rocksdb:close_backup_engine(BE)
end;
{error, _} = E ->
E
end;
{error, _} = E ->
E
end.
backup_relpath(#{kind := admin, alias := Alias}) ->
filename:join(["rocksdb", atom_to_list(Alias), "admin"]);
backup_relpath(#{kind := standalone, name := Name}) ->
filename:join(["rocksdb", "standalone", sanitize_backup_name(Name)]).
sanitize_backup_name(Name) when is_atom(Name) ->
atom_to_list(Name);
sanitize_backup_name(Name) ->
%% Index/retainer or other compound names: reuse mountpoint basename.
filename:basename(mnesia_rocksdb_lib:data_mountpoint(Name), ".extrdb").
ensure_dir_tree(Dir) ->
case filelib:is_dir(Dir) of
true ->
ok;
false ->
case filelib:ensure_dir(filename:join(Dir, "dummy")) of
ok ->
case file:make_dir(Dir) of
ok -> ok;
{error, eexist} -> ok;
{error, _} = E -> E
end;
{error, _} = E ->
E
end
end.
latest_backup_id(BE) ->
%% Erase rocksdb's incomplete -spec (list of #{id}) so we can handle the
%% real NIF shape ({ok, [#{backup_id => ...}]}).
Raw = binary_to_term(term_to_binary(rocksdb:get_backup_info(BE))),
case normalize_backup_infos(Raw) of
[] ->
{error, no_backup_info};
Infos ->
{ok, lists:max([backup_info_id(I) || I <- Infos])}
end.
-spec normalize_backup_infos(term()) -> [term()].
normalize_backup_infos({ok, Infos}) when is_list(Infos) ->
Infos;
normalize_backup_infos(Infos) when is_list(Infos) ->
Infos;
normalize_backup_infos(_) ->
[].
-spec backup_info_id(term()) -> non_neg_integer().
backup_info_id(Info) when is_map(Info) ->
case maps:find(id, Info) of
{ok, Id} when is_integer(Id), Id >= 0 ->
Id;
error ->
case maps:find(backup_id, Info) of
{ok, Id} when is_integer(Id), Id >= 0 ->
Id;
_ ->
error({missing_backup_id, Info})
end;
_ ->
error({invalid_backup_id, Info})
end;
backup_info_id(Other) ->
error({invalid_backup_info, Other}).
maybe_verify_backup(BE, BackupId, Opts) ->
case maps:get(verify, Opts, false) of
true ->
rocksdb:verify_backup(BE, BackupId);
false ->
ok
end.
write_manifest(Dir, NonRdbTabs, RdbTabs, DbInfos) ->
Manifest =
#{ version => 1
, created => erlang:system_time(second)
, node => node()
, mnesia_directory => mnesia:system_info(directory)
, mnesia_backup => "mnesia.BUP"
, mnesia_tables => NonRdbTabs
, rocksdb_tables => RdbTabs
, dbs => [maps:with([kind, alias, name, mountpoint,
backup_relpath, backup_id], I)
|| I <- DbInfos]
},
Path = filename:join(Dir, "MANIFEST"),
Bin = unicode:characters_to_binary(io_lib:format("~p.~n", [Manifest])),
case file:write_file(Path, Bin) of
ok -> ok;
{error, _} = E -> E
end.
%% ===================================================================
%% Hybrid restore
%% ===================================================================
do_restore(Dir, Opts) ->
case read_manifest(Dir) of
{ok, Manifest} ->
case maybe_stop_mnesia(Opts) of
ok ->
restore_body(Dir, Manifest, Opts);
{error, _} = E ->
E
end;
{error, _} = E ->
E
end.
restore_body(Dir, Manifest, Opts) ->
SkipMnesia = maps:get(skip_mnesia, Opts, false),
SkipRocks = maps:get(skip_rocksdb, Opts, false),
MnesiaDir = target_mnesia_dir(Opts, Manifest),
case wait_admin_down(100, 50) of
ok ->
restore_body1(Dir, Manifest, Opts, MnesiaDir,
SkipMnesia, SkipRocks);
{error, _} = E ->
E
end.
restore_body1(Dir, Manifest, Opts, MnesiaDir, SkipMnesia, SkipRocks) ->
case {SkipMnesia, SkipRocks} of
{true, true} ->
ok;
{true, false} ->
%% RocksDB only: write straight into final mountpoints.
case restore_rocksdbs(Dir, Manifest, MnesiaDir) of
ok -> maybe_start_mnesia(Opts);
{error, _} = E -> E
end;
{false, true} ->
%% Mnesia fallback only (no RocksDB data restore).
case install_and_apply_fallback(Dir, Manifest, Opts) of
ok -> maybe_start_mnesia(Opts);
{error, _} = E -> E
end;
{false, false} ->
staged_full_restore(Dir, Manifest, Opts, MnesiaDir)
end.
%% Full hybrid restore via staging sibling (survives fallback purge).
staged_full_restore(Dir, Manifest, Opts, MnesiaDir) ->
Staging = mnesia_rocksdb_lib:restore_staging_root(MnesiaDir),
try
case prepare_staging_root(Staging) of
ok ->
case restore_rocksdbs(Dir, Manifest, Staging) of
ok ->
set_restore_mountpoint_root(Staging),
case install_and_apply_fallback(Dir, Manifest, Opts) of
ok ->
finalize_staging(Staging, Manifest, MnesiaDir,
Opts);
{error, _} = E ->
clear_restore_mountpoint_root(),
E
end;
{error, _} = E ->
E
end;
{error, _} = E ->
E
end
after
clear_restore_mountpoint_root()
end.
finalize_staging(Staging, Manifest, MnesiaDir, Opts) ->
clear_restore_mountpoint_root(),
case relocate_staging_to_mnesia(Staging, Manifest, MnesiaDir) of
ok ->
_ = clear_dir(Staging),
maybe_start_mnesia(Opts);
{error, _} = E ->
E
end.
prepare_staging_root(Staging) ->
case clear_dir(Staging) of
ok ->
ensure_dir_tree(Staging);
{error, _} = E ->
E
end.
set_restore_mountpoint_root(Root) ->
Abs = filename:absname(Root),
application:set_env(mnesia_rocksdb, restore_mountpoint_root, Abs),
%% Keep PT in sync when present (admin may not be running yet).
case persistent_term:get(?PT_KEY, undefined) of
undefined ->
ok;
Meta when is_map(Meta) ->
persistent_term:put(
?PT_KEY, Meta#{ {restore_mountpoint_root} => Abs })
end,
ok.
clear_restore_mountpoint_root() ->
application:unset_env(mnesia_rocksdb, restore_mountpoint_root),
case persistent_term:get(?PT_KEY, undefined) of
undefined ->
ok;
Meta when is_map(Meta) ->
persistent_term:put(
?PT_KEY, maps:remove({restore_mountpoint_root}, Meta))
end,
ok.
install_and_apply_fallback(Dir, Manifest, Opts) ->
case install_mnesia_fallback(Dir, Manifest, Opts) of
ok ->
apply_mnesia_fallback();
{error, _} = E ->
E
end.
%% Start Mnesia so FALLBACK.BUP is applied (RocksDB opens from staging if set),
%% wait for tables, then stop so files can be relocated to the real Mnesia dir.
apply_mnesia_fallback() ->
StartRes =
case mnesia:start() of
ok -> ok;
{error, {already_started, mnesia}} -> ok;
{error, _} = StartErr -> StartErr
end,
case StartRes of
ok ->
case mnesia:system_info(is_running) of
yes ->
Tabs = mnesia:system_info(tables),
case mnesia:wait_for_tables(Tabs, 120000) of
ok ->
stopped = mnesia:stop(),
wait_admin_down(100, 50);
{timeout, _} = T ->
_ = mnesia:stop(),
_ = wait_admin_down(100, 50),
{error, T};
{error, _} = WaitErr ->
_ = mnesia:stop(),
_ = wait_admin_down(100, 50),
WaitErr
end;
_ ->
{error, fallback_apply_failed}
end;
{error, _} = Err ->
Err
end.
relocate_staging_to_mnesia(Staging, #{dbs := Dbs}, MnesiaDir) ->
relocate_loop(Dbs, Staging, MnesiaDir).
relocate_loop([], _Staging, _MnesiaDir) ->
ok;
relocate_loop([#{mountpoint := OrigMP} | Rest], Staging, MnesiaDir) ->
Base = filename:basename(OrigMP),
From = filename:join(Staging, Base),
To = filename:join(MnesiaDir, Base),
case relocate_one_mountpoint(From, To, MnesiaDir) of
ok ->
relocate_loop(Rest, Staging, MnesiaDir);
{error, _} = E ->
E
end.
relocate_one_mountpoint(From, To, MnesiaDir) ->
case filelib:is_dir(From) of
false ->
{error, {missing_staged_mountpoint, From}};
true ->
case is_under_dir(To, MnesiaDir) of
false ->
{error, {mountpoint_outside_mnesia_dir, To, MnesiaDir}};
true ->
case clear_dir(To) of
ok ->
case ensure_parent_dir(To) of
ok ->
case file:rename(From, To) of
ok -> ok;
{error, _} = E -> E
end;
{error, _} = E ->
E
end;
{error, _} = E ->
E
end
end
end.
ensure_parent_dir(Path) ->
filelib:ensure_dir(filename:join(Path, "dummy")).
read_manifest(Dir) ->
Path = filename:join(Dir, "MANIFEST"),
case file:consult(Path) of
{ok, [#{version := 1} = M]} ->
{ok, M};
{ok, [Other]} ->
{error, {invalid_manifest, Other}};
{ok, Other} ->
{error, {invalid_manifest, Other}};
{error, enoent} ->
{error, {missing_manifest, Path}};
{error, _} = E ->
E
end.
target_mnesia_dir(Opts, Manifest) ->
case maps:get(mnesia_dir, Opts, undefined) of
undefined ->
case mnesia_directory() of
undefined ->
maps:get(mnesia_directory, Manifest);
Dir ->
Dir
end;
Dir ->
filename:absname(Dir)
end.
mnesia_directory() ->
try mnesia:system_info(directory) of
Dir when is_list(Dir); is_binary(Dir) -> filename:absname(Dir);
_ -> undefined
catch
_:_ ->
case application:get_env(mnesia, dir) of
{ok, Dir} -> filename:absname(Dir);
undefined -> undefined
end
end.
maybe_stop_mnesia(Opts) ->
case maps:get(stop, Opts, true) of
false ->
case mnesia:system_info(is_running) of
yes ->
{error, {mnesia_running,
"stop Mnesia before restore, or pass #{stop => true}"}};
_ ->
ok
end;
true ->
case mnesia:system_info(is_running) of
yes ->
stopped = mnesia:stop(),
wait_admin_down(100, 50);
_ ->
%% Best-effort: admin may still hold open DBs if left over.
stop_admin_if_alive(),
wait_admin_down(100, 50)
end
end.
stop_admin_if_alive() ->
case whereis(?MODULE) of
undefined ->
ok;
Pid when is_pid(Pid) ->
try gen_server:stop(Pid, shutdown, 30000)
catch
_:_ -> ok
end
end.
wait_admin_down(0, _Sleep) ->
case whereis(?MODULE) of
undefined -> ok;
Pid -> {error, {admin_still_running, Pid}}
end;
wait_admin_down(N, Sleep) ->
case whereis(?MODULE) of
undefined ->
ok;
_ ->
timer:sleep(Sleep),
wait_admin_down(N - 1, Sleep)
end.
maybe_start_mnesia(Opts) ->
case maps:get(start, Opts, false) of
true ->
case mnesia:start() of
ok -> ok;
{error, {already_started, mnesia}} -> ok;
{error, _} = E -> E
end;
false ->
ok
end.
%% Restore RocksDB engines under Root (staging sibling or final Mnesia dir).
restore_rocksdbs(Dir, #{dbs := Dbs}, Root) ->
restore_rocksdbs_loop(Dir, Dbs, Root).
restore_rocksdbs_loop(_Dir, [], _Root) ->
ok;
restore_rocksdbs_loop(Dir, [Spec | Rest], Root) ->
case restore_one_db(Dir, Spec, Root) of
ok ->
restore_rocksdbs_loop(Dir, Rest, Root);
{error, _} = E ->
E
end.
restore_one_db(Dir, #{ backup_relpath := Rel
, backup_id := BackupId
, mountpoint := OrigMP }, Root) ->
AbsBackup = filename:join(Dir, Rel),
TargetMP = remap_mountpoint(OrigMP, Root),
case prepare_restore_mountpoint(TargetMP, Root) of
ok ->
case rocksdb:open_backup_engine(AbsBackup) of
{ok, BE} ->
try
case rocksdb:restore_db_from_backup(
BE, BackupId, TargetMP, TargetMP) of
ok -> ok;
{error, _} = E -> E
end
after
_ = rocksdb:close_backup_engine(BE)
end;
{error, _} = E ->
E
end;
{error, _} = E ->
E
end.
%% Keep only the basename under Root so restores work when the absolute path
%% differs from the backup host (and for staging siblings).
remap_mountpoint(OrigMP, Root) ->
filename:join(Root, filename:basename(OrigMP)).
prepare_restore_mountpoint(TargetMP, Root) ->
case is_under_dir(TargetMP, Root) of
true ->
case clear_dir(TargetMP) of
ok ->
ensure_dir_tree(TargetMP);
{error, _} = E ->
E
end;
false ->
{error, {mountpoint_outside_root, TargetMP, Root}}
end.
is_under_dir(Path, Root) ->
AbsPath = filename:absname(Path),
AbsRoot = filename:absname(Root),
lists:prefix(AbsRoot ++ "/", AbsPath ++ "/")
orelse AbsPath =:= AbsRoot.
clear_dir(Dir) ->
case filelib:is_dir(Dir) of
true ->
file:del_dir_r(Dir);
false ->
ok
end.
install_mnesia_fallback(Dir, Manifest, Opts) ->
Rel = maps:get(mnesia_backup, Manifest, "mnesia.BUP"),
Bup = filename:join(Dir, Rel),
case filelib:is_regular(Bup) of
false ->
{error, {missing_mnesia_backup, Bup}};
true ->
Mod = maps:get(mnesia_backup_module, Opts,
default_backup_module()),
Args = [ {module, Mod}
, {scope, local}
],
case mnesia:install_fallback(Bup, Args) of
ok -> ok;
{error, _} = E -> E
end
end.
default_backup_module() ->
try mnesia:system_info(backup_module)
catch
_:_ -> mnesia_backup
end.
src/mnesia_rocksdb_lib.erl
+42 -4
View File
@@ -11,6 +11,8 @@
, data_mountpoint/1 , data_mountpoint/1
, create_mountpoint/1 , create_mountpoint/1
, tabname/1 , tabname/1
, restore_mountpoint_root/0
, restore_staging_root/1
]). ]).
-export([ default_encoding/3 -export([ default_encoding/3
@@ -59,8 +61,16 @@ create_mountpoint(Tab) ->
file:make_dir(MPd), file:make_dir(MPd),
ok; ok;
true -> true ->
Dir = mnesia_lib:dir(), AllowedRoots =
case lists:prefix(Dir, MPd) of case restore_mountpoint_root() of
undefined -> [mnesia_lib:dir()];
Staging -> [mnesia_lib:dir(), Staging]
end,
case lists:any(
fun(Root) ->
lists:prefix(filename:absname(Root),
filename:absname(MPd))
end, AllowedRoots) of
true -> true ->
ok; ok;
false -> false ->
@@ -68,9 +78,37 @@ create_mountpoint(Tab) ->
end end
end. end.
%% @doc Absolute RocksDB directory for a logical table / admin DB.
%%
%% During hybrid restore, `application:get_env(mnesia_rocksdb,
%% restore_mountpoint_root)` may point at a sibling staging directory so
%% RocksDB data survives Mnesia's fallback purge of the real Mnesia dir.
-spec data_mountpoint(term()) -> file:filename().
data_mountpoint(Tab) -> data_mountpoint(Tab) ->
Dir = mnesia_monitor:get_env(dir), Base = tabname(Tab) ++ ".extrdb",
filename:join(Dir, tabname(Tab) ++ ".extrdb"). case restore_mountpoint_root() of
undefined ->
filename:join(mnesia_monitor:get_env(dir), Base);
Root ->
filename:join(Root, Base)
end.
%% @doc Staging root used while applying a hybrid restore, if any.
-spec restore_mountpoint_root() -> file:filename() | undefined.
restore_mountpoint_root() ->
case application:get_env(mnesia_rocksdb, restore_mountpoint_root) of
{ok, Root} when is_list(Root); is_binary(Root) ->
filename:absname(Root);
_ ->
undefined
end.
%% @doc Sibling of the Mnesia directory for staged RocksDB restore data.
-spec restore_staging_root(file:filename()) -> file:filename().
restore_staging_root(MnesiaDir) ->
Abs = filename:absname(MnesiaDir),
filename:join(filename:dirname(Abs),
filename:basename(Abs) ++ ".rocksdb_restore").
tabname({admin, Alias}) -> tabname({admin, Alias}) ->
"mnesia_rocksdb-" ++ atom_to_list(Alias) ++ "-_db"; "mnesia_rocksdb-" ++ atom_to_list(Alias) ++ "-_db";
test/mnesia_rocksdb_backup_SUITE.erl
+288
View File
@@ -0,0 +1,288 @@
%% -*- mode: erlang; erlang-indent-level: 4; indent-tabs-mode: nil -*-
-module(mnesia_rocksdb_backup_SUITE).
-export([ all/0
, suite/0
, groups/0
, init_per_suite/1
, end_per_suite/1
, init_per_group/2
, end_per_group/2
, init_per_testcase/2
, end_per_testcase/2
]).
-export([ hybrid_backup_cf_and_disc/1
, hybrid_backup_with_standalone/1
, backup_rejects_nonempty_dir/1
, backup_with_verify/1
, restore_roundtrip_cf_and_disc/1
, restore_roundtrip_with_standalone/1
]).
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").
suite() ->
[].
all() ->
[{group, all_tests}].
groups() ->
[{all_tests, [sequence],
[ hybrid_backup_cf_and_disc
, hybrid_backup_with_standalone
, backup_rejects_nonempty_dir
, backup_with_verify
, restore_roundtrip_cf_and_disc
, restore_roundtrip_with_standalone
]}].
init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
ok.
init_per_group(_, Config) ->
Config.
end_per_group(_, _Config) ->
ok.
init_per_testcase(_TC, Config) ->
mnesia:stop(),
ok = mnesia_rocksdb_tlib:start_mnesia(reset),
Config.
end_per_testcase(_TC, _Config) ->
ok.
%%--------------------------------------------------------------------
%% Tests
%%--------------------------------------------------------------------
%% disc_copies (not ram_copies): with ram_overrides_dump=false, Mnesia
%% checkpoints prefer the dumped image for ram_copies, so pure RAM writes
%% are a poor stand-in for "persistent non-RocksDB tables".
hybrid_backup_cf_and_disc(Config) ->
create_rdb_table(t, []),
{atomic, ok} =
mnesia:create_table(d, [{disc_copies, [node()]},
{attributes, [k, v]}]),
ok = mnesia:wait_for_tables([t, d], 5000),
[ok = mnesia:dirty_write({t, K, V}) || {K, V} <- [{a, 1}, {b, 2}, {c, 3}]],
[ok = mnesia:dirty_write({d, K, V}) || {K, V} <- [{x, 10}, {y, 20}]],
BackupDir = backup_dir(Config, "cf_disc"),
?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)),
assert_layout(BackupDir),
Manifest = read_manifest(BackupDir),
ct:log("MANIFEST = ~p", [Manifest]),
?assertEqual(1, maps:get(version, Manifest)),
?assert(lists:member(t, maps:get(rocksdb_tables, Manifest))),
?assertNot(lists:member(t, maps:get(mnesia_tables, Manifest))),
?assert(lists:member(schema, maps:get(mnesia_tables, Manifest))),
?assert(lists:member(d, maps:get(mnesia_tables, Manifest))),
Dbs = maps:get(dbs, Manifest),
?assert(lists:any(fun(#{kind := admin}) -> true; (_) -> false end, Dbs)),
assert_admin_backup_ok(BackupDir, Dbs),
assert_mnesia_bup_has_table(BackupDir, d),
assert_mnesia_bup_lacks_table_data(BackupDir, t),
ok.
hybrid_backup_with_standalone(Config) ->
create_rdb_table(t, []),
create_rdb_table(ts, [{user_properties, [{rocksdb_standalone, true}]}]),
{atomic, ok} =
mnesia:create_table(d, [{disc_copies, [node()]},
{attributes, [k, v]}]),
ok = mnesia:wait_for_tables([t, ts, d], 5000),
ok = mnesia:dirty_write({t, 1, one}),
ok = mnesia:dirty_write({ts, 2, two}),
ok = mnesia:dirty_write({d, 3, three}),
#{type := standalone} = mrdb:get_ref(ts),
BackupDir = backup_dir(Config, "standalone"),
?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)),
Manifest = read_manifest(BackupDir),
RdbTabs = maps:get(rocksdb_tables, Manifest),
?assert(lists:member(t, RdbTabs)),
?assert(lists:member(ts, RdbTabs)),
?assert(lists:member(d, maps:get(mnesia_tables, Manifest))),
Dbs = maps:get(dbs, Manifest),
?assert(lists:any(fun(#{kind := standalone, name := ts}) -> true;
(_) -> false end, Dbs)),
?assert(lists:any(fun(#{kind := admin}) -> true; (_) -> false end, Dbs)),
lists:foreach(
fun(#{backup_relpath := Rel, backup_id := Id}) ->
Abs = filename:join(BackupDir, Rel),
{ok, BE} = rocksdb:open_backup_engine(Abs),
try
?assertEqual(ok, rocksdb:verify_backup(BE, Id))
after
rocksdb:close_backup_engine(BE)
end
end, Dbs),
ok.
backup_rejects_nonempty_dir(Config) ->
create_rdb_table(t, []),
ok = mnesia:wait_for_tables([t], 5000),
ok = mnesia:dirty_write({t, 1, a}),
BackupDir = backup_dir(Config, "nonempty"),
ok = filelib:ensure_dir(filename:join(BackupDir, "noise")),
case file:make_dir(BackupDir) of
ok -> ok;
{error, eexist} -> ok
end,
ok = file:write_file(filename:join(BackupDir, "noise"), <<"x">>),
?assertMatch({error, {not_empty, _}}, mnesia_rocksdb:backup(BackupDir)),
ok.
backup_with_verify(Config) ->
create_rdb_table(t, []),
ok = mnesia:wait_for_tables([t], 5000),
ok = mnesia:dirty_write({t, 1, a}),
BackupDir = backup_dir(Config, "verify"),
?assertEqual(ok, mnesia_rocksdb:backup(BackupDir, #{verify => true})),
assert_layout(BackupDir),
ok.
restore_roundtrip_cf_and_disc(Config) ->
create_rdb_table(t, []),
{atomic, ok} =
mnesia:create_table(d, [{disc_copies, [node()]},
{attributes, [k, v]}]),
ok = mnesia:wait_for_tables([t, d], 5000),
[ok = mnesia:dirty_write({t, K, V}) || {K, V} <- [{a, 1}, {b, 2}]],
[ok = mnesia:dirty_write({d, K, V}) || {K, V} <- [{x, 10}, {y, 20}]],
BackupDir = backup_dir(Config, "restore_cf"),
?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)),
%% Mutate after backup — restore must discard these changes.
ok = mnesia:dirty_write({t, a, 999}),
ok = mnesia:dirty_write({t, c, 3}),
ok = mnesia:dirty_delete({d, x}),
ok = mnesia:dirty_write({d, z, 30}),
?assertEqual(ok, mnesia_rocksdb:restore(BackupDir, #{start => true})),
?assertEqual(ok, mnesia:wait_for_tables([t, d], 10000)),
?assertEqual([{t, a, 1}, {t, b, 2}],
lists:sort(mnesia:dirty_match_object({t, '_', '_'}))),
?assertEqual([{d, x, 10}, {d, y, 20}],
lists:sort(mnesia:dirty_match_object({d, '_', '_'}))),
ok.
restore_roundtrip_with_standalone(Config) ->
create_rdb_table(t, []),
create_rdb_table(ts, [{user_properties, [{rocksdb_standalone, true}]}]),
{atomic, ok} =
mnesia:create_table(d, [{disc_copies, [node()]},
{attributes, [k, v]}]),
ok = mnesia:wait_for_tables([t, ts, d], 5000),
ok = mnesia:dirty_write({t, 1, one}),
ok = mnesia:dirty_write({ts, 2, two}),
ok = mnesia:dirty_write({d, 3, three}),
#{type := standalone} = mrdb:get_ref(ts),
BackupDir = backup_dir(Config, "restore_sa"),
?assertEqual(ok, mnesia_rocksdb:backup(BackupDir)),
ok = mnesia:dirty_write({t, 1, changed}),
ok = mnesia:dirty_write({ts, 2, changed}),
ok = mnesia:dirty_write({d, 3, changed}),
?assertEqual(ok, mnesia_rocksdb:restore(BackupDir, #{start => true})),
?assertEqual(ok, mnesia:wait_for_tables([t, ts, d], 10000)),
?assertEqual([{t, 1, one}], mnesia:dirty_read(t, 1)),
?assertEqual([{ts, 2, two}], mnesia:dirty_read(ts, 2)),
?assertEqual([{d, 3, three}], mnesia:dirty_read(d, 3)),
#{type := standalone} = mrdb:get_ref(ts),
ok.
%%--------------------------------------------------------------------
%% Helpers
%%--------------------------------------------------------------------
create_rdb_table(Name, Extra) ->
{atomic, ok} =
mnesia:create_table(Name, [{rdb, [node()]},
{attributes, [k, v]} | Extra]),
ok.
backup_dir(Config, Name) ->
filename:join(?config(priv_dir, Config), "backup_" ++ Name).
assert_layout(BackupDir) ->
?assert(filelib:is_regular(filename:join(BackupDir, "MANIFEST"))),
?assert(filelib:is_regular(filename:join(BackupDir, "mnesia.BUP"))),
?assert(filelib:is_dir(filename:join(BackupDir, "rocksdb"))).
read_manifest(BackupDir) ->
Path = filename:join(BackupDir, "MANIFEST"),
{ok, [Manifest]} = file:consult(Path),
Manifest.
assert_admin_backup_ok(BackupDir, Dbs) ->
[Admin | _] = [D || #{kind := admin} = D <- Dbs],
#{backup_relpath := Rel, backup_id := Id} = Admin,
Abs = filename:join(BackupDir, Rel),
{ok, BE} = rocksdb:open_backup_engine(Abs),
try
Infos = case rocksdb:get_backup_info(BE) of
{ok, L} -> L;
L when is_list(L) -> L
end,
?assertNotEqual([], Infos),
?assertEqual(ok, rocksdb:verify_backup(BE, Id))
after
rocksdb:close_backup_engine(BE)
end.
assert_mnesia_bup_has_table(BackupDir, Tab) ->
Bup = filename:join(BackupDir, "mnesia.BUP"),
%% Fun is applied per backup item (not per batch).
{ok, Found} =
mnesia:traverse_backup(
Bup, mnesia_backup, read_only, read_only,
fun(Rec, _Acc) when is_tuple(Rec), element(1, Rec) =:= Tab ->
{[Rec], true};
(Rec, Acc) ->
{[Rec], Acc}
end, false),
?assertEqual(true, Found).
assert_mnesia_bup_lacks_table_data(BackupDir, Tab) ->
Bup = filename:join(BackupDir, "mnesia.BUP"),
{ok, Found} =
mnesia:traverse_backup(
Bup, mnesia_backup, read_only, read_only,
fun({schema, _, _} = Rec, Acc) ->
{[Rec], Acc};
(Rec, _Acc) when is_tuple(Rec), element(1, Rec) =:= Tab ->
{[Rec], true};
(Rec, Acc) ->
{[Rec], Acc}
end, false),
%% Data records for Tab should not appear; schema may still mention Tab.
?assertEqual(false, Found).