Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ulf Wiger
|
f0688f0b44 | ||
|
Ulf Wiger
|
c6969238c7 | ||
 uwiger
|
d6a4dca5f6 | ||
|
Ulf Wiger
|
b97b727605 | ||
|
uwiger
|
e0542f7f52 | ||
|
Ulf Wiger
|
13ccdb373a | ||
|
Ulf Wiger
|
29d5d6f170 | ||
|
uwiger
|
4e3c9e83c8 | ||
|
Ulf Wiger
|
f9352b6ff0 | ||
|
Ulf Wiger
|
04e6063ce0 | ||
|
Ulf Wiger
|
323fcea7a7 | ||
|
Ulf Wiger
|
d8b6ab788e |
@@ -16,21 +16,21 @@ is provided.
|
||||
### <a name="Table_of_Contents">Table of Contents</a> ###
|
||||
|
||||
|
||||
1. [Usage](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Usage)
|
||||
1. [Prerequisites](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Prerequisites)
|
||||
1. [Getting started](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Getting_started)
|
||||
1. [Special features](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Special_features)
|
||||
1. [Customization](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Customization)
|
||||
1. [Handling of errors in write operations](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Handling_of_errors_in_write_operations)
|
||||
1. [Caveats](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Caveats)
|
||||
1. [Usage](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Usage)
|
||||
1. [Prerequisites](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Prerequisites)
|
||||
1. [Getting started](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Getting_started)
|
||||
1. [Special features](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Special_features)
|
||||
1. [Customization](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Customization)
|
||||
1. [Handling of errors in write operations](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Handling_of_errors_in_write_operations)
|
||||
1. [Caveats](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Caveats)
|
||||
|
||||
1. [Mnesia backend plugins](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Mnesia_backend_plugins)
|
||||
1. [Background](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Background)
|
||||
1. [Design](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Design)
|
||||
1. [Mnesia backend plugins](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Mnesia_backend_plugins)
|
||||
1. [Background](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Background)
|
||||
1. [Design](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Design)
|
||||
|
||||
1. [Mnesia index plugins](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Mnesia_index_plugins)
|
||||
1. [Mnesia index plugins](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Mnesia_index_plugins)
|
||||
|
||||
1. [Rocksdb](https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/README.md#Rocksdb)
|
||||
1. [Rocksdb](https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/README.md#Rocksdb)
|
||||
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ RocksDB supports a number of customization options. These can be specified
|
||||
by providing a `{Key, Value}` list named `rocksdb_opts` under `user_properties`,
|
||||
for example:
|
||||
|
||||
```erlang
|
||||
```
|
||||
mnesia:create_table(foo, [{rocksdb_copies, [node()]},
|
||||
...
|
||||
{user_properties,
|
||||
@@ -93,7 +93,7 @@ for information on configuration parameters. Also see the section below on handl
|
||||
|
||||
The default configuration for tables in `mnesia_rocksdb` is:
|
||||
|
||||
```erlang
|
||||
```
|
||||
default_open_opts() ->
|
||||
[ {create_if_missing, true}
|
||||
, {cache_size,
|
||||
@@ -131,18 +131,6 @@ depend on having an up to date size count at all times, you need to maintain
|
||||
it yourself. If you only need the size occasionally, you may traverse the
|
||||
table to count the elements.
|
||||
|
||||
When `mrdb` transactions abort, they will return a stacktrace caught
|
||||
from within the transaction fun, giving much better debugging info.
|
||||
This is different from how mnesia does it.
|
||||
|
||||
If behavior closer to mnesia's abort returns are needed, say, for backwards
|
||||
compatibility, this can be controlled by setting the environment variable
|
||||
`-mnesia_rocksdb mnesia_compatible_aborts true`, or by adding a transaction
|
||||
option, e.g. `mrdb:activity({tx, #{mnesia_compatible => true}}, fun() ... end)`.
|
||||
For really performance-critical transactions which may abort often, it might
|
||||
make a difference to set this option to `true`, since there is a cost involved
|
||||
in producing stacktraces.
|
||||
|
||||
|
||||
### <a name="Mnesia_backend_plugins">Mnesia backend plugins</a> ###
|
||||
|
||||
@@ -207,7 +195,7 @@ our example. It returns a list of index terms.
|
||||
|
||||
Given the following index plugin implementation:
|
||||
|
||||
```erlang
|
||||
```
|
||||
-module(words).
|
||||
-export([words_f/3]).
|
||||
|
||||
@@ -224,7 +212,7 @@ words_(_) ->
|
||||
|
||||
We can register the plugin and use it in table definitions:
|
||||
|
||||
```erlang
|
||||
```
|
||||
Eshell V12.1.3 (abort with ^G)
|
||||
1> mnesia:start().
|
||||
ok
|
||||
@@ -240,7 +228,7 @@ as an exported function along the node's code path.
|
||||
|
||||
To see what happens when we insert an object, we can turn on call trace.
|
||||
|
||||
```erlang
|
||||
```
|
||||
4> dbg:tracer().
|
||||
{ok,<0.108.0>}
|
||||
5> dbg:tp(words, x).
|
||||
@@ -280,15 +268,17 @@ replaced by the new object.)
|
||||
|
||||
|
||||
<table width="100%" border="0" summary="list of modules">
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mnesia_rocksdb.md" class="module">mnesia_rocksdb</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mnesia_rocksdb_admin.md" class="module">mnesia_rocksdb_admin</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mnesia_rocksdb_app.md" class="module">mnesia_rocksdb_app</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mnesia_rocksdb_lib.md" class="module">mnesia_rocksdb_lib</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mnesia_rocksdb_params.md" class="module">mnesia_rocksdb_params</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mnesia_rocksdb_sup.md" class="module">mnesia_rocksdb_sup</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mnesia_rocksdb_tuning.md" class="module">mnesia_rocksdb_tuning</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mrdb.md" class="module">mrdb</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mrdb_index.md" class="module">mrdb_index</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mrdb_mutex.md" class="module">mrdb_mutex</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/g3553-refactor-plugin-migration-tmp-220318/doc/mrdb_select.md" class="module">mrdb_select</a></td></tr></table>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mnesia_rocksdb.md" class="module">mnesia_rocksdb</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mnesia_rocksdb_admin.md" class="module">mnesia_rocksdb_admin</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mnesia_rocksdb_app.md" class="module">mnesia_rocksdb_app</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mnesia_rocksdb_lib.md" class="module">mnesia_rocksdb_lib</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mnesia_rocksdb_params.md" class="module">mnesia_rocksdb_params</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mnesia_rocksdb_sup.md" class="module">mnesia_rocksdb_sup</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mnesia_rocksdb_tuning.md" class="module">mnesia_rocksdb_tuning</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mrdb.md" class="module">mrdb</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mrdb_index.md" class="module">mrdb_index</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mrdb_mutex.md" class="module">mrdb_mutex</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mrdb_mutex_serializer.md" class="module">mrdb_mutex_serializer</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mrdb_select.md" class="module">mrdb_select</a></td></tr>
|
||||
<tr><td><a href="https://github.com/aeternity/mnesia_rocksdb/blob/uw-mrdb-ms/doc/mrdb_stats.md" class="module">mrdb_stats</a></td></tr></table>
|
||||
|
||||
|
||||
+3
-1
@@ -278,5 +278,7 @@ replaced by the new object.)
|
||||
<tr><td><a href="mrdb.md" class="module">mrdb</a></td></tr>
|
||||
<tr><td><a href="mrdb_index.md" class="module">mrdb_index</a></td></tr>
|
||||
<tr><td><a href="mrdb_mutex.md" class="module">mrdb_mutex</a></td></tr>
|
||||
<tr><td><a href="mrdb_select.md" class="module">mrdb_select</a></td></tr></table>
|
||||
<tr><td><a href="mrdb_mutex_serializer.md" class="module">mrdb_mutex_serializer</a></td></tr>
|
||||
<tr><td><a href="mrdb_select.md" class="module">mrdb_select</a></td></tr>
|
||||
<tr><td><a href="mrdb_stats.md" class="module">mrdb_stats</a></td></tr></table>
|
||||
|
||||
|
||||
@@ -267,7 +267,7 @@ table_type() = set | ordered_set | bag
|
||||
|
||||
### handle_call/3 ###
|
||||
|
||||
`handle_call(X1, From, St) -> any()`
|
||||
`handle_call(M, From, St) -> any()`
|
||||
|
||||
<a name="handle_cast-2"></a>
|
||||
|
||||
@@ -359,7 +359,7 @@ where Opts are parameters for the supervised child:
|
||||
|
||||
### load_table/4 ###
|
||||
|
||||
`load_table(Alias, Tab, LoadReason, Opts) -> any()`
|
||||
`load_table(Alias, Tab, LoadReason, Props) -> any()`
|
||||
|
||||
<a name="lookup-3"></a>
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ cf() = <a href="http://www.erlang.org/doc/man/mrdb.html#type-db_ref">mrdb:db_ref
|
||||
|
||||
|
||||
<pre><code>
|
||||
db_ref() = <a href="/home/uwiger/ae/mnesia_rocksdb/_build/default/lib/rocksdb/doc/rocksdb.md#type-db_handle">rocksdb:db_handle()</a>
|
||||
db_ref() = <a href="/Users/uwiger/gj/c/mnesia_rocksdb/_build/default/lib/rocksdb/doc/rocksdb.md#type-db_handle">rocksdb:db_handle()</a>
|
||||
</code></pre>
|
||||
|
||||
|
||||
@@ -108,7 +108,17 @@ reply() = any()
|
||||
|
||||
|
||||
<pre><code>
|
||||
req() = {create_table, <a href="#type-table">table()</a>, <a href="#type-properties">properties()</a>} | {delete_table, <a href="#type-table">table()</a>} | {load_table, <a href="#type-table">table()</a>} | {related_resources, <a href="#type-table">table()</a>} | {get_ref, <a href="#type-table">table()</a>} | {add_aliases, [<a href="#type-alias">alias()</a>]} | {write_table_property, <a href="#type-tabname">tabname()</a>, tuple()} | {remove_aliases, [<a href="#type-alias">alias()</a>]} | {migrate, [{<a href="#type-tabname">tabname()</a>, map()}]} | {prep_close, <a href="#type-table">table()</a>} | {close_table, <a href="#type-table">table()</a>}
|
||||
req() = {create_table, <a href="#type-table">table()</a>, <a href="#type-properties">properties()</a>} | {delete_table, <a href="#type-table">table()</a>} | {load_table, <a href="#type-table">table()</a>, <a href="#type-properties">properties()</a>} | {related_resources, <a href="#type-table">table()</a>} | {get_ref, <a href="#type-table">table()</a>} | {add_aliases, [<a href="#type-alias">alias()</a>]} | {write_table_property, <a href="#type-tabname">tabname()</a>, tuple()} | {remove_aliases, [<a href="#type-alias">alias()</a>]} | {migrate, [<a href="#type-tabname">tabname()</a> | {<a href="#type-tabname">tabname()</a>, map()}], <a href="#type-rpt">rpt()</a>} | {prep_close, <a href="#type-table">table()</a>} | {close_table, <a href="#type-table">table()</a>} | {clear_table, <a href="#type-table">table()</a> | <a href="#type-cf">cf()</a>}
|
||||
</code></pre>
|
||||
|
||||
|
||||
|
||||
|
||||
### <a name="type-rpt">rpt()</a> ###
|
||||
|
||||
|
||||
<pre><code>
|
||||
rpt() = undefined | map()
|
||||
</code></pre>
|
||||
|
||||
|
||||
@@ -146,7 +156,7 @@ tabname() = atom()
|
||||
## Function Index ##
|
||||
|
||||
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#add_aliases-1">add_aliases/1</a></td><td></td></tr><tr><td valign="top"><a href="#close_table-2">close_table/2</a></td><td></td></tr><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr><tr><td valign="top"><a href="#create_table-3">create_table/3</a></td><td></td></tr><tr><td valign="top"><a href="#delete_table-2">delete_table/2</a></td><td></td></tr><tr><td valign="top"><a href="#ensure_started-0">ensure_started/0</a></td><td></td></tr><tr><td valign="top"><a href="#get_ref-1">get_ref/1</a></td><td></td></tr><tr><td valign="top"><a href="#get_ref-2">get_ref/2</a></td><td></td></tr><tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr><tr><td valign="top"><a href="#handle_cast-2">handle_cast/2</a></td><td></td></tr><tr><td valign="top"><a href="#handle_info-2">handle_info/2</a></td><td></td></tr><tr><td valign="top"><a href="#init-1">init/1</a></td><td></td></tr><tr><td valign="top"><a href="#load_table-2">load_table/2</a></td><td></td></tr><tr><td valign="top"><a href="#meta-0">meta/0</a></td><td></td></tr><tr><td valign="top"><a href="#migrate_standalone-2">migrate_standalone/2</a></td><td></td></tr><tr><td valign="top"><a href="#prep_close-2">prep_close/2</a></td><td></td></tr><tr><td valign="top"><a href="#read_info-1">read_info/1</a></td><td></td></tr><tr><td valign="top"><a href="#read_info-2">read_info/2</a></td><td></td></tr><tr><td valign="top"><a href="#read_info-4">read_info/4</a></td><td></td></tr><tr><td valign="top"><a href="#related_resources-2">related_resources/2</a></td><td></td></tr><tr><td valign="top"><a href="#remove_aliases-1">remove_aliases/1</a></td><td></td></tr><tr><td valign="top"><a href="#request_ref-2">request_ref/2</a></td><td></td></tr><tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td></td></tr><tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr><tr><td valign="top"><a href="#write_info-4">write_info/4</a></td><td></td></tr><tr><td valign="top"><a href="#write_table_property-3">write_table_property/3</a></td><td></td></tr></table>
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#add_aliases-1">add_aliases/1</a></td><td></td></tr><tr><td valign="top"><a href="#clear_table-1">clear_table/1</a></td><td></td></tr><tr><td valign="top"><a href="#close_table-2">close_table/2</a></td><td></td></tr><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr><tr><td valign="top"><a href="#create_table-3">create_table/3</a></td><td></td></tr><tr><td valign="top"><a href="#delete_info-3">delete_info/3</a></td><td></td></tr><tr><td valign="top"><a href="#delete_table-2">delete_table/2</a></td><td></td></tr><tr><td valign="top"><a href="#ensure_started-0">ensure_started/0</a></td><td></td></tr><tr><td valign="top"><a href="#get_cached_env-2">get_cached_env/2</a></td><td></td></tr><tr><td valign="top"><a href="#get_ref-1">get_ref/1</a></td><td></td></tr><tr><td valign="top"><a href="#get_ref-2">get_ref/2</a></td><td></td></tr><tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr><tr><td valign="top"><a href="#handle_cast-2">handle_cast/2</a></td><td></td></tr><tr><td valign="top"><a href="#handle_info-2">handle_info/2</a></td><td></td></tr><tr><td valign="top"><a href="#init-1">init/1</a></td><td></td></tr><tr><td valign="top"><a href="#load_table-3">load_table/3</a></td><td></td></tr><tr><td valign="top"><a href="#meta-0">meta/0</a></td><td></td></tr><tr><td valign="top"><a href="#migrate_standalone-2">migrate_standalone/2</a></td><td></td></tr><tr><td valign="top"><a href="#migrate_standalone-3">migrate_standalone/3</a></td><td></td></tr><tr><td valign="top"><a href="#prep_close-2">prep_close/2</a></td><td></td></tr><tr><td valign="top"><a href="#read_info-1">read_info/1</a></td><td></td></tr><tr><td valign="top"><a href="#read_info-2">read_info/2</a></td><td></td></tr><tr><td valign="top"><a href="#read_info-4">read_info/4</a></td><td></td></tr><tr><td valign="top"><a href="#related_resources-2">related_resources/2</a></td><td></td></tr><tr><td valign="top"><a href="#remove_aliases-1">remove_aliases/1</a></td><td></td></tr><tr><td valign="top"><a href="#request_ref-2">request_ref/2</a></td><td></td></tr><tr><td valign="top"><a href="#set_and_cache_env-2">set_and_cache_env/2</a></td><td></td></tr><tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td></td></tr><tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr><tr><td valign="top"><a href="#write_info-4">write_info/4</a></td><td></td></tr><tr><td valign="top"><a href="#write_table_property-3">write_table_property/3</a></td><td></td></tr></table>
|
||||
|
||||
|
||||
<a name="functions"></a>
|
||||
@@ -159,6 +169,12 @@ tabname() = atom()
|
||||
|
||||
`add_aliases(Aliases) -> any()`
|
||||
|
||||
<a name="clear_table-1"></a>
|
||||
|
||||
### clear_table/1 ###
|
||||
|
||||
`clear_table(Name) -> any()`
|
||||
|
||||
<a name="close_table-2"></a>
|
||||
|
||||
### close_table/2 ###
|
||||
@@ -177,6 +193,12 @@ tabname() = atom()
|
||||
|
||||
`create_table(Alias, Name, Props) -> any()`
|
||||
|
||||
<a name="delete_info-3"></a>
|
||||
|
||||
### delete_info/3 ###
|
||||
|
||||
`delete_info(Alias, Tab, K) -> any()`
|
||||
|
||||
<a name="delete_table-2"></a>
|
||||
|
||||
### delete_table/2 ###
|
||||
@@ -195,6 +217,12 @@ ensure_started() -> ok
|
||||
</code></pre>
|
||||
<br />
|
||||
|
||||
<a name="get_cached_env-2"></a>
|
||||
|
||||
### get_cached_env/2 ###
|
||||
|
||||
`get_cached_env(Key, Default) -> any()`
|
||||
|
||||
<a name="get_ref-1"></a>
|
||||
|
||||
### get_ref/1 ###
|
||||
@@ -240,11 +268,11 @@ handle_info(Msg::any(), St::<a href="#type-st">st()</a>) -> <a href="#type-ge
|
||||
|
||||
`init(X1) -> any()`
|
||||
|
||||
<a name="load_table-2"></a>
|
||||
<a name="load_table-3"></a>
|
||||
|
||||
### load_table/2 ###
|
||||
### load_table/3 ###
|
||||
|
||||
`load_table(Alias, Name) -> any()`
|
||||
`load_table(Alias, Name, Props) -> any()`
|
||||
|
||||
<a name="meta-0"></a>
|
||||
|
||||
@@ -256,7 +284,21 @@ handle_info(Msg::any(), St::<a href="#type-st">st()</a>) -> <a href="#type-ge
|
||||
|
||||
### migrate_standalone/2 ###
|
||||
|
||||
`migrate_standalone(Alias, Tabs) -> any()`
|
||||
<pre><code>
|
||||
migrate_standalone(Alias::<a href="#type-alias">alias()</a>, Tabs) -> Res
|
||||
</code></pre>
|
||||
|
||||
<ul class="definitions"><li><code>Tabs = [<a href="#type-tabname">tabname()</a> | {<a href="#type-tabname">tabname()</a>, map()}]</code></li><li><code>Res = [{<a href="#type-tabname">tabname()</a>, {ok, any()} | {error, any()}}]</code></li></ul>
|
||||
|
||||
<a name="migrate_standalone-3"></a>
|
||||
|
||||
### migrate_standalone/3 ###
|
||||
|
||||
<pre><code>
|
||||
migrate_standalone(Alias::<a href="#type-alias">alias()</a>, Tabs, Rpt) -> Res
|
||||
</code></pre>
|
||||
|
||||
<ul class="definitions"><li><code>Tabs = [<a href="#type-tabname">tabname()</a> | {<a href="#type-tabname">tabname()</a>, map()}]</code></li><li><code>Rpt = undefined | pid() | atom()</code></li><li><code>Res = [{<a href="#type-tabname">tabname()</a>, {ok, any()} | {error, any()}}]</code></li></ul>
|
||||
|
||||
<a name="prep_close-2"></a>
|
||||
|
||||
@@ -300,6 +342,12 @@ handle_info(Msg::any(), St::<a href="#type-st">st()</a>) -> <a href="#type-ge
|
||||
|
||||
`request_ref(Alias, Name) -> any()`
|
||||
|
||||
<a name="set_and_cache_env-2"></a>
|
||||
|
||||
### set_and_cache_env/2 ###
|
||||
|
||||
`set_and_cache_env(Key, Value) -> any()`
|
||||
|
||||
<a name="start_link-0"></a>
|
||||
|
||||
### start_link/0 ###
|
||||
|
||||
+235
-17
File diff suppressed because one or more lines are too long
+55
-2
@@ -36,7 +36,7 @@ iterator_action() = <a href="http://www.erlang.org/doc/man/mrdb.html#type-iterat
|
||||
|
||||
|
||||
<pre><code>
|
||||
ix_iterator() = #mrdb_ix_iter{i = <a href="http://www.erlang.org/doc/man/mrdb.html#type-iterator">mrdb:iterator()</a>, type = set | bag, sub = <a href="http://www.erlang.org/doc/man/mrdb.html#type-ref">mrdb:ref()</a> | pid()}
|
||||
ix_iterator() = #mrdb_ix_iter{i = <a href="http://www.erlang.org/doc/man/mrdb.html#type-mrdb_iterator">mrdb:mrdb_iterator()</a>, type = set | bag, sub = pid() | <a href="http://www.erlang.org/doc/man/mrdb.html#type-db_ref">mrdb:db_ref()</a>}
|
||||
</code></pre>
|
||||
|
||||
|
||||
@@ -54,13 +54,32 @@ object() = tuple()
|
||||
## Function Index ##
|
||||
|
||||
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#iterator-2">iterator/2</a></td><td></td></tr><tr><td valign="top"><a href="#iterator_close-1">iterator_close/1</a></td><td></td></tr><tr><td valign="top"><a href="#iterator_move-2">iterator_move/2</a></td><td></td></tr><tr><td valign="top"><a href="#with_iterator-3">with_iterator/3</a></td><td></td></tr></table>
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#fold-4">fold/4</a></td><td></td></tr><tr><td valign="top"><a href="#index_ref-2">index_ref/2</a></td><td></td></tr><tr><td valign="top"><a href="#iterator-2">iterator/2</a></td><td></td></tr><tr><td valign="top"><a href="#iterator_close-1">iterator_close/1</a></td><td></td></tr><tr><td valign="top"><a href="#iterator_move-2">iterator_move/2</a></td><td></td></tr><tr><td valign="top"><a href="#rev_fold-4">rev_fold/4</a></td><td></td></tr><tr><td valign="top"><a href="#select-3">select/3</a></td><td></td></tr><tr><td valign="top"><a href="#select-4">select/4</a></td><td></td></tr><tr><td valign="top"><a href="#select_reverse-3">select_reverse/3</a></td><td></td></tr><tr><td valign="top"><a href="#select_reverse-4">select_reverse/4</a></td><td></td></tr><tr><td valign="top"><a href="#with_iterator-3">with_iterator/3</a></td><td></td></tr></table>
|
||||
|
||||
|
||||
<a name="functions"></a>
|
||||
|
||||
## Function Details ##
|
||||
|
||||
<a name="fold-4"></a>
|
||||
|
||||
### fold/4 ###
|
||||
|
||||
<pre><code>
|
||||
fold(Tab::<a href="http://www.erlang.org/doc/man/mrdb.html#type-ref_or_tab">mrdb:ref_or_tab()</a>, IxPos::<a href="http://www.erlang.org/doc/man/mrdb.html#type-index_position">mrdb:index_position()</a>, FoldFun::fun((<a href="#type-index_value">index_value()</a>, <a href="#type-object">object()</a> | [], Acc) -> Acc), Acc) -> Acc
|
||||
</code></pre>
|
||||
|
||||
<ul class="definitions"><li><code>Acc = any()</code></li></ul>
|
||||
|
||||
<a name="index_ref-2"></a>
|
||||
|
||||
### index_ref/2 ###
|
||||
|
||||
<pre><code>
|
||||
index_ref(Tab::<a href="http://www.erlang.org/doc/man/mrdb.html#type-ref_or_tab">mrdb:ref_or_tab()</a>, Ix::<a href="http://www.erlang.org/doc/man/mrdb.html#type-index_position">mrdb:index_position()</a>) -> <a href="http://www.erlang.org/doc/man/mrdb.html#type-db_ref">mrdb:db_ref()</a>
|
||||
</code></pre>
|
||||
<br />
|
||||
|
||||
<a name="iterator-2"></a>
|
||||
|
||||
### iterator/2 ###
|
||||
@@ -88,6 +107,40 @@ iterator_move(Mrdb_ix_iter::<a href="#type-ix_iterator">ix_iterator()</a>, Dir::
|
||||
</code></pre>
|
||||
<br />
|
||||
|
||||
<a name="rev_fold-4"></a>
|
||||
|
||||
### rev_fold/4 ###
|
||||
|
||||
<pre><code>
|
||||
rev_fold(Tab::<a href="http://www.erlang.org/doc/man/mrdb.html#type-ref_or_tab">mrdb:ref_or_tab()</a>, IxPos::<a href="http://www.erlang.org/doc/man/mrdb.html#type-index_position">mrdb:index_position()</a>, FoldFun::fun((<a href="#type-index_value">index_value()</a>, <a href="#type-object">object()</a> | [], Acc) -> Acc), Acc) -> Acc
|
||||
</code></pre>
|
||||
|
||||
<ul class="definitions"><li><code>Acc = any()</code></li></ul>
|
||||
|
||||
<a name="select-3"></a>
|
||||
|
||||
### select/3 ###
|
||||
|
||||
`select(Tab, Ix, MS) -> any()`
|
||||
|
||||
<a name="select-4"></a>
|
||||
|
||||
### select/4 ###
|
||||
|
||||
`select(Tab, Ix, MS, Limit) -> any()`
|
||||
|
||||
<a name="select_reverse-3"></a>
|
||||
|
||||
### select_reverse/3 ###
|
||||
|
||||
`select_reverse(Tab, Ix, MS) -> any()`
|
||||
|
||||
<a name="select_reverse-4"></a>
|
||||
|
||||
### select_reverse/4 ###
|
||||
|
||||
`select_reverse(Tab, Ix, MS, Limit) -> any()`
|
||||
|
||||
<a name="with_iterator-3"></a>
|
||||
|
||||
### with_iterator/3 ###
|
||||
|
||||
+1
-7
@@ -9,7 +9,7 @@
|
||||
## Function Index ##
|
||||
|
||||
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#do-2">do/2</a></td><td></td></tr><tr><td valign="top"><a href="#ensure_tab-0">ensure_tab/0</a></td><td></td></tr></table>
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#do-2">do/2</a></td><td></td></tr></table>
|
||||
|
||||
|
||||
<a name="functions"></a>
|
||||
@@ -22,9 +22,3 @@
|
||||
|
||||
`do(Rsrc, F) -> any()`
|
||||
|
||||
<a name="ensure_tab-0"></a>
|
||||
|
||||
### ensure_tab/0 ###
|
||||
|
||||
`ensure_tab() -> any()`
|
||||
|
||||
|
||||
+31
-1
@@ -9,25 +9,43 @@
|
||||
## Function Index ##
|
||||
|
||||
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#fold-5">fold/5</a></td><td></td></tr><tr><td valign="top"><a href="#rdb_fold-5">rdb_fold/5</a></td><td></td></tr><tr><td valign="top"><a href="#select-1">select/1</a></td><td></td></tr><tr><td valign="top"><a href="#select-3">select/3</a></td><td></td></tr><tr><td valign="top"><a href="#select-4">select/4</a></td><td></td></tr></table>
|
||||
<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#continuation_info-2">continuation_info/2</a></td><td></td></tr><tr><td valign="top"><a href="#fold-5">fold/5</a></td><td></td></tr><tr><td valign="top"><a href="#fold_reverse-5">fold_reverse/5</a></td><td></td></tr><tr><td valign="top"><a href="#rdb_fold-5">rdb_fold/5</a></td><td></td></tr><tr><td valign="top"><a href="#rdb_fold_reverse-5">rdb_fold_reverse/5</a></td><td></td></tr><tr><td valign="top"><a href="#select-1">select/1</a></td><td></td></tr><tr><td valign="top"><a href="#select-3">select/3</a></td><td></td></tr><tr><td valign="top"><a href="#select-4">select/4</a></td><td></td></tr><tr><td valign="top"><a href="#select_reverse-3">select_reverse/3</a></td><td></td></tr><tr><td valign="top"><a href="#select_reverse-4">select_reverse/4</a></td><td></td></tr></table>
|
||||
|
||||
|
||||
<a name="functions"></a>
|
||||
|
||||
## Function Details ##
|
||||
|
||||
<a name="continuation_info-2"></a>
|
||||
|
||||
### continuation_info/2 ###
|
||||
|
||||
`continuation_info(Item, C) -> any()`
|
||||
|
||||
<a name="fold-5"></a>
|
||||
|
||||
### fold/5 ###
|
||||
|
||||
`fold(Ref, Fun, Acc, MS, Limit) -> any()`
|
||||
|
||||
<a name="fold_reverse-5"></a>
|
||||
|
||||
### fold_reverse/5 ###
|
||||
|
||||
`fold_reverse(Ref, Fun, Acc, MS, Limit) -> any()`
|
||||
|
||||
<a name="rdb_fold-5"></a>
|
||||
|
||||
### rdb_fold/5 ###
|
||||
|
||||
`rdb_fold(Ref, Fun, Acc, Prefix, Limit) -> any()`
|
||||
|
||||
<a name="rdb_fold_reverse-5"></a>
|
||||
|
||||
### rdb_fold_reverse/5 ###
|
||||
|
||||
`rdb_fold_reverse(Ref, Fun, Acc, Prefix, Limit) -> any()`
|
||||
|
||||
<a name="select-1"></a>
|
||||
|
||||
### select/1 ###
|
||||
@@ -46,3 +64,15 @@
|
||||
|
||||
`select(Ref, MS, AccKeys, Limit) -> any()`
|
||||
|
||||
<a name="select_reverse-3"></a>
|
||||
|
||||
### select_reverse/3 ###
|
||||
|
||||
`select_reverse(Ref, MS, Limit) -> any()`
|
||||
|
||||
<a name="select_reverse-4"></a>
|
||||
|
||||
### select_reverse/4 ###
|
||||
|
||||
`select_reverse(Ref, MS, AccKeys, Limit) -> any()`
|
||||
|
||||
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
# Mnesia Rocksdb - Rocksdb backend plugin for Mnesia
|
||||
|
||||
Copyright © 2013-21 Klarna AB
|
||||
|
||||
__Authors:__Ulf Wiger ([`ulf@wiger.net`](mailto:ulf@wiger.net)).
|
||||
|
||||
The Mnesia DBMS, part of Erlang/OTP, supports 'backend plugins', making it possible to utilize more capable key-value stores than the `dets` module (limited to 2 GB per table). Unfortunately, this support is undocumented. Below, some informal documentation for the plugin system is provided.
|
||||
|
||||
### Table of Contents
|
||||
|
||||
1. [Usage](#Usage)
|
||||
1. [Prerequisites](#Prerequisites)
|
||||
1. [Getting started](#Getting_started)
|
||||
1. [Special features](#Special_features)
|
||||
1. [Customization](#Customization)
|
||||
1. [Handling of errors in write operations](#Handling_of_errors_in_write_operations)
|
||||
1. [Caveats](#Caveats)
|
||||
|
||||
1. [Mnesia backend plugins](#Mnesia_backend_plugins)
|
||||
1. [Background](#Background)
|
||||
1. [Design](#Design)
|
||||
|
||||
1. [Mnesia index plugins](#Mnesia_index_plugins)
|
||||
1. [Rocksdb](#Rocksdb)
|
||||
|
||||
### Usage
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
* rocksdb (included as dependency)
|
||||
* sext (included as dependency)
|
||||
* Erlang/OTP 21.0 or newer (https://github.com/erlang/otp)
|
||||
|
||||
#### Getting started
|
||||
|
||||
Call `mnesia_rocksdb:register()` immediately after starting mnesia.
|
||||
|
||||
Put `{rocksdb_copies, [node()]}` into the table definitions of tables you want to be in RocksDB.
|
||||
|
||||
#### Special features
|
||||
|
||||
RocksDB tables support efficient selects on *prefix keys*.
|
||||
|
||||
The backend uses the `sext` module (see [`https://github.com/uwiger/sext`](https://github.com/uwiger/sext)) for mapping between Erlang terms and the binary data stored in the tables. This provides two useful properties:
|
||||
|
||||
* The records are stored in the Erlang term order of their keys.
|
||||
* A prefix of a composite key is ordered just before any key for which it is a prefix. For example, `{x, '_'}` is a prefix for keys `{x, a}`, `{x, b}` and so on.
|
||||
|
||||
This means that a prefix key identifies the start of the sequence of entries whose keys match the prefix. The backend uses this to optimize selects on prefix keys.
|
||||
|
||||
\### Customization
|
||||
|
||||
RocksDB supports a number of customization options. These can be specified by providing a `{Key, Value}` list named `rocksdb_opts` under `user_properties`, for example:
|
||||
|
||||
```text
|
||||
mnesia:create_table(foo, [{rocksdb_copies, [node()]},
|
||||
...
|
||||
{user_properties,
|
||||
[{rocksdb_opts, [{max_open_files, 1024}]}]
|
||||
}])
|
||||
```
|
||||
|
||||
Consult the [RocksDB documentation](https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning) for information on configuration parameters. Also see the section below on handling write errors.
|
||||
|
||||
The default configuration for tables in `mnesia_rocksdb` is:
|
||||
|
||||
```text
|
||||
default_open_opts() ->
|
||||
[ {create_if_missing, true}
|
||||
, {cache_size,
|
||||
list_to_integer(get_env_default("ROCKSDB_CACHE_SIZE", "32212254"))}
|
||||
, {block_size, 1024}
|
||||
, {max_open_files, 100}
|
||||
, {write_buffer_size,
|
||||
list_to_integer(get_env_default(
|
||||
"ROCKSDB_WRITE_BUFFER_SIZE", "4194304"))}
|
||||
, {compression,
|
||||
list_to_atom(get_env_default("ROCKSDB_COMPRESSION", "true"))}
|
||||
, {use_bloomfilter, true}
|
||||
].
|
||||
```
|
||||
|
||||
It is also possible, for larger databases, to produce a tuning parameter file. This is experimental, and mostly copied from `mnesia_leveldb`. Consult the source code in `mnesia_rocksdb_tuning.erl` and `mnesia_rocksdb_params.erl`. Contributions are welcome.
|
||||
|
||||
#### Caveats
|
||||
|
||||
Avoid placing `bag` tables in RocksDB. Although they work, each write requires additional reads, causing substantial runtime overheads. There are better ways to represent and process bag data (see above about *prefix keys*).
|
||||
|
||||
The `mnesia:table_info(T, size)` call always returns zero for RocksDB tables. RocksDB itself does not track the number of elements in a table, and although it is possible to make the `mnesia_rocksdb` backend maintain a size counter, it incurs a high runtime overhead for writes and deletes since it forces them to first do a read to check the existence of the key. If you depend on having an up to date size count at all times, you need to maintain it yourself. If you only need the size occasionally, you may traverse the table to count the elements.
|
||||
|
||||
### Mnesia backend plugins
|
||||
|
||||
#### Background
|
||||
|
||||
Mnesia was initially designed to be a RAM-only DBMS, and Erlang's `ets` tables were developed for this purpose. In order to support persistence, e.g. for configuration data, a disk-based version of `ets` (called `dets`) was created. The `dets` API mimicks the `ets` API, and `dets` is quite convenient and fast for (nowadays) small datasets. However, using a 32-bit bucket system, it is limited to 2GB of data. It also doesn't support ordered sets. When used in Mnesia, dets-based tables are called `disc_only_copies`.
|
||||
|
||||
To circumvent these limitations, another table type, called `disc_copies` was added. This is a combination of `ets` and `disk_log`, where Mnesia periodically snapshots the `ets` data to a log file on disk, and meanwhile maintains a log of updates, which can be applied at startup. These tables are quite performant (especially on read access), but all data is kept in RAM, which can become a serious limitation.
|
||||
|
||||
A backend plugin system was proposed by Ulf Wiger in 2016, and further developed with Klarna's support, to finally become included in OTP 19. Klarna uses a LevelDb backend, but Aeternity, in 2017, instead chose to implement a Rocksdb backend plugin.
|
||||
|
||||
### Design
|
||||
|
||||
As backend plugins were added on a long-since legacy-stable Mnesia, they had to conform to the existing code structure. For this reason, the plugin callbacks hook into the already present low-level access API in the `mnesia_lib` module. As a consequence, backend plugins have the same access semantics and granularity as `ets` and `dets`. This isn't much of a disadvantage for key-value stores like LevelDb and RocksDB, but a more serious issue is that the update part of this API is called on *after* the point of no return. That is, Mnesia does not expect these updates to fail, and has no recourse if they do. As an aside, this could also happen if a `disc_only_copies` table exceeds the 2 GB limit (mnesia will not check it, and `dets` will not complain, but simply drop the update.)
|
||||
|
||||
### Mnesia index plugins
|
||||
|
||||
When adding support for backend plugins, index plugins were also added. Unfortunately, they remain undocumented.
|
||||
|
||||
An index plugin can be added in one of two ways:
|
||||
|
||||
1. When creating a schema, provide `{index_plugins, [{Name, Module, Function}]}` options.
|
||||
1. Call the function `mnesia_schema:add_index_plugin(Name, Module, Function)`
|
||||
|
||||
`Name` must be an atom wrapped as a 1-tuple, e.g. `{words}`.
|
||||
|
||||
The plugin callback is called as `Module:Function(Table, Pos, Obj)`, where `Pos=={words}` in our example. It returns a list of index terms.
|
||||
|
||||
__Example__
|
||||
|
||||
Given the following index plugin implementation:
|
||||
|
||||
```text
|
||||
-module(words).
|
||||
-export([words_f/3]).
|
||||
|
||||
words_f(_,_,Obj) when is_tuple(Obj) ->
|
||||
words_(tuple_to_list(Obj)).
|
||||
|
||||
words_(Str) when is_binary(Str) ->
|
||||
string:lexemes(Str, [$\s, $\n, [$\r,$\n]]);
|
||||
words_(L) when is_list(L) ->
|
||||
lists:flatmap(fun words_/1, L);
|
||||
words_(_) ->
|
||||
[].
|
||||
```
|
||||
|
||||
We can register the plugin and use it in table definitions:
|
||||
|
||||
```text
|
||||
Eshell V12.1.3 (abort with ^G)
|
||||
1> mnesia:start().
|
||||
ok
|
||||
2> mnesia_schema:add_index_plugin({words}, words, words_f).
|
||||
{atomic,ok}
|
||||
3> mnesia:create_table(i, [{index, [{words}]}]).
|
||||
{atomic,ok}
|
||||
```
|
||||
|
||||
Note that in this case, we had neither a backend plugin, nor even a persistent schema. Index plugins can be used with all table types. The registered indexing function (arity 3) must exist as an exported function along the node's code path.
|
||||
|
||||
To see what happens when we insert an object, we can turn on call trace.
|
||||
|
||||
```text
|
||||
4> dbg:tracer().
|
||||
{ok,<0.108.0>}
|
||||
5> dbg:tp(words, x).
|
||||
{ok,[{matched,nonode@nohost,3},{saved,x}]}
|
||||
6> dbg:p(all,[c]).
|
||||
{ok,[{matched,nonode@nohost,60}]}
|
||||
7> mnesia:dirty_write({i,<<"one two">>, [<<"three">>, <<"four">>]}).
|
||||
(<0.84.0>) call words:words_f(i,{words},{i,<<"one two">>,[<<"three">>,<<"four">>]})
|
||||
(<0.84.0>) returned from words:words_f/3 -> [<<"one">>,<<"two">>,<<"three">>,
|
||||
<<"four">>]
|
||||
(<0.84.0>) call words:words_f(i,{words},{i,<<"one two">>,[<<"three">>,<<"four">>]})
|
||||
(<0.84.0>) returned from words:words_f/3 -> [<<"one">>,<<"two">>,<<"three">>,
|
||||
<<"four">>]
|
||||
ok
|
||||
8> dbg:ctp('_'), dbg:stop().
|
||||
ok
|
||||
9> mnesia:dirty_index_read(i, <<"one">>, {words}).
|
||||
[{i,<<"one two">>,[<<"three">>,<<"four">>]}]
|
||||
```
|
||||
|
||||
(The fact that the indexing function is called twice, seems like a performance bug.)
|
||||
|
||||
We can observe that the indexing callback is able to operate on the whole object. It needs to be side-effect free and efficient, since it will be called at least once for each update (if an old object exists in the table, the indexing function will be called on it too, before it is replaced by the new object.)
|
||||
|
||||
### Rocksdb
|
||||
|
||||
### Usage
|
||||
+5
-2
@@ -26,12 +26,15 @@
|
||||
, {meck, "0.9.2"}
|
||||
, {trace_runner, {git, "https://github.com/uwiger/trace_runner.git",
|
||||
{ref, "2e56677"}}}
|
||||
]}
|
||||
]},
|
||||
{dialyzer, [{plt_apps, all_deps},
|
||||
{base_plt_apps, [erts, kernel, stdlib, mnesia, runtime_tools, eunit,
|
||||
proper, trace_runner, common_test]}]}
|
||||
]},
|
||||
{edown,
|
||||
%% Use as `rebar3 as edown do edoc`
|
||||
[
|
||||
{deps, [{edown, "0.8.4"}]},
|
||||
{deps, [{edown, "0.9.2"}]},
|
||||
{edoc_opts,
|
||||
[{doclet, edown_doclet},
|
||||
{app_default, "http://www.erlang.org/doc/man"},
|
||||
|
||||
+3
-16
@@ -262,18 +262,9 @@ i_show_table(_, _, 0, _) ->
|
||||
i_show_table(I, Move, Limit, Ref) ->
|
||||
case rocksdb:iterator_move(I, Move) of
|
||||
{ok, EncKey, EncVal} ->
|
||||
{Type,Val} =
|
||||
case EncKey of
|
||||
<< ?INFO_TAG, K/binary >> ->
|
||||
K1 = decode_key(K, Ref),
|
||||
V = decode_val(EncVal, K1, Ref),
|
||||
{info,V};
|
||||
_ ->
|
||||
K = decode_key(EncKey, Ref),
|
||||
V = decode_val(EncVal, K, Ref),
|
||||
{data,V}
|
||||
end,
|
||||
io:fwrite("~p: ~p~n", [Type, Val]),
|
||||
Val = decode_val(EncVal, K, Ref),
|
||||
io:fwrite("~p~n", [Val]),
|
||||
i_show_table(I, next, Limit-1, Ref);
|
||||
_ ->
|
||||
ok
|
||||
@@ -675,11 +666,7 @@ select(_Alias, Tab, Ms, Limit) when Limit==infinity; is_integer(Limit) ->
|
||||
|
||||
slot(_Alias, Tab, Pos) when is_integer(Pos), Pos >= 0 ->
|
||||
#{semantics := Sem} = Ref = get_ref(Tab),
|
||||
Start = case Ref of
|
||||
#{type := standalone, vsn := 1} -> <<?DATA_START>>;
|
||||
_ -> first
|
||||
end,
|
||||
First = fun(I) -> rocksdb:iterator_move(I, Start) end,
|
||||
First = fun(I) -> rocksdb:iterator_move(I, first) end,
|
||||
F = case Sem of
|
||||
bag -> fun(I) -> slot_iter_set(First(I), I, 0, Pos, Ref) end;
|
||||
_ -> fun(I) -> slot_iter_set(First(I), I, 0, Pos, Ref) end
|
||||
|
||||
+17
-198
@@ -78,7 +78,7 @@
|
||||
| {add_aliases, [alias()]}
|
||||
| {write_table_property, tabname(), tuple()}
|
||||
| {remove_aliases, [alias()]}
|
||||
| {migrate, [{tabname(), map()}], rpt()}
|
||||
| {migrate, [tabname() | {tabname(), map()}], rpt()}
|
||||
| {prep_close, table()}
|
||||
| {close_table, table()}
|
||||
| {clear_table, table() | cf() }.
|
||||
@@ -250,31 +250,6 @@ get_info_res(Res, Default) ->
|
||||
error(E)
|
||||
end.
|
||||
|
||||
%% Admin info: metadata written by the admin proc to keep track of
|
||||
%% the derived status of tables (such as detected version and encoding
|
||||
%% of existing standalone tables.)
|
||||
%%
|
||||
write_admin_info(K, V, Alias, Name) ->
|
||||
mrdb:rdb_put(get_ref({admin, Alias}),
|
||||
admin_info_key(K, Name),
|
||||
term_to_binary(V)).
|
||||
|
||||
read_admin_info(K, Alias, Name) ->
|
||||
EncK = admin_info_key(K, Name),
|
||||
case mrdb:rdb_get(get_ref({admin,Alias}), EncK) of
|
||||
{ok, Bin} ->
|
||||
{ok, binary_to_term(Bin)};
|
||||
_ ->
|
||||
error
|
||||
end.
|
||||
|
||||
delete_admin_info(K, Alias, Name) ->
|
||||
EncK = admin_info_key(K, Name),
|
||||
mrdb:rdb_delete(get_ref({admin, Alias}), EncK).
|
||||
|
||||
admin_info_key(K, Name) ->
|
||||
mnesia_rocksdb_lib:encode_key({admin_info, Name, K}, sext).
|
||||
|
||||
%% Table metadata info maintained by users
|
||||
%%
|
||||
write_info(Alias, Tab, K, V) ->
|
||||
@@ -285,7 +260,6 @@ write_info_(Ref, Tab, K, V) ->
|
||||
|
||||
write_info_encv(Ref, Tab, K, V) ->
|
||||
EncK = mnesia_rocksdb_lib:encode_key({info,Tab,K}, sext),
|
||||
maybe_write_standalone_info(Ref, K, V),
|
||||
mrdb:rdb_put(Ref, EncK, V, []).
|
||||
|
||||
delete_info(Alias, Tab, K) ->
|
||||
@@ -293,36 +267,21 @@ delete_info(Alias, Tab, K) ->
|
||||
|
||||
delete_info_(Ref, Tab, K) ->
|
||||
EncK = mnesia_rocksdb_lib:encode_key({info, Tab, K}, sext),
|
||||
maybe_delete_standalone_info(Ref, K),
|
||||
mrdb:rdb_delete(Ref, EncK, []).
|
||||
|
||||
maybe_write_standalone_info(Ref, K, V) ->
|
||||
case Ref of
|
||||
#{type := standalone, vsn := 1, db_ref := DbRef} ->
|
||||
EncK = mnesia_rocksdb_lib:encode_key(K, sext),
|
||||
Key = <<?INFO_TAG, EncK/binary>>,
|
||||
EncV = mnesia_rocksdb_lib:encode_val(V, term),
|
||||
rocksdb:put(DbRef, Key, EncV, []);
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
maybe_delete_standalone_info(Ref, K) ->
|
||||
case Ref of
|
||||
#{type := standalone, vsn := 1, db_ref := DbRef} ->
|
||||
EncK = mnesia_rocksdb_lib:encode_key(K, sext),
|
||||
Key = <<?INFO_TAG, EncK/binary>>,
|
||||
rocksdb:delete(DbRef, Key, []);
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
write_table_property(Alias, Tab, Prop) when is_tuple(Prop), size(Prop) >= 1 ->
|
||||
call(Alias, {write_table_property, Tab, Prop}).
|
||||
|
||||
-spec migrate_standalone(alias(), Tabs) -> Res when
|
||||
Tabs :: [tabname() | {tabname(),map()}],
|
||||
Res :: [{tabname(), {ok,any()} | {error, any()}}].
|
||||
migrate_standalone(Alias, Tabs) ->
|
||||
migrate_standalone(Alias, Tabs, undefined).
|
||||
|
||||
-spec migrate_standalone(alias(), Tabs, Rpt) -> Res when
|
||||
Tabs :: [tabname() | {tabname(),map()}],
|
||||
Rpt :: 'undefined' | pid() | atom(),
|
||||
Res :: [{tabname(), {ok,any()} | {error, any()}}].
|
||||
migrate_standalone(Alias, Tabs, Rpt0) ->
|
||||
Rpt = case Rpt0 of
|
||||
undefined -> undefined;
|
||||
@@ -813,10 +772,10 @@ create_table_from_trec(Alias, Name, #{type := column_family} = TRec,
|
||||
false ->
|
||||
create_table_as_cf(Alias, Name, TRec#{db_ref => DbRef}, St);
|
||||
{false, MP} ->
|
||||
create_table_as_standalone(Alias, Name, true, MP, TRec, St);
|
||||
create_table_as_standalone(Alias, Name, MP, TRec, St);
|
||||
{true, MP} ->
|
||||
?log(debug, "will create ~p as standalone and migrate", [Name]),
|
||||
case create_table_as_standalone(Alias, Name, false, MP, TRec, St) of
|
||||
case create_table_as_standalone(Alias, Name, MP, TRec, St) of
|
||||
{ok, OldTRec, _} ->
|
||||
?log(info, "Migrating ~p to column family", [Name]),
|
||||
create_cf_and_migrate(Alias, Name, OldTRec, TRec, Backend, undefined, St);
|
||||
@@ -825,8 +784,8 @@ create_table_from_trec(Alias, Name, #{type := column_family} = TRec,
|
||||
end
|
||||
end;
|
||||
create_table_from_trec(Alias, Name, #{type := standalone} = TRec, _, St) ->
|
||||
{Exists, MP} = table_exists_as_standalone(Name),
|
||||
create_table_as_standalone(Alias, Name, Exists, MP, TRec, St).
|
||||
{_, MP} = table_exists_as_standalone(Name),
|
||||
create_table_as_standalone(Alias, Name, MP, TRec, St).
|
||||
|
||||
create_cf_and_migrate(Alias, Name, OldTRec, TRec, #{db_ref := DbRef}, Rpt, St) ->
|
||||
?log(debug, "Migrate to cf (~p)", [Name]),
|
||||
@@ -1106,29 +1065,12 @@ table_exists_as_standalone(Name) ->
|
||||
end,
|
||||
{Exists, MP}.
|
||||
|
||||
create_table_as_standalone(Alias, Name, Exists, MP, TRec, St) ->
|
||||
case create_table_as_standalone_(Alias, Name, Exists, MP, TRec, St) of
|
||||
{ok, #{type := standalone, vsn := Vsn1,
|
||||
encoding := Enc1} = Cf, _St1} = Ok ->
|
||||
write_admin_info(standalone_vsn_and_enc, {Vsn1, Enc1},
|
||||
Alias, Name),
|
||||
case Vsn1 of
|
||||
1 ->
|
||||
load_info(Alias, Name, Cf);
|
||||
_ ->
|
||||
skip
|
||||
end,
|
||||
Ok;
|
||||
Other ->
|
||||
Other
|
||||
end.
|
||||
|
||||
create_table_as_standalone_(Alias, Name, Exists, MP, TRec, St) ->
|
||||
create_table_as_standalone(Alias, Name, MP, TRec, St) ->
|
||||
Vsn = check_version(TRec),
|
||||
TRec1 = TRec#{vsn => Vsn, encoding => get_encoding(Vsn, TRec)},
|
||||
do_open_standalone(true, Alias, Name, Exists, MP, TRec1, St).
|
||||
do_open_standalone(true, Alias, Name, MP, TRec1, St).
|
||||
|
||||
do_open_standalone(CreateIfMissing, Alias, Name, Exists, MP, TRec0,
|
||||
do_open_standalone(CreateIfMissing, Alias, Name, MP, TRec0,
|
||||
#st{standalone = Ts} = St) ->
|
||||
Opts = rocksdb_opts_from_trec(TRec0),
|
||||
case open_db_(MP, Alias, Opts, [], CreateIfMissing) of
|
||||
@@ -1138,146 +1080,24 @@ do_open_standalone(CreateIfMissing, Alias, Name, Exists, MP, TRec0,
|
||||
DbRec1 = DbRec#{ cfs => CfNames,
|
||||
mountpoint => MP },
|
||||
TRec = maps:merge(TRec0, DbRec#{type => standalone}),
|
||||
TRec1 = guess_table_vsn_and_encoding(Exists, TRec),
|
||||
{ok, TRec1, St#st{standalone = Ts#{{Alias, Name} => DbRec1}}};
|
||||
{ok, TRec, St#st{standalone = Ts#{{Alias, Name} => DbRec1}}};
|
||||
{error, _} = Err ->
|
||||
?log(debug, "open_db error: ~p", [Err]),
|
||||
Err
|
||||
end.
|
||||
|
||||
%% When opening a standalone table, chances are it's a legacy table
|
||||
%% where legacy encoding is already in place. We try to read the
|
||||
%% first object and apply legacy encoding. If successful, we set
|
||||
%% legacy encoding in the TRec. If we migrate the data to a column
|
||||
%% family, we should apply the defined encoding for the cf.
|
||||
%%
|
||||
%% The first object can either be an info object (in the legacy case)
|
||||
%% or a data object, with a sext-encoded key, and a term_to_binary-
|
||||
%% encoded object as value, where the key position is set to [].
|
||||
%% The info objects would be sext-encoded key + term-encoded value.
|
||||
guess_table_vsn_and_encoding(false, TRec) ->
|
||||
TRec;
|
||||
guess_table_vsn_and_encoding(true, #{properties := #{attributes := As},
|
||||
alias := Alias, name := Name} = R) ->
|
||||
case read_admin_info(standalone_vsn_and_enc, Alias, Name) of
|
||||
{ok, {V, E}} ->
|
||||
R#{vsn => V, encoding => E};
|
||||
error ->
|
||||
R1 = set_default_guess(R),
|
||||
mrdb:with_rdb_iterator(
|
||||
R1, fun(I) ->
|
||||
guess_table_vsn_and_encoding_(
|
||||
mrdb:rdb_iterator_move(I, first), I, As, R1)
|
||||
end)
|
||||
end.
|
||||
|
||||
set_default_guess(#{type := standalone} = R) ->
|
||||
case application:get_env(mnesia_rocksdb, standalone_default_vsn, ?VSN) of
|
||||
1 ->
|
||||
R#{vsn => 1, encoding => {sext, {object, term}}};
|
||||
V ->
|
||||
R#{vsn => V}
|
||||
end.
|
||||
|
||||
guess_table_vsn_and_encoding_({ok, K, V}, _I, As, R) ->
|
||||
Arity = length(As) + 1,
|
||||
case K of
|
||||
<<?INFO_TAG, EncK/binary>> ->
|
||||
try _ = {mnesia_rocksdb_lib:decode(EncK, sext),
|
||||
mnesia_rocksdb_lib:decode(V, term)},
|
||||
%% This is a vsn 1 standalone table
|
||||
R#{vsn => 1, encoding => {sext, {object, term}}}
|
||||
catch
|
||||
error:_ ->
|
||||
R
|
||||
end;
|
||||
_ ->
|
||||
Enc = guess_obj_encoding(K, V, Arity),
|
||||
R#{encoding => Enc}
|
||||
end;
|
||||
guess_table_vsn_and_encoding_(_Other, _, _, R) ->
|
||||
R.
|
||||
|
||||
guess_obj_encoding(K, V, Arity) ->
|
||||
{guess_key_encoding(K), guess_val_encoding(V, Arity)}.
|
||||
|
||||
guess_encoding(Bin) ->
|
||||
try {sext, sext:decode(Bin)}
|
||||
catch
|
||||
error:_ ->
|
||||
try {term, binary_to_term(Bin)}
|
||||
catch
|
||||
error:_ -> raw
|
||||
end
|
||||
end.
|
||||
|
||||
guess_key_encoding(Bin) ->
|
||||
case guess_encoding(Bin) of
|
||||
raw -> raw;
|
||||
{Enc, _} -> Enc
|
||||
end.
|
||||
|
||||
guess_val_encoding(Bin, Arity) ->
|
||||
case guess_encoding(Bin) of
|
||||
raw -> {value, raw};
|
||||
{Enc, Term} ->
|
||||
if is_tuple(Term), size(Term) == Arity,
|
||||
element(2, Term) == [] ->
|
||||
{object, Enc};
|
||||
true ->
|
||||
{value, Enc}
|
||||
end
|
||||
end.
|
||||
|
||||
%% This is slightly different from `rocksdb:is_empty/1`, since it allows
|
||||
%% for the presence of some metadata, and still considers it empty if there
|
||||
%% is no user data.
|
||||
table_is_empty(#{} = DbRec) ->
|
||||
Start = iterator_data_start(DbRec),
|
||||
mrdb:with_rdb_iterator(
|
||||
DbRec, fun(I) ->
|
||||
case mrdb:rdb_iterator_move(I, Start) of
|
||||
case mrdb:rdb_iterator_move(I, first) of
|
||||
{ok, _, _} -> false;
|
||||
_ -> true
|
||||
end
|
||||
end).
|
||||
|
||||
iterator_data_start(#{vsn := 1}) -> <<?DATA_START>>;
|
||||
iterator_data_start(_) -> first.
|
||||
|
||||
load_info(Alias, Name, Cf) ->
|
||||
ARef = get_ref({admin, Alias}),
|
||||
mrdb:with_rdb_iterator(
|
||||
Cf, fun(I) ->
|
||||
load_info_(rocksdb:iterator_move(I, first), I, ARef, Name)
|
||||
end).
|
||||
|
||||
load_info_(Res, I, ARef, Tab) ->
|
||||
case Res of
|
||||
{ok, << ?INFO_TAG, K/binary >>, V} ->
|
||||
DecK = mnesia_rocksdb_lib:decode_key(K),
|
||||
case read_info_(ARef, Tab, DecK, undefined) of
|
||||
undefined ->
|
||||
write_info_encv(ARef, Tab, DecK, V);
|
||||
<<131,_/binary>> = Value ->
|
||||
%% Due to a previous bug, info values could be double-encoded with binary_to_term()
|
||||
try binary_to_term(Value) of
|
||||
_DecVal ->
|
||||
%% We haven't been storing erlang-term encoded data as info,
|
||||
%% so assume this is double-encoded and correct
|
||||
write_info_encv(ARef, Tab, DecK, Value)
|
||||
catch
|
||||
error:_ ->
|
||||
skip
|
||||
end;
|
||||
_ ->
|
||||
skip
|
||||
end,
|
||||
load_info_(rocksdb:iterator_move(I, next), I, ARef, Tab);
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
check_version(TRec) ->
|
||||
user_property(mrdb_version, TRec, ?VSN).
|
||||
|
||||
@@ -1607,7 +1427,6 @@ close_and_delete_standalone(#{alias := Alias,
|
||||
case get_table_mountpoint(Alias, Name, St) of
|
||||
{ok, MP} ->
|
||||
close_and_delete(DbRef, MP),
|
||||
delete_admin_info(standalone_vsn_and_enc, Alias, Name),
|
||||
St#st{standalone = maps:remove({Alias,Name}, St#st.standalone)};
|
||||
error ->
|
||||
St
|
||||
|
||||
+108
-39
@@ -66,9 +66,13 @@
|
||||
, prev/2 , prev/3
|
||||
, last/1 , last/2
|
||||
, select/2 , select/3
|
||||
, select_reverse/2, select_reverse/3
|
||||
, select/1
|
||||
, ms/2
|
||||
, fold/3 , fold/4 , fold/5
|
||||
, fold_reverse/3 , fold_reverse/4, fold_reverse/5
|
||||
, rdb_fold/4 , rdb_fold/5
|
||||
, rdb_fold_reverse/4, rdb_fold_reverse/5
|
||||
, write_info/3
|
||||
, read_info/2
|
||||
, read_info/1
|
||||
@@ -101,6 +105,7 @@
|
||||
, db_ref/0
|
||||
, ref_or_tab/0
|
||||
, index_position/0
|
||||
, match_pattern/0
|
||||
]).
|
||||
|
||||
-include("mnesia_rocksdb.hrl").
|
||||
@@ -122,6 +127,9 @@
|
||||
-type outer() :: non_neg_integer().
|
||||
-type retries() :: outer() | {inner(), outer()}.
|
||||
|
||||
-type matchpat_map() :: #{atom() => _}.
|
||||
-type match_pattern() :: matchpat_map() | ets:match_pattern().
|
||||
|
||||
%% activity type 'ets' makes no sense in this context
|
||||
-type mnesia_activity_type() :: transaction
|
||||
| sync_transaction
|
||||
@@ -129,14 +137,15 @@
|
||||
| sync_dirty.
|
||||
|
||||
-type tx_options() :: #{ retries => retries()
|
||||
, no_snapshot => boolean() }.
|
||||
, no_snapshot => boolean()
|
||||
, mnesia_compatible => boolean() }.
|
||||
-type mrdb_activity_type() :: tx | {tx, tx_options()} | batch.
|
||||
|
||||
-type activity_type() :: mrdb_activity_type() | mnesia_activity_type().
|
||||
|
||||
-type key() :: any().
|
||||
-type obj() :: tuple().
|
||||
-type index_position() :: atom() | pos().
|
||||
-type index_position() :: atom() | pos() | plugin_ix_pos().
|
||||
|
||||
-type db_handle() :: rocksdb:db_handle().
|
||||
-type cf_handle() :: rocksdb:cf_handle().
|
||||
@@ -152,10 +161,14 @@
|
||||
-type activity() :: tx_activity() | batch_activity().
|
||||
|
||||
-type pos() :: non_neg_integer().
|
||||
-type plugin_ix_pos() :: {atom()}.
|
||||
|
||||
-type propkey() :: any().
|
||||
-type propvalue() :: any().
|
||||
-type properties() :: #{ record_name := atom()
|
||||
, attributes := [atom()]
|
||||
, index := [{pos(), bag | ordered}]
|
||||
, user_properties => #{propkey() => propvalue()}
|
||||
}.
|
||||
-type semantics() :: bag | set.
|
||||
-type key_encoding() :: 'raw' | 'sext' | 'term'.
|
||||
@@ -540,7 +553,6 @@ re_throw(Cat, Err) ->
|
||||
throw -> throw(Err)
|
||||
end.
|
||||
|
||||
|
||||
mnesia_compatible_aborts() ->
|
||||
mnesia_rocksdb_admin:get_cached_env(mnesia_compatible_aborts, false).
|
||||
|
||||
@@ -661,10 +673,25 @@ maybe_tx_ctxt([#{activity := #{type := Type} = A} = C|_], R) ->
|
||||
inherit_ctxt(Ref, R) ->
|
||||
maps:merge(Ref, maps:with([snapshot, activity], R)).
|
||||
|
||||
%% @doc Create an iterator on table `Tab' for the duration of `Fun'
|
||||
%%
|
||||
%% The iterator is passed to the provided fun as `Fun(Iterator)', and is
|
||||
%% closed once the fun terminates.
|
||||
%% @equiv with_iterator(Tab, Fun, [])
|
||||
%% @end
|
||||
-spec with_iterator(ref_or_tab(), fun( (mrdb_iterator()) -> Res )) -> Res.
|
||||
with_iterator(Tab, Fun) ->
|
||||
with_iterator(Tab, Fun, []).
|
||||
|
||||
%% @doc Create an iterator on table `Tab' with `ReadOptions' for the duration of `Fun'
|
||||
%%
|
||||
%% The iterator is passed to the provided fun as `Fun(Iterator)', and is
|
||||
%% closed once the fun terminates.
|
||||
%%
|
||||
%% The iterator respects `mnesia_rocksdb' metadata, so accesses through the iterator
|
||||
%% will return `{ok, Obj}' where `Obj' is the complete decoded object.
|
||||
%% For rocksdb-level iterators, see {@link with_rdb_iterator/3}.
|
||||
%% @end
|
||||
-spec with_iterator(ref_or_tab(), fun( (mrdb_iterator()) -> Res ), read_options()) -> Res.
|
||||
with_iterator(Tab, Fun, Opts) ->
|
||||
R = ensure_ref(Tab),
|
||||
@@ -1300,9 +1327,65 @@ select(Tab, Pat, Limit) when Limit == infinity; is_integer(Limit), Limit > 0 ->
|
||||
true = valid_limit(Limit),
|
||||
mrdb_select:select(ensure_ref(Tab), Pat, Limit).
|
||||
|
||||
select_reverse(Tab, Pat) ->
|
||||
select_reverse(Tab, Pat, infinity).
|
||||
|
||||
select_reverse(Tab, Pat, Limit) when Limit == infinity; is_integer(Limit), Limit > 0 ->
|
||||
true = valid_limit(Limit),
|
||||
mrdb_select:select_reverse(ensure_ref(Tab), Pat, Limit).
|
||||
|
||||
select(Cont) ->
|
||||
mrdb_select:select(Cont).
|
||||
|
||||
%% @doc Produce a match specification for select(), supporting map-based match patterns
|
||||
%%
|
||||
%% Using record syntax in match patterns tends to conflict with type checking. This
|
||||
%% function offers an alternative approach, drawing on the fact that mnesia_rocksdb
|
||||
%% keeps the record name and attribute names readily available as persistent terms.
|
||||
%%
|
||||
%% When using the map-based representation, the match pattern is built by matching
|
||||
%% attribute names to map elements; any attribute not found in the map gets set to '_'.
|
||||
%% Thus, ```[{#balance{key = {Acct,'$1'},_='_'},[{'>=','$1',Height}],['$_']}]''' can be
|
||||
%% created as ```ms(balance,[{#{key => {Acct,'$1'}},[{'>=','$1',Height}],['$_']}])'''.
|
||||
%%
|
||||
%% This has the advantage over `ms_transform' that it can handle bound variables
|
||||
%% in the match pattern. The function works on all mnesia table types.
|
||||
%% @end
|
||||
-spec ms(ref_or_tab(), [{match_pattern(), [_], [_]}]) -> ets:match_spec().
|
||||
ms(Tab, Pat) ->
|
||||
#{ attributes := Attrs
|
||||
, record_name := RecName } = any_tab_props(Tab),
|
||||
[{headpat(RecName, Attrs, Hd), Gs, Body}
|
||||
|| {Hd, Gs, Body} <- Pat].
|
||||
|
||||
any_tab_props(Tab) ->
|
||||
try mrdb_props(Tab)
|
||||
catch
|
||||
exit:{aborted,{bad_type,_}} ->
|
||||
mnesia_props(Tab)
|
||||
end.
|
||||
|
||||
mrdb_props(Tab) ->
|
||||
#{properties := Props} = get_ref(Tab),
|
||||
Props.
|
||||
|
||||
mnesia_props(Tab) ->
|
||||
try mnesia_props_(Tab)
|
||||
catch
|
||||
exit:{aborted, _} ->
|
||||
mnesia:abort({bad_type, Tab})
|
||||
end.
|
||||
|
||||
mnesia_props_(Tab) ->
|
||||
#{ record_name => mnesia:table_info(Tab, record_name)
|
||||
, attributes => mnesia:table_info(Tab, attributes) }.
|
||||
|
||||
headpat(RecName, Attrs, Hd) when is_map(Hd) ->
|
||||
list_to_tuple([RecName | [maps:get(A, Hd, '_')
|
||||
|| A <- Attrs]]);
|
||||
headpat(_, _, Hd) when is_tuple(Hd); is_atom(Hd) ->
|
||||
Hd.
|
||||
|
||||
clear_table(Tab) ->
|
||||
match_delete(Tab, '_').
|
||||
|
||||
@@ -1354,6 +1437,16 @@ fold(Tab, Fun, Acc, MatchSpec, Limit) ->
|
||||
true = valid_limit(Limit),
|
||||
mrdb_select:fold(ensure_ref(Tab), Fun, Acc, MatchSpec, Limit).
|
||||
|
||||
fold_reverse(Tab, Fun, Acc) ->
|
||||
fold_reverse(Tab, Fun, Acc, [{'_', [], ['$_']}]).
|
||||
|
||||
fold_reverse(Tab, Fun, Acc, MatchSpec) ->
|
||||
fold_reverse(Tab, Fun, Acc, MatchSpec, infinity).
|
||||
|
||||
fold_reverse(Tab, Fun, Acc, MatchSpec, Limit) ->
|
||||
true = valid_limit(Limit),
|
||||
mrdb_select:fold_reverse(ensure_ref(Tab), Fun, Acc, MatchSpec, Limit).
|
||||
|
||||
rdb_fold(Tab, Fun, Acc, Prefix) when is_function(Fun, 3)
|
||||
, is_binary(Prefix) ->
|
||||
rdb_fold(Tab, Fun, Acc, Prefix, infinity).
|
||||
@@ -1363,6 +1456,15 @@ rdb_fold(Tab, Fun, Acc, Prefix, Limit) when is_function(Fun, 3)
|
||||
true = valid_limit(Limit),
|
||||
mrdb_select:rdb_fold(ensure_ref(Tab), Fun, Acc, Prefix, Limit).
|
||||
|
||||
rdb_fold_reverse(Tab, Fun, Acc, Prefix) when is_function(Fun, 3)
|
||||
, is_binary(Prefix) ->
|
||||
rdb_fold_reverse(Tab, Fun, Acc, Prefix, infinity).
|
||||
|
||||
rdb_fold_reverse(Tab, Fun, Acc, Prefix, Limit) when is_function(Fun, 3)
|
||||
, is_binary(Prefix) ->
|
||||
true = valid_limit(Limit),
|
||||
mrdb_select:rdb_fold_reverse(ensure_ref(Tab), Fun, Acc, Prefix, Limit).
|
||||
|
||||
valid_limit(L) ->
|
||||
case L of
|
||||
infinity ->
|
||||
@@ -1374,15 +1476,7 @@ valid_limit(L) ->
|
||||
end.
|
||||
|
||||
write_info(Tab, K, V) ->
|
||||
R = ensure_ref(Tab),
|
||||
Alias = case R of
|
||||
#{type := standalone, vsn := 1, alias := A} = TRef ->
|
||||
%% Also write on legacy info format
|
||||
write_info_standalone(TRef, K, V),
|
||||
A;
|
||||
#{alias := A} ->
|
||||
A
|
||||
end,
|
||||
#{alias := Alias} = ensure_ref(Tab),
|
||||
write_info_(ensure_ref({admin, Alias}), Tab, K, V).
|
||||
|
||||
write_info_(#{} = R, Tab, K, V) ->
|
||||
@@ -1399,13 +1493,8 @@ read_info(Tab, K) ->
|
||||
read_info(Tab, K, Default) when K==size; K==memory ->
|
||||
read_direct_info_(ensure_ref(Tab), K, Default);
|
||||
read_info(Tab, K, Default) ->
|
||||
#{alias := Alias} = R = ensure_ref(Tab),
|
||||
case R of
|
||||
#{type := standalone, vsn := 1} = TRef ->
|
||||
read_info_standalone(TRef, K, Default);
|
||||
#{alias := Alias} ->
|
||||
mnesia_rocksdb_admin:read_info(Alias, Tab, K, Default)
|
||||
end.
|
||||
#{alias := Alias} = ensure_ref(Tab),
|
||||
mnesia_rocksdb_admin:read_info(Alias, Tab, K, Default).
|
||||
|
||||
read_direct_info_(R, memory, _Def) ->
|
||||
get_property(R, <<"rocksdb.total-sst-files-size">>, integer, 0);
|
||||
@@ -1429,26 +1518,6 @@ get_property(#{db_ref := R, cf_handle := CfH}, Prop, Type, Default) ->
|
||||
%%rocksdb_boolean(<<"1">>) -> true;
|
||||
%%rocksdb_boolean(<<"0">>) -> false.
|
||||
|
||||
write_info_standalone(#{} = R, K, V) ->
|
||||
EncK = <<?INFO_TAG, (encode_key(K, sext))/binary>>,
|
||||
EncV = term_to_binary(V),
|
||||
rdb_put(R, EncK, EncV, write_opts(R, [])).
|
||||
|
||||
read_info_standalone(#{} = R, K, Default) ->
|
||||
EncK = <<?INFO_TAG, (encode_key(K, sext))/binary>>,
|
||||
get_info_res(rdb_get(R, EncK, read_opts(R, [])), Default).
|
||||
|
||||
get_info_res(Res, Default) ->
|
||||
case Res of
|
||||
not_found ->
|
||||
Default;
|
||||
{ok, Bin} ->
|
||||
%% no fancy tricks when encoding/decoding info values
|
||||
binary_to_term(Bin);
|
||||
{error, E} ->
|
||||
error(E)
|
||||
end.
|
||||
|
||||
%% insert_bag_v2(Ref, K, V, Opts) ->
|
||||
%% rdb_merge(Ref, K, {list_append, [V]}
|
||||
|
||||
|
||||
+52
-4
@@ -9,6 +9,10 @@
|
||||
, fold/4
|
||||
, rev_fold/4
|
||||
, index_ref/2
|
||||
, select/3
|
||||
, select/4
|
||||
, select_reverse/3
|
||||
, select_reverse/4
|
||||
]).
|
||||
|
||||
-record(mrdb_ix_iter, { i :: mrdb:mrdb_iterator()
|
||||
@@ -33,6 +37,11 @@
|
||||
|
||||
-export_type([ ix_iterator/0 ]).
|
||||
|
||||
-spec index_ref(mrdb:ref_or_tab(), mrdb:index_position()) -> mrdb:db_ref().
|
||||
index_ref(Tab, Ix) ->
|
||||
#{} = R = mrdb:ensure_ref(Tab),
|
||||
ensure_index_ref(Ix, R).
|
||||
|
||||
-spec with_iterator(mrdb:ref_or_tab(), mrdb:index_position(), fun( (ix_iterator()) -> Res)) -> Res.
|
||||
with_iterator(Tab, IxPos, Fun) when is_function(Fun, 1) ->
|
||||
{ok, I} = iterator(Tab, IxPos),
|
||||
@@ -88,6 +97,26 @@ fold_(Tab, IxPos, Start, Dir, FoldFun, Acc) ->
|
||||
iter_fold(I, Start, Dir, FoldFun, Acc)
|
||||
end).
|
||||
|
||||
select(Tab, Ix, MS) ->
|
||||
select(Tab, Ix, MS, infinity).
|
||||
|
||||
select(Tab, Ix, MS, Limit) ->
|
||||
#{} = R = mrdb:ensure_ref(Tab),
|
||||
#{} = IxR = ensure_index_ref(Ix, R),
|
||||
MSpre = pre_match_spec(MS),
|
||||
mrdb_select:select(IxR#{ derive_obj_f => mk_derive_obj_f(R)
|
||||
, pre_ms => MSpre }, MS, Limit).
|
||||
|
||||
select_reverse(Tab, Ix, MS) ->
|
||||
select_reverse(Tab, Ix, MS, infinity).
|
||||
|
||||
select_reverse(Tab, Ix, MS, Limit) ->
|
||||
#{} = R = mrdb:ensure_ref(Tab),
|
||||
#{} = IxR = ensure_index_ref(Ix, R),
|
||||
MSpre = pre_match_spec(MS),
|
||||
mrdb_select:select_reverse(IxR#{ derive_obj_f => mk_derive_obj_f(R)
|
||||
, pre_ms => MSpre }, MS, Limit).
|
||||
|
||||
iter_fold(I, Start, Dir, Fun, Acc) ->
|
||||
iter_fold_(iterator_move(I, Start), I, Dir, Fun, Acc).
|
||||
|
||||
@@ -96,10 +125,6 @@ iter_fold_({ok, IxVal, Obj}, I, Dir, Fun, Acc) ->
|
||||
iter_fold_({error, _}, _, _, _, Acc) ->
|
||||
Acc.
|
||||
|
||||
index_ref(Tab, Pos) ->
|
||||
TRef = mrdb:ensure_ref(Tab),
|
||||
ensure_index_ref(Pos, TRef).
|
||||
|
||||
iterator_move_set(#mrdb_ix_iter{i = I, sub = Sub}, Dir) ->
|
||||
case mrdb:iterator_move(I, Dir) of
|
||||
{ok, {{FKey, PKey}}} ->
|
||||
@@ -121,6 +146,29 @@ iterator_move_bag(#mrdb_ix_iter{i = I, sub = Sub}, Dir) ->
|
||||
Other
|
||||
end.
|
||||
|
||||
pre_match_spec([{{KeyPat,ObjPat}, Gs, MatchBody} | T]) ->
|
||||
[{{{KeyPat,'_'},ObjPat}, Gs, MatchBody} | pre_match_spec(T)];
|
||||
pre_match_spec([H|T]) ->
|
||||
[H | pre_match_spec(T)];
|
||||
pre_match_spec([]) ->
|
||||
[].
|
||||
|
||||
%% Used for mrdb_select:select()
|
||||
%% The select operation folds over index keys, and for matching keys,
|
||||
%% calls the `derive_obj_f/1` fun, which normally just does `Obj -> [Obj]`.
|
||||
%% In the case of indexes, it fetches the object, if it exists, and then
|
||||
%% returns `[{IndexKey, Object}]`, which is what the matchspec should operate on.
|
||||
%%
|
||||
mk_derive_obj_f(Ref) ->
|
||||
fun({{IxK, Key}}) ->
|
||||
case mrdb:read(Ref, Key, []) of
|
||||
[Obj] ->
|
||||
[{IxK, Obj}];
|
||||
[] ->
|
||||
[]
|
||||
end
|
||||
end.
|
||||
|
||||
-spec opt_read(mrdb:ref_or_tab(), Key :: any()) -> any().
|
||||
opt_read(R, Key) ->
|
||||
case mrdb:read(R, Key, []) of
|
||||
|
||||
+150
-33
@@ -3,9 +3,13 @@
|
||||
|
||||
-export([ select/3 %% (Ref, MatchSpec, Limit)
|
||||
, select/4 %% (Ref, MatchSpec, AccKeys, Limit)
|
||||
, select_reverse/3
|
||||
, select_reverse/4
|
||||
, select/1 %% (Cont)
|
||||
, fold/5 %% (Ref, Fun, Acc, MatchSpec, Limit)
|
||||
, fold_reverse/5
|
||||
, rdb_fold/5 %% (Ref, Fun, Acc, Prefix, Limit)
|
||||
, rdb_fold_reverse/5 %% (Ref, Fun, Acc, Prefix, Limit)
|
||||
]).
|
||||
|
||||
-export([continuation_info/2]).
|
||||
@@ -25,26 +29,41 @@
|
||||
, compiled_ms
|
||||
, limit
|
||||
, key_only = false % TODO: not used
|
||||
, direction = forward % TODO: not used
|
||||
, direction = forward
|
||||
, pre_ms
|
||||
, derive_obj_f = fun unit_l/1
|
||||
}).
|
||||
|
||||
select(Ref, MS, Limit) when is_map(Ref), is_list(MS) ->
|
||||
select(Ref, MS, false, Limit).
|
||||
select(Ref, MS, false, forward, Limit).
|
||||
|
||||
select(Ref, MS, AccKeys, Limit)
|
||||
select(Ref, MS, AccKeys, Limit) ->
|
||||
select(Ref, MS, AccKeys, forward, Limit).
|
||||
|
||||
select_reverse(Ref, MS, Limit) ->
|
||||
select(Ref, MS, false, reverse, Limit).
|
||||
|
||||
select_reverse(Ref, MS, AccKeys, Limit) ->
|
||||
select(Ref, MS, AccKeys, reverse, Limit).
|
||||
|
||||
select(Ref, MS, AccKeys, Dir, Limit)
|
||||
when is_map(Ref), is_list(MS), is_boolean(AccKeys) ->
|
||||
Sel = mk_sel(Ref, MS, Limit),
|
||||
Sel = mk_sel(Ref, MS, Dir, Limit),
|
||||
mrdb:with_rdb_iterator(Ref, fun(I) -> i_select(I, Sel, AccKeys, []) end).
|
||||
|
||||
mk_sel(#{name := Tab} = Ref, MS, Limit) ->
|
||||
Keypat = keypat(MS, keypos(Tab), Ref),
|
||||
mk_sel(#{name := Tab} = Ref, MS, Dir, Limit) ->
|
||||
MSpre = maps:get(pre_ms, Ref, MS),
|
||||
Keypat = keypat(MSpre, keypos(Tab), Ref),
|
||||
#sel{tab = Tab,
|
||||
ref = Ref,
|
||||
keypat = Keypat,
|
||||
ms = MS,
|
||||
compiled_ms = ets:match_spec_compile(MS),
|
||||
ms = MSpre,
|
||||
pre_ms = MSpre,
|
||||
compiled_ms = ms_compile(MS),
|
||||
key_only = needs_key_only(MS),
|
||||
limit = Limit}.
|
||||
direction = Dir,
|
||||
limit = Limit,
|
||||
derive_obj_f = derive_f(Ref)}.
|
||||
|
||||
select(Cont) ->
|
||||
case Cont of
|
||||
@@ -59,11 +78,18 @@ continuation_info(_, _) -> undefined.
|
||||
|
||||
continuation_info_(ref, #sel{ref = Ref}) -> Ref;
|
||||
continuation_info_(ms, #sel{ms = MS }) -> MS;
|
||||
continuation_info_(pre_ms, #sel{pre_ms = MSp}) -> MSp;
|
||||
continuation_info_(limit, #sel{limit = L }) -> L;
|
||||
continuation_info_(direction, #sel{direction = Dir}) -> Dir;
|
||||
continuation_info_(_, _) -> undefined.
|
||||
|
||||
fold(Ref, Fun, Acc, MS, Limit) ->
|
||||
do_fold(Ref, Fun, Acc, MS, forward, Limit).
|
||||
|
||||
fold_reverse(Ref, Fun, Acc, MS, Limit) ->
|
||||
do_fold(Ref, Fun, Acc, MS, reverse, Limit).
|
||||
|
||||
do_fold(Ref, Fun, Acc, MS, Dir, Limit) ->
|
||||
{AccKeys, F} =
|
||||
if is_function(Fun, 3) ->
|
||||
{true, fun({K, Obj}, Acc1) ->
|
||||
@@ -74,7 +100,7 @@ fold(Ref, Fun, Acc, MS, Limit) ->
|
||||
true ->
|
||||
mrdb:abort(invalid_fold_fun)
|
||||
end,
|
||||
fold_(select(Ref, MS, AccKeys, Limit), F, Acc).
|
||||
fold_(select(Ref, MS, AccKeys, Dir, Limit), F, Acc).
|
||||
|
||||
fold_('$end_of_table', _, Acc) ->
|
||||
Acc;
|
||||
@@ -86,12 +112,57 @@ fold_({L, Cont}, Fun, Acc) ->
|
||||
rdb_fold(Ref, Fun, Acc, Prefix, Limit) ->
|
||||
mrdb:with_rdb_iterator(
|
||||
Ref, fun(I) ->
|
||||
MovRes = rocksdb:iterator_move(I, first(Ref)),
|
||||
MovRes = fwd_init_seek(I, Prefix),
|
||||
i_rdb_fold(MovRes, I, Prefix, Fun, Acc, Limit)
|
||||
end).
|
||||
|
||||
first(#{vsn := 1}) -> <<?DATA_START>>;
|
||||
first(_) -> first.
|
||||
rdb_fold_reverse(Ref, Fun, Acc, Prefix, Limit) ->
|
||||
mrdb:with_rdb_iterator(
|
||||
Ref, fun(I) ->
|
||||
MovRes = rev_init_seek(I, Prefix),
|
||||
i_rdb_fold_reverse(MovRes, I, Prefix, Fun, Acc, Limit)
|
||||
end).
|
||||
|
||||
fwd_init_seek(I, Pfx) ->
|
||||
rocksdb:iterator_move(I, fwd_init_seek_tgt(Pfx)).
|
||||
|
||||
fwd_init_seek_tgt(<<>> ) -> first;
|
||||
fwd_init_seek_tgt(Prefix) -> {seek, Prefix}.
|
||||
|
||||
rev_init_seek(I, Pfx) ->
|
||||
case rev_init_seek_tgt(Pfx) of
|
||||
last ->
|
||||
i_move(I, last);
|
||||
{seek, Bin} ->
|
||||
%% An 'incremented' prefix.
|
||||
%% This will fail if we seek past the end of the table.
|
||||
%% Then, try to seek_for_prev instead (fails if table empty).
|
||||
%% This because rocksdb lacks a "seek backward to last matching prefix".
|
||||
case i_move(I, {seek, Bin}) of
|
||||
{error, invalid_iterator} ->
|
||||
i_move(I, {seek_for_prev, Bin});
|
||||
{ok, _, _} = Ok ->
|
||||
Ok
|
||||
end
|
||||
end.
|
||||
|
||||
rev_init_seek_tgt(<<>>) -> last;
|
||||
rev_init_seek_tgt(Prefix) ->
|
||||
case incr_prefix(Prefix) of
|
||||
last -> last;
|
||||
Pfx1 when is_binary(Pfx1) ->
|
||||
{seek, Pfx1}
|
||||
end.
|
||||
|
||||
incr_prefix(<<>>) -> last;
|
||||
incr_prefix(Pfx) when is_binary(Pfx) ->
|
||||
PfxI = binary:decode_unsigned(Pfx),
|
||||
MaxI = (1 bsl (byte_size(Pfx) * 8)) - 1,
|
||||
case PfxI + 1 of
|
||||
I1 when I1 >= MaxI -> last;
|
||||
I1 ->
|
||||
binary:encode_unsigned(I1)
|
||||
end.
|
||||
|
||||
i_rdb_fold({ok, K, V}, I, Pfx, Fun, Acc, Limit) when Limit > 0 ->
|
||||
case is_prefix(Pfx, K) of
|
||||
@@ -104,20 +175,37 @@ i_rdb_fold({ok, K, V}, I, Pfx, Fun, Acc, Limit) when Limit > 0 ->
|
||||
i_rdb_fold(_, _, _, _, Acc, _) ->
|
||||
Acc.
|
||||
|
||||
i_rdb_fold_reverse({ok, K, V}, I, Pfx, Fun, Acc, Limit) when Limit > 0 ->
|
||||
case is_prefix(Pfx, K) of
|
||||
true ->
|
||||
i_rdb_fold_reverse(rocksdb:iterator_move(I, prev), I, Pfx, Fun,
|
||||
Fun(K, V, Acc), decr(Limit));
|
||||
false when K > Pfx ->
|
||||
i_rdb_fold_reverse(rocksdb:iterator_move(I, prev), I, Pfx, Fun, Acc, Limit);
|
||||
false ->
|
||||
Acc
|
||||
end;
|
||||
i_rdb_fold_reverse(_, _, _, _, Acc, _) ->
|
||||
Acc.
|
||||
|
||||
i_select(I, #sel{ keypat = Pfx
|
||||
, compiled_ms = MS
|
||||
, limit = Limit
|
||||
, ref = #{vsn := Vsn, encoding := Enc} } = Sel, AccKeys, Acc) ->
|
||||
StartKey = case {Pfx, Vsn, Enc} of
|
||||
{<<>>, 1, {sext, _}} ->
|
||||
<<?DATA_START>>;
|
||||
{_, _, {term, _}} ->
|
||||
<<>>;
|
||||
, direction = Dir
|
||||
, ref = #{encoding := Enc} } = Sel0, AccKeys, Acc) ->
|
||||
{MoveRes, Sel} = case Enc of
|
||||
{term, _} ->
|
||||
%% No defined ordering - do forward select
|
||||
{i_move(I, first), Sel0#sel{direction = forward}};
|
||||
_ ->
|
||||
Pfx
|
||||
case Dir of
|
||||
forward ->
|
||||
{fwd_init_seek(I, Pfx), Sel0};
|
||||
reverse ->
|
||||
{rev_init_seek(I, Pfx), Sel0}
|
||||
end
|
||||
end,
|
||||
select_traverse(rocksdb:iterator_move(I, StartKey), Limit,
|
||||
Pfx, MS, I, Sel, AccKeys, Acc).
|
||||
select_traverse(MoveRes, Limit, Pfx, MS, I, Sel, AccKeys, Acc).
|
||||
|
||||
needs_key_only([Pat]) ->
|
||||
needs_key_only_(Pat);
|
||||
@@ -193,16 +281,17 @@ map_vars([H|T], P) ->
|
||||
map_vars([], _) ->
|
||||
[].
|
||||
|
||||
select_traverse({ok, K, V}, Limit, Pfx, MS, I, #sel{ref = R} = Sel,
|
||||
select_traverse({ok, K, V}, Limit, Pfx, MS, I, #sel{ref = R, direction = Dir} = Sel,
|
||||
AccKeys, Acc) ->
|
||||
case is_prefix(Pfx, K) of
|
||||
true ->
|
||||
DecKey = decode_key(K, R),
|
||||
Rec = decode_val(V, DecKey, R),
|
||||
case ets:match_spec_run([Rec], MS) of
|
||||
Rec0 = decode_val(V, DecKey, R),
|
||||
RecL = derive_object(Rec0, Sel),
|
||||
case ms_run(RecL, MS) of
|
||||
[] ->
|
||||
select_traverse(
|
||||
rocksdb:iterator_move(I, next), Limit, Pfx, MS,
|
||||
rocksdb:iterator_move(I, next_or_prev(Dir)), Limit, Pfx, MS,
|
||||
I, Sel, AccKeys, Acc);
|
||||
[Match] ->
|
||||
Acc1 = if AccKeys ->
|
||||
@@ -212,6 +301,9 @@ select_traverse({ok, K, V}, Limit, Pfx, MS, I, #sel{ref = R} = Sel,
|
||||
end,
|
||||
traverse_continue(K, decr(Limit), Pfx, MS, I, Sel, AccKeys, Acc1)
|
||||
end;
|
||||
false when Dir == reverse, K > Pfx ->
|
||||
select_traverse(rocksdb:iterator_move(I, prev), Limit, Pfx, MS,
|
||||
I, Sel, AccKeys, Acc);
|
||||
false when Limit == infinity ->
|
||||
lists:reverse(Acc);
|
||||
false ->
|
||||
@@ -220,6 +312,9 @@ select_traverse({ok, K, V}, Limit, Pfx, MS, I, #sel{ref = R} = Sel,
|
||||
select_traverse({error, _}, Limit, _, _, _, _, _, Acc) ->
|
||||
select_return(Limit, {lists:reverse(Acc), '$end_of_table'}).
|
||||
|
||||
next_or_prev(forward) -> next;
|
||||
next_or_prev(reverse) -> prev.
|
||||
|
||||
select_return(infinity, {L, '$end_of_table'}) ->
|
||||
L;
|
||||
select_return(_, Ret) ->
|
||||
@@ -239,29 +334,40 @@ decr(I) when is_integer(I) ->
|
||||
decr(infinity) ->
|
||||
infinity.
|
||||
|
||||
traverse_continue(K, 0, Pfx, MS, _I, #sel{limit = Limit, ref = Ref} = Sel, AccKeys, Acc) ->
|
||||
traverse_continue(K, 0, Pfx, MS, _I, #sel{limit = Limit, ref = Ref, direction = Dir} = Sel, AccKeys, Acc) ->
|
||||
{lists:reverse(Acc),
|
||||
fun(sel) -> Sel;
|
||||
(cont) ->
|
||||
mrdb:with_rdb_iterator(
|
||||
Ref,
|
||||
fun(NewI) ->
|
||||
select_traverse(iterator_next(NewI, K),
|
||||
select_traverse(iterator_next(NewI, K, Dir),
|
||||
Limit, Pfx, MS, NewI, Sel,
|
||||
AccKeys, [])
|
||||
end)
|
||||
end};
|
||||
traverse_continue(_K, Limit, Pfx, MS, I, Sel, AccKeys, Acc) ->
|
||||
select_traverse(rocksdb:iterator_move(I, next), Limit, Pfx, MS, I, Sel, AccKeys, Acc).
|
||||
traverse_continue(_K, Limit, Pfx, MS, I, #sel{direction = Dir} = Sel, AccKeys, Acc) ->
|
||||
select_traverse(rocksdb:iterator_move(I, next_or_prev(Dir)), Limit, Pfx, MS, I, Sel, AccKeys, Acc).
|
||||
|
||||
iterator_next(I, K) ->
|
||||
case rocksdb:iterator_move(I, K) of
|
||||
iterator_next(I, K, Dir) ->
|
||||
case i_move(I, K, Dir) of
|
||||
{ok, K, _} ->
|
||||
rocksdb:iterator_move(I, next);
|
||||
rocksdb:iterator_move(I, next_or_prev(Dir));
|
||||
Other ->
|
||||
Other
|
||||
end.
|
||||
|
||||
i_move(I, Tgt) ->
|
||||
rocksdb:iterator_move(I, Tgt).
|
||||
|
||||
i_move(I, K, reverse) ->
|
||||
rocksdb:iterator_move(I, {seek_for_prev, K});
|
||||
i_move(I, K, forward) ->
|
||||
rocksdb:iterator_move(I, K).
|
||||
|
||||
derive_object(R, #sel{derive_obj_f = F}) ->
|
||||
F(R).
|
||||
|
||||
keypat([H|T], KeyPos, Ref) ->
|
||||
keypat(T, KeyPos, Ref, keypat_pfx(H, KeyPos, Ref)).
|
||||
|
||||
@@ -283,3 +389,14 @@ keypat_pfx({HeadPat,_Gs,_}, KeyPos, #{encoding := {sext,_}}) when is_tuple(HeadP
|
||||
keypat_pfx(_, _, _) ->
|
||||
<<>>.
|
||||
|
||||
derive_f(#{derive_obj_f := F}) when is_function(F, 1) -> F;
|
||||
derive_f(_) -> fun unit_l/1.
|
||||
|
||||
unit_l(X) ->
|
||||
[X].
|
||||
|
||||
ms_compile(MS) ->
|
||||
ets:match_spec_compile(MS).
|
||||
|
||||
ms_run(RecL, MS) ->
|
||||
ets:match_spec_run(RecL, MS).
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% basho_bench: Benchmarking Suite
|
||||
%%
|
||||
%% Copyright (c) 2009-2010 Basho Techonologies
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(basho_bench_driver_mnesia_rocksdb).
|
||||
|
||||
-export([new/1,
|
||||
run/4]).
|
||||
|
||||
-include("mnesia_rocksdb_basho_bench.hrl").
|
||||
|
||||
%% ====================================================================
|
||||
%% API
|
||||
%% ====================================================================
|
||||
|
||||
new(_Id) ->
|
||||
Type = basho_bench_config:get(backend, ram_copies),
|
||||
Tab = basho_bench_config:get(mnesia_table, t),
|
||||
ok = bootstrap_mnesia(Tab, Type),
|
||||
{ok, Tab}.
|
||||
|
||||
bootstrap_mnesia(Tab, Type) ->
|
||||
ok = mnesia:create_schema([node()],
|
||||
[{backend_types,
|
||||
[{rocksdb_copies, mnesia_rocksdb}]}]),
|
||||
ok = mnesia:start(),
|
||||
{atomic,ok} = mnesia:create_table(Tab, [{Type, [node()]}]),
|
||||
mnesia:wait_for_tables([Tab], 10000).
|
||||
|
||||
run(get, KeyGen, _ValueGen, State) ->
|
||||
Tab = State,
|
||||
Key = KeyGen(),
|
||||
case mnesia:dirty_read({Tab, Key}) of
|
||||
[] ->
|
||||
{ok, State};
|
||||
[{_, Key, _}] ->
|
||||
{ok, State}
|
||||
end;
|
||||
run(put, KeyGen, ValueGen, State) ->
|
||||
Tab = State,
|
||||
ok = mnesia:dirty_write({Tab, KeyGen(), ValueGen()}),
|
||||
{ok, State};
|
||||
run(delete, KeyGen, _ValueGen, State) ->
|
||||
Tab = State,
|
||||
ok = mnesia:dirty_delete({Tab, KeyGen()}),
|
||||
{ok, State}.
|
||||
@@ -216,6 +216,7 @@ mrdb_transactions_(Config) ->
|
||||
delete_tabs(Created),
|
||||
ok.
|
||||
|
||||
-dialyzer(no_return).
|
||||
mrdb_abort_reasons(_Config) ->
|
||||
Prev = mnesia_rocksdb_admin:set_and_cache_env(mnesia_compatible_aborts, true),
|
||||
X = some_value,
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
|
||||
-define(FAIL_MSG(Str, Args), ?ERROR(Str, Args), basho_bench_app:halt_or_kill()).
|
||||
-define(STD_ERR(Str, Args), io:format(standard_error, Str, Args)).
|
||||
|
||||
-define(CONSOLE(Str, Args), lager:info(Str, Args)).
|
||||
|
||||
-define(DEBUG(Str, Args), lager:debug(Str, Args)).
|
||||
-define(INFO(Str, Args), lager:info(Str, Args)).
|
||||
-define(WARN(Str, Args), lager:warning(Str, Args)).
|
||||
-define(ERROR(Str, Args), lager:error(Str, Args)).
|
||||
|
||||
-define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))).
|
||||
|
||||
-define(VAL_GEN_BLOB_CFG, value_generator_blob_file).
|
||||
-define(VAL_GEN_SRC_SIZE, value_generator_source_size).
|
||||
@@ -1,18 +0,0 @@
|
||||
{mode, max}.
|
||||
|
||||
{duration, 10}.
|
||||
|
||||
{concurrent, 1}.
|
||||
|
||||
{driver, basho_bench_driver_mnesia_rocksdb}.
|
||||
|
||||
{key_generator, {int_to_bin,{uniform_int, 5000000}}}.
|
||||
|
||||
{value_generator, {fixed_bin, 10000}}.
|
||||
|
||||
{operations, [{get, 2}, {put, 2}, {delete, 1}]}.
|
||||
|
||||
{code_paths, []}.
|
||||
|
||||
{mnesia_table, doc}.
|
||||
{backend, disc_only_copies}.
|
||||
@@ -1,19 +0,0 @@
|
||||
{mode, max}.
|
||||
|
||||
{duration, 10}.
|
||||
|
||||
{concurrent, 1}.
|
||||
|
||||
{driver, basho_bench_driver_mnesia_rocksdb}.
|
||||
|
||||
{key_generator, {int_to_bin,{uniform_int, 5000000}}}.
|
||||
|
||||
{value_generator, {fixed_bin, 10000}}.
|
||||
|
||||
{operations, [{get, 2}, {put, 2}, {delete, 1}]}.
|
||||
|
||||
{code_paths, ["/Users/uwiger/git/rocksdb",
|
||||
"/Users/uwiger/git/mnesia_rocksdb"]}.
|
||||
|
||||
{mnesia_table, rdb}.
|
||||
{backend, rocksdb_copies}.
|
||||
@@ -22,6 +22,7 @@
|
||||
|
||||
-record(t, {k, i, v}).
|
||||
|
||||
-dialyzer({nowarn_function, run/1}).
|
||||
run(Sz) ->
|
||||
mnesia:stop(),
|
||||
init(),
|
||||
|
||||
@@ -79,20 +79,20 @@ cleanup() ->
|
||||
os:cmd("rm *.BUP").
|
||||
|
||||
mods(0) ->
|
||||
[];
|
||||
mods(1) ->
|
||||
[
|
||||
{l, mnesia_rocksdb},
|
||||
{g, rocksdb}
|
||||
];
|
||||
mods(2) ->
|
||||
[
|
||||
%% {l, mnesia_monitor},
|
||||
{g, mnesia_rocksdb},
|
||||
{l, mnesia_bup},
|
||||
{g, mnesia_lib},
|
||||
{g, mnesia_schema},
|
||||
%% {g, mnesia_loader},
|
||||
{g, mnesia_index},
|
||||
{l, mnesia_tm}
|
||||
].
|
||||
[].
|
||||
%% mods(1) ->
|
||||
%% [
|
||||
%% {l, mnesia_rocksdb},
|
||||
%% {g, rocksdb}
|
||||
%% ];
|
||||
%% mods(2) ->
|
||||
%% [
|
||||
%% %% {l, mnesia_monitor},
|
||||
%% {g, mnesia_rocksdb},
|
||||
%% {l, mnesia_bup},
|
||||
%% {g, mnesia_lib},
|
||||
%% {g, mnesia_schema},
|
||||
%% %% {g, mnesia_loader},
|
||||
%% {g, mnesia_index},
|
||||
%% {l, mnesia_tm}
|
||||
%% ].
|
||||
|
||||
@@ -152,8 +152,8 @@ end_per_testcase(_, _) ->
|
||||
%% ======================================================================
|
||||
|
||||
cfg([Tab, Type, IxType], Config) ->
|
||||
[{my_config, #{tab => Tab, type => Type, ixtype => IxType}} | Config];
|
||||
cfg(Cfg, Config) when is_map(Cfg) -> [{my_config, Cfg} | Config].
|
||||
[{my_config, #{tab => Tab, type => Type, ixtype => IxType}} | Config].
|
||||
%% cfg(Cfg, Config) when is_map(Cfg) -> [{my_config, Cfg} | Config].
|
||||
|
||||
cfg(Config) -> ?config(my_config, Config).
|
||||
|
||||
@@ -236,12 +236,25 @@ test_index_plugin(Config) ->
|
||||
if Type == rdb ->
|
||||
Res1 = lists:sort(mrdb:index_read(Tab,<<"sen">>, {pfx})),
|
||||
Res2 = lists:sort(mrdb:index_read(Tab,<<"whi">>, {pfx})),
|
||||
ok = test_select(Tab,{pfx},[{'_', [], ['$_']}]),
|
||||
MS2 = [{{<<"whi">>,{Tab,"truth",'_'}},[],['$_']}],
|
||||
[_] = mrdb_index:select(Tab, {pfx}, MS2),
|
||||
ok = test_select(Tab, {pfx}, MS2),
|
||||
[{Tab,"foobar","sentence"}] = mrdb:index_read(
|
||||
Tab, <<"foo">>, {pfx});
|
||||
true ->
|
||||
ok
|
||||
end.
|
||||
|
||||
test_select(Tab, Ix, MS) ->
|
||||
Res = mrdb_index:select(Tab, Ix, MS),
|
||||
ct:log("mrdb_index:select(~p, ~p, ~p) -> ~p", [Tab, Ix, MS, Res]),
|
||||
RevRes = mrdb_index:select_reverse(Tab, Ix, MS),
|
||||
ct:log("mrdb_index:select_reverse(~p, ~p, ~p) -> ~p", [Tab, Ix, MS, RevRes]),
|
||||
{Res,Res} = {Res, lists:reverse(RevRes)},
|
||||
ok.
|
||||
|
||||
|
||||
ixtype(T) when T==bag;
|
||||
T==ordered ->
|
||||
{{pfx}, T};
|
||||
|
||||
@@ -48,14 +48,18 @@
|
||||
-record(st, {}).
|
||||
-define(KEYS, [a,b,c]).
|
||||
|
||||
-dialyzer({no_return, [basic_test_/0, test/1, prop_seq/0]}).
|
||||
|
||||
basic_test_() ->
|
||||
{timeout, 60000, [fun() -> test(100) end]}.
|
||||
|
||||
-dialyzer({no_opaque, test/1}).
|
||||
test(N) ->
|
||||
setup_mnesia(),
|
||||
true = proper:quickcheck(?MODULE:prop_seq(), N),
|
||||
ok.
|
||||
|
||||
-dialyzer({no_opaque, prop_seq/0}).
|
||||
prop_seq() ->
|
||||
?FORALL(Cmds, proper_statem:commands(?MODULE),
|
||||
begin
|
||||
|
||||
@@ -260,10 +260,10 @@ plain_transform1(Fun, [F|Fs]) when is_atom(element(1,F)) ->
|
||||
continue ->
|
||||
[list_to_tuple(plain_transform1(Fun, tuple_to_list(F))) |
|
||||
plain_transform1(Fun, Fs)];
|
||||
{done, NewF} ->
|
||||
[NewF | Fs];
|
||||
{error, Reason} ->
|
||||
io:format("Error: ~p (~p)~n", [F,Reason]);
|
||||
%% {done, NewF} ->
|
||||
%% [NewF | Fs];
|
||||
%% {error, Reason} ->
|
||||
%% io:format("Error: ~p (~p)~n", [F,Reason]);
|
||||
NewF when is_tuple(NewF) ->
|
||||
[NewF | plain_transform1(Fun, Fs)]
|
||||
end;
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
-module(mrdb_fold_SUITE).
|
||||
|
||||
-export([
|
||||
all/0
|
||||
, groups/0
|
||||
, suite/0
|
||||
, init_per_suite/1
|
||||
, end_per_suite/1
|
||||
, init_per_group/2
|
||||
, end_per_group/2
|
||||
, init_per_testcase/2
|
||||
, end_per_testcase/2
|
||||
]).
|
||||
|
||||
-export([
|
||||
no_prefix_fwd_rdb_fold/1
|
||||
, prefixed_fwd_rdb_fold/1
|
||||
, no_prefix_rev_rdb_fold/1
|
||||
, prefixed_rev_rdb_fold/1
|
||||
, prefixed_rev_rdb_fold_max/1
|
||||
, fwd_fold/1
|
||||
, filtered_fwd_fold/1
|
||||
, rev_fold/1
|
||||
, filtered_rev_fold/1
|
||||
, fwd_select/1
|
||||
, rev_select/1
|
||||
, select_cont/1
|
||||
, rev_select_cont/1
|
||||
]).
|
||||
|
||||
-include_lib("common_test/include/ct.hrl").
|
||||
|
||||
-record(r, {k, v}).
|
||||
|
||||
suite() ->
|
||||
[].
|
||||
|
||||
all() ->
|
||||
[{group, all_tests}].
|
||||
|
||||
groups() ->
|
||||
[
|
||||
{all_tests, [sequence], [ {group, rdb_fold}
|
||||
, {group, fold}
|
||||
, {group, select} ]}
|
||||
, {rdb_fold, [sequence], [ no_prefix_fwd_rdb_fold
|
||||
, prefixed_fwd_rdb_fold
|
||||
, no_prefix_rev_rdb_fold
|
||||
, prefixed_rev_rdb_fold
|
||||
, prefixed_rev_rdb_fold_max ]}
|
||||
, {fold, [sequence], [ fwd_fold
|
||||
, filtered_fwd_fold
|
||||
, rev_fold
|
||||
, filtered_rev_fold ]}
|
||||
, {select, [sequence], [ fwd_select
|
||||
, rev_select
|
||||
, select_cont
|
||||
, rev_select_cont ]}
|
||||
].
|
||||
|
||||
init_per_suite(Config) ->
|
||||
mnesia:stop(),
|
||||
ok = mnesia_rocksdb_tlib:start_mnesia(reset),
|
||||
Config.
|
||||
|
||||
end_per_suite(_Config) ->
|
||||
mnesia:stop(),
|
||||
ok.
|
||||
|
||||
init_per_group(G, Config) when G==rdb_fold; G==fold ->
|
||||
mk_tab(G),
|
||||
Config;
|
||||
init_per_group(_, Config) ->
|
||||
Config.
|
||||
|
||||
end_per_group(_, _Config) ->
|
||||
ok.
|
||||
|
||||
init_per_testcase(_, Config) ->
|
||||
Config.
|
||||
|
||||
end_per_testcase(_, _Config) ->
|
||||
ok.
|
||||
|
||||
mk_tab(G) ->
|
||||
T = tab_name(G),
|
||||
case create_tab(T) of
|
||||
{atomic, ok} -> fill_tab(T);
|
||||
{aborted,{already_exists,T}} -> ok
|
||||
end.
|
||||
|
||||
tab_name(fold ) -> r;
|
||||
%% tab_name(select ) -> r;
|
||||
tab_name(rdb_fold) -> t.
|
||||
|
||||
create_tab(T) ->
|
||||
Opts = tab_opts(T),
|
||||
mnesia:create_table(T, [{rdb, [node()]} | Opts]).
|
||||
|
||||
tab_opts(r) ->
|
||||
[{attributes, [k, v]}, {type, ordered_set}];
|
||||
tab_opts(t) ->
|
||||
[{attributes, [k, v]}, {user_properties, [{mrdb_encoding, {raw, raw}}]}].
|
||||
|
||||
no_prefix_fwd_rdb_fold(_Config) ->
|
||||
Res = mrdb:rdb_fold(t, fun simple_rdb_acc/3, [], <<>>, infinity),
|
||||
Expected = lists:reverse(raw_objs()),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
fwd_fold(_Config) ->
|
||||
Res = mrdb:fold(r, fun simple_acc/2, []),
|
||||
Expected = lists:reverse(objs()),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
prefixed_fwd_rdb_fold(_Config) ->
|
||||
Pfx = <<"aa">>,
|
||||
Res = mrdb:rdb_fold(t, fun simple_rdb_acc/3, [], Pfx, infinity),
|
||||
Expected = lists:reverse(prefixed_raw_objs(Pfx)),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
filtered_fwd_fold(_Config) ->
|
||||
MS = [{#r{k = {a,b,'_'}, v = '_'}, [], ['$_']}],
|
||||
Res = mrdb:fold(r, fun simple_acc/2, [], MS, infinity),
|
||||
Expected = lists:reverse(filtered_objs(MS)),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
no_prefix_rev_rdb_fold(_Config) ->
|
||||
Res = mrdb:rdb_fold_reverse(t, fun simple_rdb_acc/3, [], <<>>, infinity),
|
||||
Expected = raw_objs(),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
rev_fold(_Config) ->
|
||||
Res = mrdb:fold_reverse(r, fun simple_acc/2, []),
|
||||
Expected = objs(),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
prefixed_rev_rdb_fold(_Config) ->
|
||||
Pfx = <<"aa">>,
|
||||
Res = mrdb:rdb_fold_reverse(t, fun simple_rdb_acc/3, [], Pfx, infinity),
|
||||
Expected = prefixed_raw_objs(Pfx),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
filtered_rev_fold(_Config) ->
|
||||
MS = [{#r{k = {a,b,'_'}, v = '_'}, [], ['$_']}],
|
||||
Res = mrdb:fold_reverse(r, fun simple_acc/2, [], MS, infinity),
|
||||
Expected = filtered_objs(MS),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
prefixed_rev_rdb_fold_max(_Config) ->
|
||||
Pfx = <<255,255>>,
|
||||
Res = mrdb:rdb_fold_reverse(t, fun simple_rdb_acc/3, [], Pfx, infinity),
|
||||
Expected = prefixed_raw_objs(Pfx),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
fwd_select(_Config) ->
|
||||
MS = [{#r{k = {a,b,'_'}, v = '_'}, [], ['$_']}],
|
||||
Res = mrdb:select(r, MS),
|
||||
Expected = filtered_objs(MS),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
rev_select(_Config) ->
|
||||
MS = [{#r{k = {a,b,'_'}, v = '_'}, [], ['$_']}],
|
||||
Res = mrdb:select_reverse(r, MS),
|
||||
Expected = lists:reverse(filtered_objs(MS)),
|
||||
{Res, Res} = {Res, Expected}.
|
||||
|
||||
select_cont(_Cont) ->
|
||||
MS = [{#r{k = {a,b,'_'}, v = '_'}, [], ['$_']}],
|
||||
Expected = filtered_objs(MS),
|
||||
select_cont_loop(mrdb:select(r, MS, 1), Expected).
|
||||
|
||||
rev_select_cont(_Cont) ->
|
||||
MS = [{#r{k = {a,b,'_'}, v = '_'}, [], ['$_']}],
|
||||
Expected = lists:reverse(filtered_objs(MS)),
|
||||
select_cont_loop(mrdb:select_reverse(r, MS, 1), Expected).
|
||||
|
||||
select_cont_loop({[Obj], Cont}, [Obj|Rest]) ->
|
||||
select_cont_loop(mrdb:select(Cont), Rest);
|
||||
select_cont_loop({[], '$end_of_table'}, []) ->
|
||||
ok.
|
||||
|
||||
simple_acc(#r{} = Obj, Acc) ->
|
||||
[Obj | Acc].
|
||||
|
||||
simple_rdb_acc(K, V, Acc) ->
|
||||
[{K,V} | Acc].
|
||||
|
||||
fill_tab(t = Tab) ->
|
||||
[mrdb:insert(Tab, {Tab, K, V}) || {K,V} <- raw_objs()],
|
||||
ok;
|
||||
fill_tab(r = Tab) ->
|
||||
[mrdb:insert(Tab, Obj) || Obj <- objs()],
|
||||
ok.
|
||||
|
||||
prefixed_raw_objs(Pfx) ->
|
||||
[Obj || {K,_} = Obj <- raw_objs(),
|
||||
is_prefix(Pfx, K)].
|
||||
|
||||
raw_objs() ->
|
||||
[ {<<"aa">> , <<"1">>}
|
||||
, {<<"aa1">>, <<"2">>}
|
||||
, {<<"ab">> , <<"3">>}
|
||||
, {<<"ab1">>, <<"4">>}
|
||||
, {<<255,255>>, <<"5">>} ].
|
||||
|
||||
filtered_objs(MS) ->
|
||||
MSC = ets:match_spec_compile(MS),
|
||||
ets:match_spec_run(objs(), MSC).
|
||||
|
||||
objs() ->
|
||||
[ #r{k = {a,a,1}, v = 1}
|
||||
, #r{k = {a,b,2}, v = 2}
|
||||
, #r{k = {a,b,3}, v = 3}
|
||||
, #r{k = {a,c,4}, v = 4}
|
||||
, #r{k = {b,b,5}, v = 5}
|
||||
].
|
||||
|
||||
%% copied from mrdb_select.erl
|
||||
is_prefix(A, B) when is_binary(A), is_binary(B) ->
|
||||
Sa = byte_size(A),
|
||||
case B of
|
||||
<<A:Sa/binary, _/binary>> ->
|
||||
true;
|
||||
_ ->
|
||||
false
|
||||
end.
|
||||
Reference in New Issue
Block a user