From b55c3726f4a21063721c68d6fa7fda39121edf11 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Mon, 25 Feb 2019 16:46:33 +0100 Subject: [PATCH] New application aeserialization for shared deps with compiler --- .gitignore | 19 ++ LICENSE | 15 ++ README.md | 9 + rebar.config | 3 + src/aeser_api_encoder.erl | 280 +++++++++++++++++++++++++++++ src/aeser_chain_objects.erl | 143 +++++++++++++++ src/aeser_id.erl | 111 ++++++++++++ src/aeser_rlp.erl | 91 ++++++++++ src/aeserialization.app.src | 15 ++ src/aeserialization.erl | 134 ++++++++++++++ test/aeser_api_encoder_tests.erl | 138 ++++++++++++++ test/aeser_chain_objects_tests.erl | 87 +++++++++ test/aeser_rlp_tests.erl | 132 ++++++++++++++ 13 files changed, 1177 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 rebar.config create mode 100644 src/aeser_api_encoder.erl create mode 100644 src/aeser_chain_objects.erl create mode 100644 src/aeser_id.erl create mode 100644 src/aeser_rlp.erl create mode 100644 src/aeserialization.app.src create mode 100644 src/aeserialization.erl create mode 100644 test/aeser_api_encoder_tests.erl create mode 100644 test/aeser_chain_objects_tests.erl create mode 100644 test/aeser_rlp_tests.erl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f1c4554 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..142825a --- /dev/null +++ b/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2017, aeternity developers + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ca9ace2 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +aeserialization +===== + +Serialization helpers for Aeternity node. + +Build +----- + + $ rebar3 compile diff --git a/rebar.config b/rebar.config new file mode 100644 index 0000000..d43c4df --- /dev/null +++ b/rebar.config @@ -0,0 +1,3 @@ +{erl_opts, [debug_info]}. +{deps, [{base58, {git, "https://github.com/aeternity/erl-base58.git", {ref,"60a3356"}}} + ]}. diff --git a/src/aeser_api_encoder.erl b/src/aeser_api_encoder.erl new file mode 100644 index 0000000..32a253c --- /dev/null +++ b/src/aeser_api_encoder.erl @@ -0,0 +1,280 @@ +%%%------------------------------------------------------------------- +%%% @copyright (C) 2017, Aeternity Anstalt +%%% @doc +%%% API encoding for the Aeternity node. +%%% @end +%%%------------------------------------------------------------------- +-module(aeser_api_encoder). + +-export([encode/2, + decode/1, + safe_decode/2, + byte_size_for_type/1]). + +-export_type([encoded/0]). + +-type known_type() :: key_block_hash + | micro_block_hash + | block_pof_hash + | block_tx_hash + | block_state_hash + | channel + | contract_pubkey + | transaction + | tx_hash + | oracle_pubkey + | oracle_query + | oracle_query_id + | oracle_response + | account_pubkey + | signature + | name + | commitment + | peer_pubkey + | state + | poi + | state_trees. + +-type extended_type() :: known_type() | block_hash | {id_hash, [known_type()]}. + + +-type payload() :: binary(). +-type encoded() :: binary(). + +-define(BASE58, 1). +-define(BASE64, 2). + +-spec encode(known_type(), payload() | aeser_id:id()) -> encoded(). +encode(id_hash, Payload) -> + {IdType, Val} = aeser_id:specialize(Payload), + encode(id2type(IdType), Val); +encode(Type, Payload) -> + Pfx = type2pfx(Type), + Enc = case type2enc(Type) of + ?BASE58 -> base58_check(Payload); + ?BASE64 -> base64_check(Payload) + end, + <>. + +-spec decode(binary()) -> {known_type(), payload()}. +decode(Bin0) -> + case split(Bin0) of + [Pfx, Payload] -> + Type = pfx2type(Pfx), + Bin = decode_check(Type, Payload), + case type_size_check(Type, Bin) of + ok -> {Type, Bin}; + {error, Reason} -> erlang:error(Reason) + end; + _ -> + %% {<<>>, decode_check(Bin)} + erlang:error(missing_prefix) + end. + +type_size_check(Type, Bin) -> + case byte_size_for_type(Type) of + not_applicable -> ok; + CorrectSize -> + Size = byte_size(Bin), + case Size =:= CorrectSize of + true -> ok; + false -> {error, incorrect_size} + end + end. + +-spec safe_decode(extended_type(), encoded()) -> {'ok', payload() | aeser_id:id()} + | {'error', atom()}. +safe_decode({id_hash, AllowedTypes}, Enc) -> + try decode(Enc) of + {ActualType, Dec} -> + case lists:member(ActualType, AllowedTypes) of + true -> + try {ok, aeser_id:create(type2id(ActualType), Dec)} + catch error:_ -> {error, invalid_prefix} + end; + false -> + {error, invalid_prefix} + end + catch + error:_ -> + {error, invalid_encoding} + end; +safe_decode(block_hash, Enc) -> + try decode(Enc) of + {key_block_hash, Dec} -> + {ok, Dec}; + {micro_block_hash, Dec} -> + {ok, Dec}; + {_, _} -> + {error, invalid_prefix} + catch + error:_ -> + {error, invalid_encoding} + end; +safe_decode(Type, Enc) -> + try decode(Enc) of + {Type, Dec} -> + {ok, Dec}; + {_, _} -> + {error, invalid_prefix} + catch + error:_ -> + {error, invalid_encoding} + end. + +decode_check(Type, Bin) -> + Dec = + case type2enc(Type) of + ?BASE58 -> base58_to_binary(Bin); + ?BASE64 -> base64_to_binary(Bin) + end, + Sz = byte_size(Dec), + BSz = Sz - 4, + <> = Dec, + C = check_str(Body), + Body. + +base64_check(Bin) -> + C = check_str(Bin), + binary_to_base64(iolist_to_binary([Bin, C])). + +%% modified from github.com/mbrix/lib_hd +base58_check(Bin) -> + C = check_str(Bin), + binary_to_base58(iolist_to_binary([Bin, C])). + +split(Bin) -> + binary:split(Bin, [<<"_">>], []). + +check_str(Bin) -> + <> = + sha256_hash(sha256_hash(Bin)), + C. + +sha256_hash(Bin) -> + crypto:hash(sha256, Bin). + + +id2type(account) -> account_pubkey; +id2type(channel) -> channel; +id2type(commitment) -> commitment; +id2type(contract) -> contract_pubkey; +id2type(name) -> name; +id2type(oracle) -> oracle_pubkey. + +type2id(account_pubkey) -> account; +type2id(channel) -> channel; +type2id(commitment) -> commitment; +type2id(contract_pubkey) -> contract; +type2id(name) -> name; +type2id(oracle_pubkey) -> oracle. + +type2enc(key_block_hash) -> ?BASE58; +type2enc(micro_block_hash) -> ?BASE58; +type2enc(block_pof_hash) -> ?BASE58; +type2enc(block_tx_hash) -> ?BASE58; +type2enc(block_state_hash) -> ?BASE58; +type2enc(channel) -> ?BASE58; +type2enc(contract_pubkey) -> ?BASE58; +type2enc(contract_bytearray)-> ?BASE64; +type2enc(transaction) -> ?BASE64; +type2enc(tx_hash) -> ?BASE58; +type2enc(oracle_pubkey) -> ?BASE58; +type2enc(oracle_query) -> ?BASE64; +type2enc(oracle_query_id) -> ?BASE58; +type2enc(oracle_response) -> ?BASE64; +type2enc(account_pubkey) -> ?BASE58; +type2enc(signature) -> ?BASE58; +type2enc(commitment) -> ?BASE58; +type2enc(peer_pubkey) -> ?BASE58; +type2enc(name) -> ?BASE58; +type2enc(state) -> ?BASE64; +type2enc(poi) -> ?BASE64; +type2enc(state_trees) -> ?BASE64. + + +type2pfx(key_block_hash) -> <<"kh">>; +type2pfx(micro_block_hash) -> <<"mh">>; +type2pfx(block_pof_hash) -> <<"bf">>; +type2pfx(block_tx_hash) -> <<"bx">>; +type2pfx(block_state_hash) -> <<"bs">>; +type2pfx(channel) -> <<"ch">>; +type2pfx(contract_pubkey) -> <<"ct">>; +type2pfx(contract_bytearray)-> <<"cb">>; +type2pfx(transaction) -> <<"tx">>; +type2pfx(tx_hash) -> <<"th">>; +type2pfx(oracle_pubkey) -> <<"ok">>; +type2pfx(oracle_query) -> <<"ov">>; +type2pfx(oracle_query_id) -> <<"oq">>; +type2pfx(oracle_response) -> <<"or">>; +type2pfx(account_pubkey) -> <<"ak">>; +type2pfx(signature) -> <<"sg">>; +type2pfx(commitment) -> <<"cm">>; +type2pfx(peer_pubkey) -> <<"pp">>; +type2pfx(name) -> <<"nm">>; +type2pfx(state) -> <<"st">>; +type2pfx(poi) -> <<"pi">>; +type2pfx(state_trees) -> <<"ss">>. + +pfx2type(<<"kh">>) -> key_block_hash; +pfx2type(<<"mh">>) -> micro_block_hash; +pfx2type(<<"bf">>) -> block_pof_hash; +pfx2type(<<"bx">>) -> block_tx_hash; +pfx2type(<<"bs">>) -> block_state_hash; +pfx2type(<<"ch">>) -> channel; +pfx2type(<<"cb">>) -> contract_bytearray; +pfx2type(<<"ct">>) -> contract_pubkey; +pfx2type(<<"tx">>) -> transaction; +pfx2type(<<"th">>) -> tx_hash; +pfx2type(<<"ok">>) -> oracle_pubkey; +pfx2type(<<"ov">>) -> oracle_query; +pfx2type(<<"oq">>) -> oracle_query_id; +pfx2type(<<"or">>) -> oracle_response; +pfx2type(<<"ak">>) -> account_pubkey; +pfx2type(<<"sg">>) -> signature; +pfx2type(<<"cm">>) -> commitment; +pfx2type(<<"pp">>) -> peer_pubkey; +pfx2type(<<"nm">>) -> name; +pfx2type(<<"st">>) -> state; +pfx2type(<<"pi">>) -> poi; +pfx2type(<<"ss">>) -> state_trees. + +-spec byte_size_for_type(known_type()) -> non_neg_integer() | not_applicable. + +byte_size_for_type(key_block_hash) -> 32; +byte_size_for_type(micro_block_hash) -> 32; +byte_size_for_type(block_pof_hash) -> 32; +byte_size_for_type(block_tx_hash) -> 32; +byte_size_for_type(block_state_hash) -> 32; +byte_size_for_type(channel) -> 32; +byte_size_for_type(contract_pubkey) -> 32; +byte_size_for_type(contract_bytearray)-> not_applicable; +byte_size_for_type(transaction) -> not_applicable; +byte_size_for_type(tx_hash) -> 32; +byte_size_for_type(oracle_pubkey) -> 32; +byte_size_for_type(oracle_query) -> not_applicable; +byte_size_for_type(oracle_query_id) -> 32; +byte_size_for_type(oracle_response) -> not_applicable; +byte_size_for_type(account_pubkey) -> 32; +byte_size_for_type(signature) -> 64; +byte_size_for_type(name) -> not_applicable; +byte_size_for_type(commitment) -> 32; +byte_size_for_type(peer_pubkey) -> 32; +byte_size_for_type(state) -> 32; +byte_size_for_type(poi) -> not_applicable; +byte_size_for_type(state_trees) -> not_applicable. + + +%% TODO: Fix the base58 module so that it consistently uses binaries instead +%% +binary_to_base58(Bin) -> + iolist_to_binary(base58:binary_to_base58(Bin)). + +base58_to_binary(Bin) when is_binary(Bin) -> + base58:base58_to_binary(binary_to_list(Bin)). + +binary_to_base64(Bin) -> + base64:encode(Bin). + +base64_to_binary(Bin) when is_binary(Bin) -> + base64:decode(Bin). diff --git a/src/aeser_chain_objects.erl b/src/aeser_chain_objects.erl new file mode 100644 index 0000000..6ae1dbd --- /dev/null +++ b/src/aeser_chain_objects.erl @@ -0,0 +1,143 @@ +%%% -*- erlang-indent-level:4; indent-tabs-mode: nil -*- +%%%------------------------------------------------------------------- +%%% @copyright (C) 2018, Aeternity Anstalt +%%% @doc +%%% Functions for serializing chain objects to binary format. +%%% @end +%%%------------------------------------------------------------------- + +-module(aeser_chain_objects). + +-export([ serialize/4 + , deserialize/4 + , deserialize_type_and_vsn/1 + ]). + +-type template() :: aeserialization:template(). +-type fields() :: aeserialization:fields(). + +%%%=================================================================== +%%% API +%%%=================================================================== + +-spec serialize(atom(), non_neg_integer(), template(), fields()) -> binary(). +serialize(Type, Vsn, Template, Fields) -> + aeserialization:serialize(tag(Type), Vsn, Template, Fields). + +deserialize_type_and_vsn(Binary) -> + {Tag, Vsn, Fields} = aeserialization:deserialize_tag_and_vsn(Binary), + {rev_tag(Tag), Vsn, Fields}. + +-spec deserialize(atom(), non_neg_integer(), template(), binary()) -> fields(). +deserialize(Type, Vsn, Template, Binary) -> + aeserialization:deserialize(Type, tag(Type), Vsn, Template, Binary). + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +tag(account) -> 10; +tag(signed_tx) -> 11; +tag(spend_tx) -> 12; +tag(oracle) -> 20; +tag(oracle_query) -> 21; +tag(oracle_register_tx) -> 22; +tag(oracle_query_tx) -> 23; +tag(oracle_response_tx) -> 24; +tag(oracle_extend_tx) -> 25; +tag(name) -> 30; +tag(name_commitment) -> 31; +tag(name_claim_tx) -> 32; +tag(name_preclaim_tx) -> 33; +tag(name_update_tx) -> 34; +tag(name_revoke_tx) -> 35; +tag(name_transfer_tx) -> 36; +tag(contract) -> 40; +tag(contract_call) -> 41; +tag(contract_create_tx) -> 42; +tag(contract_call_tx) -> 43; +tag(channel_create_tx) -> 50; +tag(channel_deposit_tx) -> 51; +tag(channel_withdraw_tx) -> 52; +tag(channel_force_progress_tx) -> 521; +tag(channel_close_mutual_tx) -> 53; +tag(channel_close_solo_tx) -> 54; +tag(channel_slash_tx) -> 55; +tag(channel_settle_tx) -> 56; +tag(channel_offchain_tx) -> 57; +tag(channel_offchain_update_transfer) -> 570; +tag(channel_offchain_update_deposit) -> 571; +tag(channel_offchain_update_withdraw) -> 572; +tag(channel_offchain_update_create_contract) -> 573; +tag(channel_offchain_update_call_contract) -> 574; +tag(channel) -> 58; +tag(channel_snapshot_solo_tx) -> 59; +tag(trees_poi) -> 60; +tag(trees_db) -> 61; +tag(state_trees) -> 62; +tag(mtree) -> 63; +tag(mtree_value) -> 64; +tag(contracts_mtree) -> 621; +tag(calls_mtree) -> 622; +tag(channels_mtree) -> 623; +tag(nameservice_mtree) -> 624; +tag(oracles_mtree) -> 625; +tag(accounts_mtree) -> 626; +tag(compiler_sophia) -> 70; +tag(key_block) -> 100; +tag(micro_block) -> 101; +tag(light_micro_block) -> 102; +tag(pof) -> 200. + +rev_tag(10) -> account; +rev_tag(11) -> signed_tx; +rev_tag(12) -> spend_tx; +rev_tag(20) -> oracle; +rev_tag(21) -> oracle_query; +rev_tag(22) -> oracle_register_tx; +rev_tag(23) -> oracle_query_tx; +rev_tag(24) -> oracle_response_tx; +rev_tag(25) -> oracle_extend_tx; +rev_tag(30) -> name; +rev_tag(31) -> name_commitment; +rev_tag(32) -> name_claim_tx; +rev_tag(33) -> name_preclaim_tx; +rev_tag(34) -> name_update_tx; +rev_tag(35) -> name_revoke_tx; +rev_tag(36) -> name_transfer_tx; +rev_tag(40) -> contract; +rev_tag(41) -> contract_call; +rev_tag(42) -> contract_create_tx; +rev_tag(43) -> contract_call_tx; +rev_tag(50) -> channel_create_tx; +rev_tag(51) -> channel_deposit_tx; +rev_tag(52) -> channel_withdraw_tx; +rev_tag(521) -> channel_force_progress_tx; +rev_tag(53) -> channel_close_mutual_tx; +rev_tag(54) -> channel_close_solo_tx; +rev_tag(55) -> channel_slash_tx; +rev_tag(56) -> channel_settle_tx; +rev_tag(57) -> channel_offchain_tx; +rev_tag(570) -> channel_offchain_update_transfer; +rev_tag(571) -> channel_offchain_update_deposit; +rev_tag(572) -> channel_offchain_update_withdraw; +rev_tag(573) -> channel_offchain_update_create_contract; +rev_tag(574) -> channel_offchain_update_call_contract; +rev_tag(58) -> channel; +rev_tag(59) -> channel_snapshot_solo_tx; +rev_tag(60) -> trees_poi; +rev_tag(61) -> trees_db; +rev_tag(62) -> state_trees; +rev_tag(63) -> mtree; +rev_tag(64) -> mtree_value; +rev_tag(621) -> contracts_mtree; +rev_tag(622) -> calls_mtree; +rev_tag(623) -> channels_mtree; +rev_tag(624) -> nameservice_mtree; +rev_tag(625) -> oracles_mtree; +rev_tag(626) -> accounts_mtree; +rev_tag(70) -> compiler_sophia; +rev_tag(100) -> key_block; +rev_tag(101) -> micro_block; +rev_tag(102) -> light_micro_block; +rev_tag(200) -> pof. diff --git a/src/aeser_id.erl b/src/aeser_id.erl new file mode 100644 index 0000000..229cfda --- /dev/null +++ b/src/aeser_id.erl @@ -0,0 +1,111 @@ +%%% -*- erlang-indent-level:4; indent-tabs-mode: nil -*- +%%%------------------------------------------------------------------- +%%% @copyright (C) 2018, Aeternity Anstalt +%%% @doc +%%% ADT for identifiers +%%% @end +%%%------------------------------------------------------------------- + +-module(aeser_id). + +-export([ create/2 + , specialize/1 + , specialize/2 + , specialize_type/1 + , is_id/1 + ]). + +%% For aec_serialization +-export([ encode/1 + , decode/1 + ]). + +-record(id, { tag + , val + }). + +-type tag() :: 'account' | 'oracle' | 'name' + | 'commitment' | 'contract' | 'channel'. +-type val() :: <<_:256>>. +-opaque(id() :: #id{}). + +-export_type([ id/0 + , tag/0 + , val/0 + ]). + +-define(PUB_SIZE, 32). +-define(TAG_SIZE, 1). +-define(SERIALIZED_SIZE, 33). %% ?TAG_SIZE + ?PUB_SIZE + +-define(IS_TAG(___TAG___), ___TAG___ =:= account; + ___TAG___ =:= oracle; + ___TAG___ =:= name; + ___TAG___ =:= commitment; + ___TAG___ =:= contract; + ___TAG___ =:= channel + ). +-define(IS_VAL(___VAL___), byte_size(___VAL___) =:= 32). + +%%%=================================================================== +%%% API +%%%=================================================================== + +-spec create(tag(), val()) -> id(). +create(Tag, Val) when ?IS_TAG(Tag), ?IS_VAL(Val) -> + #id{ tag = Tag + , val = Val}; +create(Tag, Val) when ?IS_VAL(Val) -> + error({illegal_tag, Tag}); +create(Tag, Val) when ?IS_TAG(Tag)-> + error({illegal_val, Val}); +create(Tag, Val) -> + error({illegal_tag_and_val, Tag, Val}). + + +-spec specialize(id()) -> {tag(), val()}. +specialize(#id{tag = Tag, val = Val}) -> + {Tag, Val}. + +-spec specialize(id(), tag()) -> val(). +specialize(#id{tag = Tag, val = Val}, Tag) when ?IS_TAG(Tag), ?IS_VAL(Val) -> + Val. + +-spec specialize_type(id()) -> tag(). +specialize_type(#id{tag = Tag}) when ?IS_TAG(Tag) -> + Tag. + +-spec is_id(term()) -> boolean(). +is_id(#id{}) -> true; +is_id(_) -> false. + +-spec encode(id()) -> binary(). +encode(#id{tag = Tag, val = Val}) -> + Res = <<(encode_tag(Tag)):?TAG_SIZE/unit:8, Val/binary>>, + true = ?SERIALIZED_SIZE =:= byte_size(Res), + Res. + +-spec decode(binary()) -> id(). +decode(<>) -> + #id{ tag = decode_tag(Tag) + , val = Val}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +encode_tag(account) -> 1; +encode_tag(name) -> 2; +encode_tag(commitment) -> 3; +encode_tag(oracle) -> 4; +encode_tag(contract) -> 5; +encode_tag(channel) -> 6; +encode_tag(Other) -> error({illegal_id_tag_name, Other}). + +decode_tag(1) -> account; +decode_tag(2) -> name; +decode_tag(3) -> commitment; +decode_tag(4) -> oracle; +decode_tag(5) -> contract; +decode_tag(6) -> channel; +decode_tag(X) -> error({illegal_id_tag, X}). diff --git a/src/aeser_rlp.erl b/src/aeser_rlp.erl new file mode 100644 index 0000000..bb4eb8c --- /dev/null +++ b/src/aeser_rlp.erl @@ -0,0 +1,91 @@ +%%%------------------------------------------------------------------- +%%% @copyright (C) 2017, Aeternity Anstalt +%%% @doc +%%% Implementation of the Recursive Length Prefix. +%%% +%%% https://github.com/ethereum/wiki/wiki/RLP +%%% +%%% @end +%%%------------------------------------------------------------------- + +-module(aeser_rlp). +-export([ decode/1 + , decode_one/1 + , encode/1 + ]). + +-export_type([ encodable/0 + , encoded/0 + ]). + +-type encodable() :: [encodable()] | binary(). +-type encoded() :: <<_:8, _:_*8>>. + +-define(UNTAGGED_SIZE_LIMIT , 55). +-define(UNTAGGED_LIMIT , 127). +-define(BYTE_ARRAY_OFFSET , 128). +-define(LIST_OFFSET , 192). + + +-spec encode(encodable()) -> encoded(). +encode(X) -> + encode(X, []). + +encode(<> = X,_Opts) when B =< ?UNTAGGED_LIMIT -> + %% An untagged value + X; +encode(X,_Opts) when is_binary(X) -> + %% Byte array + add_size(?BYTE_ARRAY_OFFSET, X); +encode(L, Opts) when is_list(L) -> + %% Lists items are encoded and concatenated + ByteArray = << << (encode(X, Opts))/binary >> || X <- L >>, + add_size(?LIST_OFFSET, ByteArray). + +add_size(Offset, X) when byte_size(X) =< ?UNTAGGED_SIZE_LIMIT -> + %% The size fits in one tagged byte + <<(Offset + byte_size(X)), X/binary>>; +add_size(Offset, X) when is_binary(X) -> + %% The size itself needs to be encoded as a byte array + %% Add the tagged size of the size byte array + SizeBin = binary:encode_unsigned(byte_size(X)), + TaggedSize = ?UNTAGGED_SIZE_LIMIT + Offset + byte_size(SizeBin), + true = (TaggedSize < 256 ), %% Assert + <>. + +-spec decode(encoded()) -> encodable(). +decode(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> + case decode_one(Bin) of + {X, <<>>} -> X; + {X, Left} -> error({trailing, X, Bin, Left}) + end. + +decode_one(<>) when X =< ?UNTAGGED_LIMIT -> + %% Untagged value + {<>, B}; +decode_one(<> = B) when L < ?LIST_OFFSET -> + %% Byte array + {Size, Rest} = decode_size(B, ?BYTE_ARRAY_OFFSET), + <> = Rest, + {X, Tail}; +decode_one(<<_/binary>> = B) -> + %% List + {Size, Rest} = decode_size(B, ?LIST_OFFSET), + <> = Rest, + {decode_list(X), Tail}. + +decode_size(<>, Offset) when L =< Offset + ?UNTAGGED_SIZE_LIMIT-> + %% One byte tagged size. + {L - Offset, B}; +decode_size(<<_, 0, _/binary>>,_Offset) -> + error(leading_zeroes_in_size); +decode_size(<>, Offset) -> + %% Actual size is in a byte array. + BinSize = L - Offset - ?UNTAGGED_SIZE_LIMIT, + <> = B, + {Size, Rest}. + +decode_list(<<>>) -> []; +decode_list(B) -> + {Element, Rest} = decode_one(B), + [Element|decode_list(Rest)]. diff --git a/src/aeserialization.app.src b/src/aeserialization.app.src new file mode 100644 index 0000000..dce80c9 --- /dev/null +++ b/src/aeserialization.app.src @@ -0,0 +1,15 @@ +{application, aeserialization, + [{description, "Serialization of data for Aeternity"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, + [kernel, + stdlib, + crypto, + base58 + ]}, + {env,[]}, + {modules, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/src/aeserialization.erl b/src/aeserialization.erl new file mode 100644 index 0000000..633c586 --- /dev/null +++ b/src/aeserialization.erl @@ -0,0 +1,134 @@ +%%%------------------------------------------------------------------- +%%% @copyright (C) 2018, Aeternity Anstalt +%%% @doc +%%% Functions for serializing generic objects to/from binary format. +%%% @end +%%%------------------------------------------------------------------- + +-module(aeserialization). + +-export([ decode_fields/2 + , deserialize/5 + , deserialize_tag_and_vsn/1 + , encode_fields/2 + , serialize/4 ]). + +%%%=================================================================== +%%% Types +%%%=================================================================== + +-export_type([ template/0 + , fields/0 + ]). + +-type template() :: [{field_name(), type()}]. +-type field_name() :: atom(). +-type type() :: 'int' + | 'bool' + | 'binary' + | 'id' %% As defined in aec_id.erl + | [type()] %% Length one in the type. This means a list of any length. + | tuple(). %% Any arity, containing type(). This means a static size array. + +-type encodable_term() :: non_neg_integer() + | binary() + | boolean() + | [encodable_term()] %% Of any length + | tuple() %% Any arity, containing encodable_term(). + | aeser_id:id(). + +-type fields() :: [{field_name(), encodable_term()}]. + +%%%=================================================================== +%%% API +%%%=================================================================== + +-spec serialize(non_neg_integer(), non_neg_integer(), template(), fields()) -> binary(). +serialize(Tag, Vsn, Template, Fields) -> + List = encode_fields([{tag, int}, {vsn, int}|Template], + [{tag, Tag}, {vsn, Vsn}|Fields]), + aeser_rlp:encode(List). + +%% Type isn't strictly necessary, but will give a better error reason +-spec deserialize(atom(), non_neg_integer(), non_neg_integer(), + template(), binary()) -> fields(). +deserialize(Type, Tag, Vsn, Template0, Binary) -> + Decoded = aeser_rlp:decode(Binary), + Template = [{tag, int}, {vsn, int}|Template0], + case decode_fields(Template, Decoded) of + [{tag, Tag}, {vsn, Vsn}|Left] -> + Left; + Other -> + error({illegal_serialization, Type, Vsn, + Other, Binary, Decoded, Template}) + end. + +-spec deserialize_tag_and_vsn(binary()) -> + {non_neg_integer(), non_neg_integer(), fields()}. +deserialize_tag_and_vsn(Binary) -> + [TagBin, VsnBin|Fields] = aeser_rlp:decode(Binary), + Template = [{tag, int}, {vsn, int}], + [{tag, Tag}, {vsn, Vsn}] = decode_fields(Template, [TagBin, VsnBin]), + {Tag, Vsn, Fields}. + +encode_fields([{Field, Type}|TypesLeft], + [{Field, Val}|FieldsLeft]) -> + try encode_field(Type, Val) of + Encoded -> [Encoded | encode_fields(TypesLeft, FieldsLeft)] + catch error:{illegal, T, V} -> + error({illegal_field, Field, Type, Val, T, V}) + end; +encode_fields([], []) -> + []; +encode_fields(Template, Values) -> + error({illegal_template_or_values, Template, Values}). + +decode_fields([{Field, Type}|TypesLeft], + [Bin |FieldsLeft]) -> + try decode_field(Type, Bin) of + Decoded -> [{Field, Decoded} | decode_fields(TypesLeft, FieldsLeft)] + catch error:{illegal, T, V} -> + error({illegal_field, Field, Type, Bin, T, V}) + end; +decode_fields([], []) -> + []; +decode_fields(Template, Values) -> + error({illegal_template_or_values, Template, Values}). + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +encode_field([Type], L) when is_list(L) -> + [encode_field(Type, X) || X <- L]; +encode_field(Type, T) when tuple_size(Type) =:= tuple_size(T) -> + Zipped = lists:zip(tuple_to_list(Type), tuple_to_list(T)), + [encode_field(X, Y) || {X, Y} <- Zipped]; +encode_field(int, X) when is_integer(X), X >= 0 -> + binary:encode_unsigned(X); +encode_field(binary, X) when is_binary(X) -> X; +encode_field(bool, true) -> <<1:8>>; +encode_field(bool, false) -> <<0:8>>; +encode_field(id, Val) -> + try aeser_id:encode(Val) + catch _:_ -> error({illegal, id, Val}) + end; +encode_field(Type, Val) -> error({illegal, Type, Val}). + +decode_field([Type], List) when is_list(List) -> + [decode_field(Type, X) || X <- List]; +decode_field(Type, List) when length(List) =:= tuple_size(Type) -> + Zipped = lists:zip(tuple_to_list(Type), List), + list_to_tuple([decode_field(X, Y) || {X, Y} <- Zipped]); +decode_field(int, <<0:8, X/binary>> = B) when X =/= <<>> -> + error({illegal, int, B}); +decode_field(int, X) when is_binary(X) -> binary:decode_unsigned(X); +decode_field(binary, X) when is_binary(X) -> X; +decode_field(bool, <<1:8>>) -> true; +decode_field(bool, <<0:8>>) -> false; +decode_field(id, Val) -> + try aeser_id:decode(Val) + catch _:_ -> error({illegal, id, Val}) + end; +decode_field(Type, X) -> error({illegal, Type, X}). + diff --git a/test/aeser_api_encoder_tests.erl b/test/aeser_api_encoder_tests.erl new file mode 100644 index 0000000..732b32d --- /dev/null +++ b/test/aeser_api_encoder_tests.erl @@ -0,0 +1,138 @@ +%%%------------------------------------------------------------------- +%%% @copyright (C) 2018, Aeternity Anstalt +%%%------------------------------------------------------------------- + +-module(aeser_api_encoder_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-define(TEST_MODULE, aeser_api_encoder). +-define(TYPES, [ {key_block_hash , 32} + , {micro_block_hash , 32} + , {block_tx_hash , 32} + , {block_state_hash , 32} + , {channel , 32} + , {contract_pubkey , 32} + , {transaction , not_applicable} + , {tx_hash , 32} + , {oracle_pubkey , 32} + , {oracle_query_id , 32} + , {account_pubkey , 32} + , {signature , 64} + , {name , not_applicable} + , {commitment , 32} + , {peer_pubkey , 32} + , {state , 32} + , {poi , not_applicable}]). + +encode_decode_test_() -> + [{"Byte sizes are correct", + fun() -> + lists:foreach( + fun({Type, ByteSize}) -> + {_Type, _, ByteSize} = {Type, ByteSize, + ?TEST_MODULE:byte_size_for_type(Type)} + end, + ?TYPES) + end + }, + {"Serialize/deserialize known types", + fun() -> + lists:foreach( + fun({Type, Size0}) -> + ByteSize = + case Size0 of + not_applicable -> 42; + _ when is_integer(Size0) -> Size0 + end, + Key = <<42:ByteSize/unit:8>>, + EncodedKey = ?TEST_MODULE:encode(Type, Key), + {Type, Key} = ?TEST_MODULE:decode(EncodedKey), + {ok, Key} = ?TEST_MODULE:safe_decode(Type, EncodedKey) + end, + ?TYPES) + end + }, + {"Key size check works", + fun() -> + lists:foreach( + fun({_Type, not_applicable}) -> ok; + ({Type, ByteSize}) -> + CheckIlligalSize = + fun(S) -> + Key = <<42:S/unit:8>>, + EncodedKey = ?TEST_MODULE:encode(Type, Key), + {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, EncodedKey) + end, + CheckIlligalSize(0), + CheckIlligalSize(ByteSize - 1), + CheckIlligalSize(ByteSize + 1) + end, + ?TYPES) + end + }, + {"Missing prefix", + fun() -> + lists:foreach( + fun({Type, Size0}) -> + ByteSize = + case Size0 of + not_applicable -> 42; + _ when is_integer(Size0) -> Size0 + end, + Key = <<42:ByteSize/unit:8>>, + EncodedKey = ?TEST_MODULE:encode(Type, Key), + <<_PartOfPrefix:1/unit:8, RestOfKey/binary>> = EncodedKey, + {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, RestOfKey), + + <<_PrefixWithoutDelimiter:2/unit:8, RestOfKey1/binary>> = EncodedKey, + {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, RestOfKey1), + + <<_WholePrefix:3/unit:8, RestOfKey2/binary>> = EncodedKey, + {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, RestOfKey2) + end, + ?TYPES) + end + }, + {"Piece of encoded key", + fun() -> + lists:foreach( + fun({Type, Size0}) -> + ByteSize = + case Size0 of + not_applicable -> 42; + _ when is_integer(Size0) -> Size0 + end, + Key = <<42:ByteSize/unit:8>>, + EncodedKey = ?TEST_MODULE:encode(Type, Key), + HalfKeySize = byte_size(EncodedKey) div 2, + <> = EncodedKey, + {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, HalfKey), + {error, invalid_encoding} = ?TEST_MODULE:safe_decode(Type, RestOfKey) + end, + ?TYPES) + end + }, + {"Encode/decode binary with only zeros", + fun() -> + Bins = [<<0:Size/unit:8>> || Size <- lists:seq(1,64)], + lists:foreach( + fun(Bin) -> + lists:foreach( + fun({Type, S}) -> + case S =:= byte_size(Bin) orelse S =:= not_applicable of + true -> + Encoded = ?TEST_MODULE:encode(Type, Bin), + {ok, Decoded} = ?TEST_MODULE:safe_decode(Type, Encoded), + ?assertEqual(Decoded, Bin); + false -> + ok + end, + Encoded1 = base58:binary_to_base58(Bin), + Decoded1 = base58:base58_to_binary(Encoded1), + ?assertEqual(Bin, Decoded1) + end, ?TYPES) + end, + Bins) + end} + ]. diff --git a/test/aeser_chain_objects_tests.erl b/test/aeser_chain_objects_tests.erl new file mode 100644 index 0000000..555b926 --- /dev/null +++ b/test/aeser_chain_objects_tests.erl @@ -0,0 +1,87 @@ +%%% -*- erlang-indent-level:4; indent-tabs-mode: nil -*- +%%%------------------------------------------------------------------- +%%% @copyright (C) 2018, Aeternity Anstalt +%%% @doc +%%% EUnit tests for aeser_chain_objects +%%% @end +%%%------------------------------------------------------------------- +-module(aeser_chain_objects_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-define(DEFAULT_TAG, account). +-define(DEFAULT_VERSION, 1). + +basic_test() -> + Template = [{foo, int}, {bar, binary}], + Values = [{foo, 1}, {bar, <<2>>}], + ?assertEqual(Values, deserialize(Template, serialize(Template, Values))). + +basic_fail_test() -> + Template = [{foo, int}, {bar, binary}], + Values = [{foo, 1}, {bar, 1}], + ?assertError({illegal_field, _, _, _, _, _}, serialize(Template, Values)). + +list_test() -> + Template = [{foo, [int]}, {bar, [binary]}, {baz, [int]}], + Values = [{foo, [1]}, {bar, [<<2>>, <<2>>]}, {baz, []}], + ?assertEqual(Values, deserialize(Template, serialize(Template, Values))). + +list_fail_test() -> + Template = [{foo, [int]}, {bar, [binary]}], + Values = [{foo, [1]}, {bar, [2, <<2>>]}], + ?assertError({illegal_field, _, _, _, _, _}, serialize(Template, Values)). + +deep_list_test() -> + Template = [{foo, [[int]]}, {bar, [[[[[binary]]]]]}], + Values = [{foo, [[1]]}, {bar, [[[[[<<2>>]]]]]}], + ?assertEqual(Values, deserialize(Template, serialize(Template, Values))). + +deep_list_fail_test() -> + Template = [{foo, [[int]]}, {bar, [[[[[binary]]]]]}], + Values = [{foo, [[1]]}, {bar, [[[[[2]]]]]}], + ?assertError({illegal_field, _, _, _, _, _}, serialize(Template, Values)). + +array_test() -> + Template = [{foo, {int, binary}}, {bar, [{int, int}]}, {baz, {int}}], + Values = [{foo, {1, <<"foo">>}}, {bar, [{1, 2}, {3, 4}, {5, 6}]}, {baz, {1}}], + ?assertEqual(Values, deserialize(Template, serialize(Template, Values))). + +array_fail_test() -> + Template = [{foo, {int, binary}}, {bar, [{int, int}]}, {baz, {int}}], + Values = [{foo, {1, <<"foo">>}}, {bar, [{1, 2}, {3, 4}, {5, 6}]}, {baz, {1, 1}}], + ?assertError({illegal_field, _, _, _, _, _}, serialize(Template, Values)). + +deep_array_test() -> + Template = [{foo, {{int, binary}}}, {bar, [{{int}, int}]}, {baz, {{int}}}], + Values = [{foo, {{1, <<"foo">>}}}, {bar, [{{1}, 2}, {{3}, 4}, {{5}, 6}]}, {baz, {{1}}}], + ?assertEqual(Values, deserialize(Template, serialize(Template, Values))). + +deep_array_fail_test() -> + Template = [{foo, {{int, binary}}}, {bar, [{{int}, int}]}, {baz, {{binary}}}], + Values = [{foo, {{1, <<"foo">>}}}, {bar, [{{1}, 2}, {{3}, 4}, {{5}, 6}]}, {baz, {{1}}}], + ?assertError({illegal_field, _, _, _, _, _}, serialize(Template, Values)). + +tag_fail_test() -> + Template = [{foo, int}, {bar, binary}], + Values = [{foo, 1}, {bar, <<2>>}], + ?assertError({illegal_serialization, _, _, _, _, _, _}, + deserialize(Template, serialize(Template, Values), signed_tx, ?DEFAULT_VERSION)). + +vsn_fail_test() -> + Template = [{foo, int}, {bar, binary}], + Values = [{foo, 1}, {bar, <<2>>}], + ?assertError({illegal_serialization, _, _, _, _, _, _}, + deserialize(Template, serialize(Template, Values), ?DEFAULT_TAG, 2)). + +deserialize(Template, Bin) -> + deserialize(Template, Bin, ?DEFAULT_TAG, ?DEFAULT_VERSION). + +deserialize(Template, Bin, Tag, Vsn) -> + aeser_chain_objects:deserialize(Tag, Vsn, Template, Bin). + +serialize(Template, Bin) -> + serialize(Template, Bin, ?DEFAULT_TAG, ?DEFAULT_VERSION). + +serialize(Template, Bin, Tag, Vsn) -> + aeser_chain_objects:serialize(Tag, Vsn, Template, Bin). diff --git a/test/aeser_rlp_tests.erl b/test/aeser_rlp_tests.erl new file mode 100644 index 0000000..4527ff1 --- /dev/null +++ b/test/aeser_rlp_tests.erl @@ -0,0 +1,132 @@ +%%%------------------------------------------------------------------- +%%% @copyright (C) 2017, Aeternity Anstalt +%%% @doc Tests for Recursive Length Prefix +%%% @end +%%%------------------------------------------------------------------- + +-module(aeser_rlp_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-define(UNTAGGED_SIZE_LIMIT , 55). +-define(UNTAGGED_LIMIT , 127). +-define(BYTE_ARRAY_OFFSET , 128). +-define(LIST_OFFSET , 192). + +-define(TEST_MODULE, aeser_rlp). + +rlp_one_byte_test() -> + B = <<42>>, + B = ?TEST_MODULE:encode(B), + B = ?TEST_MODULE:decode(B). + +rlp_another_one_byte_test() -> + B = <<127>>, + B = ?TEST_MODULE:encode(B), + B = ?TEST_MODULE:decode(B). + +rlp_zero_bytes_test() -> + B = <<>>, + S = ?BYTE_ARRAY_OFFSET + 0, + <> = ?TEST_MODULE:encode(B). + +rlp_two_bytes_test() -> + B = <<128>>, + S = ?BYTE_ARRAY_OFFSET + 1, + <> = ?TEST_MODULE:encode(B). + +rlp_one_byte_size_bytes_test() -> + L = 55, + S = ?BYTE_ARRAY_OFFSET + L, + X = << <> || X <- lists:seq(1,L)>>, + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +rlp_tagged_size_one_byte_bytes_test() -> + L = 56, + Tag = ?BYTE_ARRAY_OFFSET + ?UNTAGGED_SIZE_LIMIT + 1, + X = list_to_binary(lists:duplicate(L, 42)), + S = byte_size(X), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +rlp_tagged_size_two_bytes_bytes_test() -> + L = 256, + SizeSize = 2, + Tag = ?BYTE_ARRAY_OFFSET + ?UNTAGGED_SIZE_LIMIT + SizeSize, + X = list_to_binary(lists:duplicate(L, 42)), + S = byte_size(X), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +rlp_zero_bytes_list_test() -> + L = 0, + Tag = ?LIST_OFFSET + L, + X = [], + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +rlp_one_byte_list_test() -> + L = 1, + Tag = ?LIST_OFFSET + L, + X = lists:duplicate(L, <<42>>), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +rlp_byte_array_list_test() -> + L = 55, + Tag = ?LIST_OFFSET + L, + X = lists:duplicate(L, <<42>>), + Y = list_to_binary(X), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +rlp_byte_array_tagged_size_one_byte_list_test() -> + L = 56, + SizeSize = 1, + Tag = ?LIST_OFFSET + ?UNTAGGED_SIZE_LIMIT + SizeSize, + X = lists:duplicate(L, <<42>>), + Y = list_to_binary(X), + S = byte_size(Y), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +rlp_byte_array_tagged_size_two_bytes_list_test() -> + L = 256, + SizeSize = 2, + Tag = ?LIST_OFFSET + ?UNTAGGED_SIZE_LIMIT + SizeSize, + X = lists:duplicate(L, <<42>>), + Y = list_to_binary(X), + S = byte_size(Y), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E). + +illegal_size_encoding_list_test() -> + %% Ensure we start with somehting legal. + L = 56, + SizeSize = 1, + Tag = ?LIST_OFFSET + ?UNTAGGED_SIZE_LIMIT + SizeSize, + X = lists:duplicate(L, <<42>>), + Y = list_to_binary(X), + S = byte_size(Y), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E), + + %% Add leading zeroes to the size field. + E1 = <<(Tag + 1), 0, S:SizeSize/unit:8, Y/binary>>, + ?assertError(leading_zeroes_in_size, ?TEST_MODULE:decode(E1)). + +illegal_size_encoding_byte_array_test() -> + %% Ensure we start with somehting legal. + L = 256, + SizeSize = 2, + Tag = ?BYTE_ARRAY_OFFSET + ?UNTAGGED_SIZE_LIMIT + SizeSize, + X = list_to_binary(lists:duplicate(L, 42)), + S = byte_size(X), + E = <> = ?TEST_MODULE:encode(X), + X = ?TEST_MODULE:decode(E), + + %% Add leading zeroes to the size field. + E1 = <<(Tag + 1), 0, S:SizeSize/unit:8, X/binary>>, + ?assertError(leading_zeroes_in_size, ?TEST_MODULE:decode(E1)). +