Fixed performance issues. Changed include management

src/aeso_compiler.erl

@@ -140,7 +140,16 @@ from_string1(fate, ContractString, Options) ->
 
 -spec string_to_code(string(), options()) -> map().
 string_to_code(ContractString, Options) ->
-    Ast = parse(ContractString, Options),
+    Ast = case lists:member(no_implicit_stdlib, Options) of
+              true -> parse(ContractString, Options);
+              false ->
+                  IncludedSTD = sets:from_list(
+                                  [aeso_parser:hash_include(F, C)
+                                   || {F, C} <- aeso_stdlib:stdlib_list()]),
+                  InitAst = parse(ContractString, IncludedSTD, Options),
+                  STD = parse_stdlib(),
+                  STD ++ InitAst
+          end,
     pp_sophia_code(Ast, Options),
     pp_ast(Ast, Options),
     {TypeEnv, TypedAst} = aeso_ast_infer_types:infer(Ast, [return_env]),
@@ -569,6 +578,14 @@ pp(Code, Options, Option, PPFun) ->
 %% -------------------------------------------------------------------
 %% TODO: Tempoary parser hook below...
 
+parse_stdlib() ->
+    lists:foldr(
+      fun ({Lib, LibCode}, Acc) ->
+              parse(LibCode, [{src_file, Lib}]) ++ Acc
+      end,
+      [],
+      aeso_stdlib:stdlib_list()).
+
 sophia_type_to_typerep(String) ->
     {ok, Ast} = aeso_parser:type(String),
     try aeso_ast_to_icode:ast_typerep(Ast) of
@@ -577,8 +594,10 @@ sophia_type_to_typerep(String) ->
     end.
 
 parse(Text, Options) ->
+    parse(Text, sets:new(), Options).
+parse(Text, Included, Options) ->
     %% Try and return something sensible here!
-    case aeso_parser:string(Text, Options) of
+    case aeso_parser:string(Text, Included, Options) of
         %% Yay, it worked!
         {ok, Contract} -> Contract;
         %% Scan errors.

src/aeso_parser.erl

@@ -7,6 +7,7 @@
 -export([string/1,
          string/2,
          string/3,
+         hash_include/2,
          type/1]).
 
 -include("aeso_parse_lib.hrl").
@@ -27,18 +28,11 @@ string(String, Opts) ->
         false -> string(String, sets:new(), Opts)
     end.
 
--spec string(string(), sets:set(string()), aeso_compiler:options()) -> parse_result().
+-spec string(string(), sets:set(binary()), aeso_compiler:options()) -> parse_result().
 string(String, Included, Opts) ->
     case parse_and_scan(file(), String, Opts) of
         {ok, AST} ->
-            STD = case lists:member(no_implicit_stdlib, Opts) of
+            expand_includes(AST, Included, Opts);
-                      false -> [{ include, [{src_file, File}, {origin, system}]
-                                , {string, [{src_file, File}, {origin, system}], File}}
-                                || {File, _} <- aeso_stdlib:stdlib_list()
-                               ];
-                      true -> []
-                  end,
-            expand_includes(STD ++ AST, Included, Opts);
         Err = {error, _} ->
             Err
     end.
@@ -558,32 +552,28 @@ expand_includes(AST, Included, Opts) ->
 
 expand_includes([], _Included, Acc, _Opts) ->
     {ok, lists:reverse(Acc)};
-expand_includes([{include, Ann, S = {string, _, File}} | AST], Included, Acc, Opts) ->
+expand_includes([{include, Ann, {string, SAnn, File}} | AST], Included, Acc, Opts) ->
-    case sets:is_element(File, Included) of
+    case get_include_code(File, Ann, Opts) of
-        false ->
+        {ok, Code} ->
-            Opts1 = lists:keystore(src_file, 1, Opts, {src_file, File}),
+            Hashed = hash_include(File, Code),
-            Included1 = sets:add_element(File, Included),
+            case sets:is_element(Hashed, Included) of
-            case {read_file(File, Opts), maps:find(File, aeso_stdlib:stdlib())} of
+                false ->
-                {{ok, _}, {ok,_ }} ->
+                    Opts1 = lists:keystore(src_file, 1, Opts, {src_file, File}),
-                    return_error(ann_pos(Ann), "Illegal redefinition of standard library " ++ File);
+                    Included1 = sets:add_element(Hashed, Included),
-                {_, {ok, Lib}} ->
+                    case string(Code, Included1, Opts1) of
-                    case string(Lib, Included1, [no_implicit_stdlib, Opts1]) of
                         {ok, AST1} ->
-                            expand_includes(AST1 ++ AST, Included1, Acc, Opts);
+                            Dependencies = [ {include, Ann, {string, SAnn, Dep}}
+                                             || Dep <- aeso_stdlib:dependencies(File)
+                                           ],
+                            expand_includes(Dependencies ++ AST1 ++ AST, Included1, Acc, Opts);
                         Err = {error, _} ->
                             Err
                     end;
-                {{ok, Bin}, _} ->
+                true ->
-                    case string(binary_to_list(Bin), Included1, Opts1) of
+                    expand_includes(AST, Included, Acc, Opts)
-                        {ok, AST1} ->
-                            expand_includes(AST1 ++ AST, Included1, Acc, Opts);
-                        Err = {error, _} ->
-                            Err
-                    end;
-                {_, _} ->
-                    {error, {get_pos(S), include_error, File}}
             end;
-        true -> expand_includes(AST, Included, Acc, Opts)
+        Err = {error, _} ->
+            Err
     end;
 expand_includes([E | AST], Included, Acc, Opts) ->
     expand_includes(AST, Included, [E | Acc], Opts).
@@ -601,3 +591,20 @@ read_file(File, Opts) ->
             end
     end.
 
+get_include_code(File, Ann, Opts) ->
+    case {read_file(File, Opts), maps:find(File, aeso_stdlib:stdlib())} of
+        {{ok, _}, {ok,_ }} ->
+            return_error(ann_pos(Ann), "Illegal redefinition of standard library " ++ File);
+        {_, {ok, Lib}} ->
+            {ok, Lib};
+        {{ok, Bin}, _} ->
+            {ok, binary_to_list(Bin)};
+        {_, _} ->
+            {error, {ann_pos(Ann), include_error, File}}
+    end.
+
+-spec hash_include(string() | binary(), string()) -> binary().
+hash_include(File, Code) when is_binary(File) ->
+    hash_include(binary_to_list(File), Code);
+hash_include(File, Code) when is_list(File) ->
+    {filename:basename(File), crypto:hash(sha256, Code)}.

src/aeso_stdlib.erl

@@ -10,7 +10,7 @@
 
 -module(aeso_stdlib).
 
--export([stdlib/0, stdlib_list/0]).
+-export([stdlib/0, stdlib_list/0, dependencies/1]).
 
 stdlib() ->
     maps:from_list(stdlib_list()).
@@ -23,6 +23,13 @@ stdlib_list() ->
     , {<<"Triple.aes">>, std_triple()}
     ].
 
+dependencies(Q) ->
+    case Q of
+        <<"Option.aes">> ->
+            [<<"List.aes">>];
+        _ -> []
+    end.
+
 std_func() ->
 "
 namespace Func =
@@ -286,8 +293,6 @@ namespace List =
 ".
 
 std_option() -> "
-include \"List.aes\"
-
 namespace Option =
 
   function is_none(o : option('a)) : bool = switch(o)

test/aeso_abi_tests.erl

@@ -80,11 +80,11 @@ encode_decode_sophia_string(SophiaType, String) ->
            , "  record r = {x : an_alias(int), y : variant}\n"
            , "  datatype variant = Red | Blue(map(string, int))\n"
            , "  entrypoint foo : arg_type => arg_type\n" ],
-    case aeso_compiler:check_call(lists:flatten(Code), "foo", [String], []) of
+    case aeso_compiler:check_call(lists:flatten(Code), "foo", [String], [no_implicit_stdlib]) of
         {ok, _, {[Type], _}, [Arg]} ->
             io:format("Type ~p~n", [Type]),
             Data = encode(Arg),
-            case aeso_compiler:to_sophia_value(Code, "foo", ok, Data) of
+            case aeso_compiler:to_sophia_value(Code, "foo", ok, Data, [no_implicit_stdlib]) of
                 {ok, Sophia} ->
                     lists:flatten(io_lib:format("~s", [prettypr:format(aeso_pretty:expr(Sophia))]));
                 {error, Err} ->
@@ -152,7 +152,7 @@ oracle_test() ->
         "    Oracle.get_question(o, q)\n",
     {ok, _, {[word, word], {list, string}}, [16#123, 16#456]} =
         aeso_compiler:check_call(Contract, "question", ["ok_111111111111111111111111111111ZrdqRz9",
-                                                        "oq_1111111111111111111111111111113AFEFpt5"], []),
+                                                        "oq_1111111111111111111111111111113AFEFpt5"], [no_implicit_stdlib]),
 
     ok.
 
@@ -162,7 +162,7 @@ permissive_literals_fail_test() ->
         "  stateful entrypoint haxx(o : oracle(list(string), option(int))) =\n"
         "    Chain.spend(o, 1000000)\n",
         {error, <<"Type errors\nCannot unify", _/binary>>} =
-            aeso_compiler:check_call(Contract, "haxx", ["#123"], []),
+            aeso_compiler:check_call(Contract, "haxx", ["#123"], [no_implicit_stdlib]),
     ok.
 
 encode_decode_calldata(FunName, Types, Args) ->
@@ -173,8 +173,8 @@ encode_decode_calldata(FunName, Types, Args, RetType) ->
     encode_decode_calldata_(Code, FunName, Args, RetType).
 
 encode_decode_calldata_(Code, FunName, Args, RetVMType) ->
-    {ok, Calldata} = aeso_compiler:create_calldata(Code, FunName, Args),
+    {ok, Calldata} = aeso_compiler:create_calldata(Code, FunName, Args, [no_implicit_stdlib]),
-    {ok, _, {ArgTypes, RetType}, _} = aeso_compiler:check_call(Code, FunName, Args, [{backend, aevm}]),
+    {ok, _, {ArgTypes, RetType}, _} = aeso_compiler:check_call(Code, FunName, Args, [{backend, aevm}, no_implicit_stdlib]),
     ?assertEqual(RetType, RetVMType),
     CalldataType = {tuple, [word, {tuple, ArgTypes}]},
     {ok, {_Hash, ArgTuple}} = aeb_heap:from_binary(CalldataType, Calldata),
@@ -182,7 +182,7 @@ encode_decode_calldata_(Code, FunName, Args, RetVMType) ->
         "init" ->
             ok;
         _ ->
-            {ok, _ArgTypes, ValueASTs} = aeso_compiler:decode_calldata(Code, FunName, Calldata),
+            {ok, _ArgTypes, ValueASTs} = aeso_compiler:decode_calldata(Code, FunName, Calldata, [no_implicit_stdlib]),
             Values = [ prettypr:format(aeso_pretty:expr(V)) || V <- ValueASTs ],
             ?assertMatch({X, X}, {Args, Values})
     end,

test/aeso_aci_tests.erl

@@ -87,7 +87,8 @@ aci_test_() ->
       fun() -> aci_test_contract(ContractName) end}
      || ContractName <- all_contracts()].
 
-all_contracts() -> aeso_compiler_tests:compilable_contracts().
+all_contracts() -> [C || C <- aeso_compiler_tests:compilable_contracts()
+                             , not aeso_compiler_tests:wants_stdlib(C)].
 
 aci_test_contract(Name) ->
     String = aeso_test_utils:read_contract(Name),
@@ -98,7 +99,7 @@ aci_test_contract(Name) ->
     {ok, ContractStub} = aeso_aci:render_aci_json(JSON),
 
     io:format("STUB:\n~s\n", [ContractStub]),
-    check_stub(ContractStub, [{src_file, Name}, no_implicit_stdlib]),
+    check_stub(ContractStub, [{src_file, Name}]),
 
     ok.
 

test/aeso_calldata_tests.erl

@@ -21,12 +21,14 @@ calldata_test_() ->
            ContractString = aeso_test_utils:read_contract(ContractName),
            AevmExprs =
               case not lists:member(ContractName, not_yet_compilable(aevm)) of
-                  true -> ast_exprs(ContractString, Fun, Args, [{backend, aevm}]);
+                  true -> ast_exprs(ContractString, Fun, Args, [{backend, aevm}]
+                                    ++ [no_implicit_stdlib || not aeso_compiler_tests:wants_stdlib(ContractName)]);
                   false -> undefined
               end,
            FateExprs =
               case not lists:member(ContractName, not_yet_compilable(fate)) of
-                  true -> ast_exprs(ContractString, Fun, Args, [{backend, fate}]);
+                  true -> ast_exprs(ContractString, Fun, Args, [{backend, fate}]
+                                    ++ [no_implicit_stdlib || not aeso_compiler_tests:wants_stdlib(ContractName)]);
                   false -> undefined
               end,
            case FateExprs == undefined orelse AevmExprs == undefined of
@@ -45,12 +47,14 @@ calldata_aci_test_() ->
            io:format("ACI:\n~s\n", [ContractACIBin]),
            AevmExprs =
               case not lists:member(ContractName, not_yet_compilable(aevm)) of
-                  true -> ast_exprs(ContractACI, Fun, Args, [{backend, aevm}]);
+                  true -> ast_exprs(ContractACI, Fun, Args, [{backend, aevm}]
+                                    ++ [no_implicit_stdlib || not aeso_compiler_tests:wants_stdlib(ContractName)]);
                   false -> undefined
               end,
            FateExprs =
               case not lists:member(ContractName, not_yet_compilable(fate)) of
-                  true -> ast_exprs(ContractACI, Fun, Args, [{backend, fate}]);
+                  true -> ast_exprs(ContractACI, Fun, Args, [{backend, fate}]
+                                    ++ [no_implicit_stdlib || not aeso_compiler_tests:wants_stdlib(ContractName)]);
                   false -> undefined
               end,
            case FateExprs == undefined orelse AevmExprs == undefined of

test/aeso_compiler_tests.erl

@@ -73,7 +73,9 @@ check_errors(Expect, ErrorString) ->
     end.
 
 compile(Backend, Name) ->
-    compile(Backend, Name, [{include, {file_system, [aeso_test_utils:contract_path()]}}]).
+    compile(Backend, Name,
+            [{include, {file_system, [aeso_test_utils:contract_path()]}}]
+            ++ [no_implicit_stdlib || not wants_stdlib(Name)]).
 
 compile(Backend, Name, Options) ->
     String = aeso_test_utils:read_contract(Name),
@@ -121,6 +123,8 @@ compilable_contracts() ->
      "tuple_match",
      "cyclic_include",
      "stdlib_include",
+     "double_include",
+     "manual_stdlib_include",
      "list_comp"
     ].
 
@@ -367,3 +371,13 @@ failing_contracts() ->
       [<<"Cannot unify int\n         and string\nwhen checking the type of the pattern at line 2, column 53\n  x : int\nagainst the expected type\n  string">>
       ]}
     ].
+
+wants_stdlib(Name) ->
+    lists:member
+      (Name,
+       [ "stdlib_include",
+         "list_comp",
+         "list_comp_not_a_list",
+         "list_comp_if_not_bool",
+         "list_comp_bad_shadow"
+       ]).

test/contracts/double_include.aes

@@ -0,0 +1,7 @@
+include "included.aes"
+include "../contracts/included.aes"
+
+contract Include =
+  entrypoint foo() =
+    Included.foo()
+

test/contracts/manual_stdlib_include.aes

@@ -0,0 +1,7 @@
+// This contract should be compiled with no_implicit_stdlib option.
+// It should include Lists.aes implicitly however, because Option.aes depends on it.
+include "Option.aes"
+
+contract Test =
+  entrypoint i_should_build() =
+    List.is_empty(Option.to_list(None))