From f81dc885264f0101377ffbc5aa3665df03ae3dcf Mon Sep 17 00:00:00 2001
From: Ulf Norell <ulf.norell@gmail.com>
Date: Mon, 23 Sep 2019 14:04:09 +0200
Subject: [PATCH] Allow underscore separators in number and bytes literals

For instance, `1_000_000_000` or `#FFFF_FFFF_FFFF_FFFF`
---
 src/aeso_scan.erl                             | 25 +++++++++++++------
 test/aeso_compiler_tests.erl                  |  3 ++-
 test/contracts/underscore_number_literals.aes | 13 ++++++++++
 3 files changed, 32 insertions(+), 9 deletions(-)
 create mode 100644 test/contracts/underscore_number_literals.aes

diff --git a/src/aeso_scan.erl b/src/aeso_scan.erl
index 8dc47d4..1c30016 100644
--- a/src/aeso_scan.erl
+++ b/src/aeso_scan.erl
@@ -13,14 +13,15 @@
                         override/2, push/2, pop/1]).
 
 lexer() ->
+    Number   = fun(Digit) -> [Digit, "+(_", Digit, "+)*"] end,
     DIGIT    = "[0-9]",
     HEXDIGIT = "[0-9a-fA-F]",
     LOWER    = "[a-z_]",
     UPPER    = "[A-Z]",
     CON      = [UPPER, "[a-zA-Z0-9_]*"],
-    INT      = [DIGIT, "+"],
-    HEX      = ["0x", HEXDIGIT, "+"],
-    BYTES    = ["#", HEXDIGIT, "+"],
+    INT      = Number(DIGIT),
+    HEX      = ["0x", Number(HEXDIGIT)],
+    BYTES    = ["#", Number(HEXDIGIT)],
     WS       = "[\\000-\\ ]+",
     ID       = [LOWER, "[a-zA-Z0-9_']*"],
     TVAR     = ["'", ID],
@@ -53,7 +54,7 @@ lexer() ->
         , {CHAR,   token(char,   fun parse_char/1)}
         , {STRING, token(string, fun parse_string/1)}
         , {HEX,    token(hex,    fun parse_hex/1)}
-        , {INT,    token(int,    fun list_to_integer/1)}
+        , {INT,    token(int,    fun parse_int/1)}
         , {BYTES,  token(bytes,  fun parse_bytes/1)}
 
           %% Identifiers (qualified first!)
@@ -117,10 +118,18 @@ unescape([$\\, Code | Chars], Acc) ->
 unescape([C | Chars], Acc) ->
     unescape(Chars, [C | Acc]).
 
-parse_hex("0x" ++ Chars) -> list_to_integer(Chars, 16).
+strip_underscores(S) ->
+    lists:filter(fun(C) -> C /= $_ end, S).
 
-parse_bytes("#" ++ Chars) ->
-    N      = list_to_integer(Chars, 16),
-    Digits = (length(Chars) + 1) div 2,
+parse_hex("0x" ++ S) ->
+    list_to_integer(strip_underscores(S), 16).
+
+parse_int(S) ->
+    list_to_integer(strip_underscores(S)).
+
+parse_bytes("#" ++ S0) ->
+    S      = strip_underscores(S0),
+    N      = list_to_integer(S, 16),
+    Digits = (length(S) + 1) div 2,
     <<N:Digits/unit:8>>.
 
diff --git a/test/aeso_compiler_tests.erl b/test/aeso_compiler_tests.erl
index 5c4bed8..9d6def9 100644
--- a/test/aeso_compiler_tests.erl
+++ b/test/aeso_compiler_tests.erl
@@ -152,7 +152,8 @@ compilable_contracts() ->
      "manual_stdlib_include",
      "list_comp",
      "payable",
-     "unapplied_builtins"
+     "unapplied_builtins",
+     "underscore_number_literals"
     ].
 
 not_yet_compilable(fate) -> [];
diff --git a/test/contracts/underscore_number_literals.aes b/test/contracts/underscore_number_literals.aes
new file mode 100644
index 0000000..cf926aa
--- /dev/null
+++ b/test/contracts/underscore_number_literals.aes
@@ -0,0 +1,13 @@
+
+contract UnderscoreNumberLiterals =
+
+  entrypoint ints() : list(int) =
+    [ 1_999_000_000,
+      19_99_00_00_00,
+      0xfff_FFF_010 ]
+
+  entrypoint bytes() : list(bytes(4)) =
+    [ #abcd_ef_00,
+      #01_02_03_04,
+      #aaaa_FFFF ]
+