From bb56b2e2b2277b9d4c45a94017c1be49220e691e Mon Sep 17 00:00:00 2001
From: Thomas Calvet <calvet.thomas@gmail.com>
Date: Mon, 15 Nov 2021 17:49:46 +0100
Subject: [PATCH] [ExpressionLanguage] Support lexing numbers with underscores
 and decimals with no leading zero

---
 .../Component/ExpressionLanguage/CHANGELOG.md |  6 ++++++
 .../Component/ExpressionLanguage/Lexer.php    | 12 ++++++------
 .../ExpressionLanguage/Tests/LexerTest.php    | 19 +++++++++++++++++++
 .../ExpressionLanguage/Tests/ParserTest.php   |  4 ++++
 4 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/Symfony/Component/ExpressionLanguage/CHANGELOG.md b/src/Symfony/Component/ExpressionLanguage/CHANGELOG.md
index f5c1f6de1596a..9c4f52b899f89 100644
--- a/src/Symfony/Component/ExpressionLanguage/CHANGELOG.md
+++ b/src/Symfony/Component/ExpressionLanguage/CHANGELOG.md
@@ -1,6 +1,12 @@
 CHANGELOG
 =========
 
+6.1
+-----
+
+ * Support lexing numbers with the numeric literal separator `_`
+ * Support lexing decimals with no leading zero
+
 5.1.0
 -----
 
diff --git a/src/Symfony/Component/ExpressionLanguage/Lexer.php b/src/Symfony/Component/ExpressionLanguage/Lexer.php
index 2c69d344dd44e..1eeb7ac1852c1 100644
--- a/src/Symfony/Component/ExpressionLanguage/Lexer.php
+++ b/src/Symfony/Component/ExpressionLanguage/Lexer.php
@@ -38,13 +38,13 @@ public function tokenize(string $expression): TokenStream
                 continue;
             }
 
-            if (preg_match('/[0-9]+(?:\.[0-9]+)?([Ee][\+\-][0-9]+)?/A', $expression, $match, 0, $cursor)) {
+            if (preg_match('/
+                (?(DEFINE)(?P<LNUM>[0-9]+(_[0-9]+)*))
+                (?:\.(?&LNUM)|(?&LNUM)(?:\.(?!\.)(?&LNUM)?)?)(?:[eE][+-]?(?&LNUM))?/Ax',
+                $expression, $match, 0, $cursor)
+            ) {
                 // numbers
-                $number = (float) $match[0];  // floats
-                if (preg_match('/^[0-9]+$/', $match[0]) && $number <= \PHP_INT_MAX) {
-                    $number = (int) $match[0]; // integers lower than the maximum
-                }
-                $tokens[] = new Token(Token::NUMBER_TYPE, $number, $cursor + 1);
+                $tokens[] = new Token(Token::NUMBER_TYPE, 0 + str_replace('_', '', $match[0]), $cursor + 1);
                 $cursor += \strlen($match[0]);
             } elseif (str_contains('([{', $expression[$cursor])) {
                 // opening bracket
diff --git a/src/Symfony/Component/ExpressionLanguage/Tests/LexerTest.php b/src/Symfony/Component/ExpressionLanguage/Tests/LexerTest.php
index 67e551f587eb7..b0efa2b168f51 100644
--- a/src/Symfony/Component/ExpressionLanguage/Tests/LexerTest.php
+++ b/src/Symfony/Component/ExpressionLanguage/Tests/LexerTest.php
@@ -135,6 +135,25 @@ public function getTokenizeData()
                 ],
                 'foo.not in [bar]',
             ],
+            [
+                [new Token('number', 0.787, 1)],
+                '0.787',
+            ],
+            [
+                [new Token('number', 0.1234, 1)],
+                '.1234',
+            ],
+            [
+                [new Token('number', 188165.1178, 1)],
+                '188_165.1_178',
+            ],
+            [
+                [
+                    new Token('operator', '-', 1),
+                    new Token('number', 7189000000.0, 2),
+                ],
+                '-.7_189e+10',
+            ],
         ];
     }
 }
diff --git a/src/Symfony/Component/ExpressionLanguage/Tests/ParserTest.php b/src/Symfony/Component/ExpressionLanguage/Tests/ParserTest.php
index 3cec3e2cebdee..8eb61643c9bb2 100644
--- a/src/Symfony/Component/ExpressionLanguage/Tests/ParserTest.php
+++ b/src/Symfony/Component/ExpressionLanguage/Tests/ParserTest.php
@@ -189,6 +189,10 @@ public function getParseData()
                 new Node\BinaryNode('..', new Node\ConstantNode(0), new Node\ConstantNode(3)),
                 '0..3',
             ],
+            [
+                new Node\BinaryNode('+', new Node\ConstantNode(0), new Node\ConstantNode(0.1)),
+                '0+.1',
+            ],
         ];
     }