Add support for float and bool as token value

That means that `Lexer\::getType()` might transform `$value` into
string|int|float|bool, but never into objects.

Author: PowerKiKi

psalm.xml

@@ -68,5 +68,8 @@
                 <file name="tests/AbstractLexerTest.php" />
             </errorLevel>
         </RedundantConditionGivenDocblockType>
+
+        <!-- cannot fix that because we support PHP 8.1+ -->
+        <MissingClassConstType errorLevel="suppress" />
     </issueHandlers>
 </psalm>

src/AbstractLexer.php

@@ -20,7 +20,7 @@
  * Base class for writing simple lexers, i.e. for creating small DSLs.
  *
  * @template T of UnitEnum|string|int
- * @template V of string|int
+ * @template V of string|int|float|bool
  */
 abstract class AbstractLexer
 {

src/Token.php

@@ -10,7 +10,7 @@
 
 /**
  * @template T of UnitEnum|string|int
- * @template V of string|int
+ * @template V of string|int|float|bool
  */
 final class Token
 {
@@ -20,7 +20,7 @@ final class Token
      * @readonly
      * @var V
      */
-    public string|int $value;
+    public string|int|float|bool $value;
 
     /**
      * The type of the token (identifier, numeric, string, input parameter, none)
@@ -41,7 +41,7 @@ final class Token
      * @param V      $value
      * @param T|null $type
      */
-    public function __construct(string|int $value, $type, int $position)
+    public function __construct(string|int|float|bool $value, $type, int $position)
     {
         $this->value    = $value;
         $this->type     = $type;

tests/AbstractLexerTest.php

@@ -4,13 +4,16 @@
 
 namespace Doctrine\Tests\Common\Lexer;
 
+use Doctrine\Common\Lexer\AbstractLexer;
 use Doctrine\Common\Lexer\Token;
 use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\TestCase;
 
 use function array_map;
 use function assert;
 use function count;
+use function is_int;
+use function is_numeric;
 use function setlocale;
 
 use const LC_ALL;
@@ -279,4 +282,60 @@ public function testMarkerAnnotationLocaleTr(): void
         self::assertEquals('@', $mutableLexer->token->value);
         self::assertEquals('ODM\Id', $mutableLexer->lookahead->value);
     }
+
+    public function testCanTokenizeFloatValue(): void
+    {
+        $lexer = new /** @template-extends AbstractLexer<int, string|int|float|bool> */ class () extends AbstractLexer {
+            final public const T_NONE    = 1;
+            final public const T_INTEGER = 2;
+            final public const T_FLOAT   = 4;
+            final public const T_BOOL    = 8;
+
+            protected function getType(string|int|float|bool &$value): int
+            {
+                if ($value === 'y') {
+                    $value = true;
+
+                    return self::T_BOOL;
+                }
+
+                if (is_numeric($value)) {
+                    $value += 0;
+
+                    if (is_int($value)) {
+                        return self::T_INTEGER;
+                    }
+
+                    return self::T_FLOAT;
+                }
+
+                return self::T_NONE;
+            }
+
+            /** {@inheritDoc} */
+            protected function getCatchablePatterns(): array
+            {
+                return [
+                    '(?:[0-9]+)(?:[\.][0-9]+)?(?:e[+-]?[0-9]+)?',
+                    'y',
+                ];
+            }
+
+            /** {@inheritDoc} */
+            protected function getNonCatchablePatterns(): array
+            {
+                return ['\s+'];
+            }
+        };
+
+        $lexer->setInput('123.456');
+        $token = $lexer->peek();
+        assert($token !== null);
+        self::assertSame(123.456, $token->value, 'expect a real float, not a numerical string');
+
+        $lexer->setInput('y');
+        $token = $lexer->peek();
+        assert($token !== null);
+        self::assertTrue($token->value, 'expect a real bool, not a numerical string');
+    }
 }