Skip to content

Commit d9c80bb

Browse files
jrfnljrfnl
committed
PHP 8.3 | Tokenizer/PHP: add support for typed OO constants
PHP 8.3 introduced typed OO constants, where the type is between the `const` keyword and the constant name. All type variations are supported, including nullable types, union types, intersection types, with the exception of `callable`, `void` and `never`. `self` and `static` types are only allowed in Enum constants. This PR adds support for typed OO constants in the Tokenizer layer of PHPCS. The following issues had to be fixed to support typed constants: 1. Consistently tokenizing the constant _name_ as `T_STRING`, even if the name mirrors a reserved keyword, like `foreach` or a special keyword, like `self` or `true`. 2. Tokenizing a `?` at the start of a constant type declaration as `T_NULLABLE`. 3. Tokenizing a `|` and `&` operators within a constant type declaration as `T_TYPE_UNION` and `T_TYPE_INTERSECTION` respectively. Each and every part of the above has been covered by extensive tests. Includes additional tests safeguarding that the `array` keyword when used in a type declaration for a constant is tokenized as `T_STRING`. Ref: https://wiki.php.net/rfc/typed_class_constants
1 parent d70640e commit d9c80bb

13 files changed

+939
-8
lines changed

‎src/Tokenizers/PHP.php

+98-4
Original file line numberDiff line numberDiff line change
@@ -534,8 +534,9 @@ protected function tokenize($string)
534534
$numTokens = count($tokens);
535535
$lastNotEmptyToken = 0;
536536

537-
$insideInlineIf = [];
538-
$insideUseGroup = false;
537+
$insideInlineIf = [];
538+
$insideUseGroup = false;
539+
$insideConstDeclaration = false;
539540

540541
$commentTokenizer = new Comment();
541542

@@ -617,7 +618,8 @@ protected function tokenize($string)
617618
if ($tokenIsArray === true
618619
&& isset(Tokens::$contextSensitiveKeywords[$token[0]]) === true
619620
&& (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
620-
|| $finalTokens[$lastNotEmptyToken]['content'] === '&')
621+
|| $finalTokens[$lastNotEmptyToken]['content'] === '&'
622+
|| $insideConstDeclaration === true)
621623
) {
622624
if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
623625
$preserveKeyword = false;
@@ -674,6 +676,30 @@ protected function tokenize($string)
674676
}
675677
}//end if
676678

679+
// Types in typed constants should not be touched, but the constant name should be.
680+
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
681+
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
682+
|| $insideConstDeclaration === true
683+
) {
684+
$preserveKeyword = true;
685+
686+
// Find the next non-empty token.
687+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
688+
if (is_array($tokens[$i]) === true
689+
&& isset(Tokens::$emptyTokens[$tokens[$i][0]]) === true
690+
) {
691+
continue;
692+
}
693+
694+
break;
695+
}
696+
697+
if ($tokens[$i] === '=' || $tokens[$i] === ';') {
698+
$preserveKeyword = false;
699+
$insideConstDeclaration = false;
700+
}
701+
}//end if
702+
677703
if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
678704
$preserveKeyword = true;
679705

@@ -707,6 +733,26 @@ protected function tokenize($string)
707733
}
708734
}//end if
709735

736+
/*
737+
Mark the start of a constant declaration to allow for handling keyword to T_STRING
738+
convertion for constant names using reserved keywords.
739+
*/
740+
741+
if ($tokenIsArray === true && $token[0] === T_CONST) {
742+
$insideConstDeclaration = true;
743+
}
744+
745+
/*
746+
Close an open "inside constant declaration" marker when no keyword convertion was needed.
747+
*/
748+
749+
if ($insideConstDeclaration === true
750+
&& $tokenIsArray === false
751+
&& ($token[0] === '=' || $token[0] === ';')
752+
) {
753+
$insideConstDeclaration = false;
754+
}
755+
710756
/*
711757
Special case for `static` used as a function name, i.e. `static()`.
712758
*/
@@ -1713,6 +1759,20 @@ protected function tokenize($string)
17131759
$newToken = [];
17141760
$newToken['content'] = '?';
17151761

1762+
// For typed constants, we only need to check the token before the ? to be sure.
1763+
if ($finalTokens[$lastNotEmptyToken]['code'] === T_CONST) {
1764+
$newToken['code'] = T_NULLABLE;
1765+
$newToken['type'] = 'T_NULLABLE';
1766+
1767+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1768+
echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL;
1769+
}
1770+
1771+
$finalTokens[$newStackPtr] = $newToken;
1772+
$newStackPtr++;
1773+
continue;
1774+
}
1775+
17161776
/*
17171777
* Check if the next non-empty token is one of the tokens which can be used
17181778
* in type declarations. If not, it's definitely a ternary.
@@ -2058,7 +2118,30 @@ function return types. We want to keep the parenthesis map clean,
20582118
if ($tokenIsArray === true && $token[0] === T_STRING) {
20592119
$preserveTstring = false;
20602120

2061-
if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
2121+
// True/false/parent/self/static in typed constants should be fixed to their own token,
2122+
// but the constant name should not be.
2123+
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
2124+
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
2125+
|| $insideConstDeclaration === true
2126+
) {
2127+
// Find the next non-empty token.
2128+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
2129+
if (is_array($tokens[$i]) === true
2130+
&& isset(Tokens::$emptyTokens[$tokens[$i][0]]) === true
2131+
) {
2132+
continue;
2133+
}
2134+
2135+
break;
2136+
}
2137+
2138+
if ($tokens[$i] === '=') {
2139+
$preserveTstring = true;
2140+
$insideConstDeclaration = false;
2141+
}
2142+
} else if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
2143+
&& $finalTokens[$lastNotEmptyToken]['code'] !== T_CONST
2144+
) {
20622145
$preserveTstring = true;
20632146

20642147
// Special case for syntax like: return new self/new parent
@@ -2835,6 +2918,12 @@ protected function processAdditional()
28352918
$suspectedType = 'return';
28362919
}
28372920

2921+
if ($this->tokens[$x]['code'] === T_EQUAL) {
2922+
// Possible constant declaration, the `T_STRING` name will have been skipped over already.
2923+
$suspectedType = 'constant';
2924+
break;
2925+
}
2926+
28382927
break;
28392928
}//end for
28402929

@@ -2876,6 +2965,11 @@ protected function processAdditional()
28762965
break;
28772966
}
28782967

2968+
if ($suspectedType === 'constant' && $this->tokens[$x]['code'] === T_CONST) {
2969+
$confirmed = true;
2970+
break;
2971+
}
2972+
28792973
if ($suspectedType === 'property or parameter'
28802974
&& (isset(Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
28812975
|| $this->tokens[$x]['code'] === T_VAR

‎tests/Core/Tokenizer/ArrayKeywordTest.inc

+8-2
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,21 @@ $var = array(
2121
);
2222

2323
/* testFunctionDeclarationParamType */
24-
function foo(array $a) {}
24+
function typedParam(array $a) {}
2525

2626
/* testFunctionDeclarationReturnType */
27-
function foo($a) : int|array|null {}
27+
function returnType($a) : int|array|null {}
2828

2929
class Bar {
3030
/* testClassConst */
3131
const ARRAY = [];
3232

3333
/* testClassMethod */
3434
public function array() {}
35+
36+
/* testOOConstType */
37+
const array /* testTypedOOConstName */ ARRAY = /* testOOConstDefault */ array();
38+
39+
/* testOOPropertyType */
40+
protected array $property;
3541
}

‎tests/Core/Tokenizer/ArrayKeywordTest.php

+15-2
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ public static function dataArrayKeyword()
6868
'nested: inner array' => [
6969
'testMarker' => '/* testNestedArray */',
7070
],
71+
'OO constant default value' => [
72+
'testMarker' => '/* testOOConstDefault */',
73+
],
7174
];
7275

7376
}//end dataArrayKeyword()
@@ -122,6 +125,12 @@ public static function dataArrayType()
122125
'function union return type' => [
123126
'testMarker' => '/* testFunctionDeclarationReturnType */',
124127
],
128+
'OO constant type' => [
129+
'testMarker' => '/* testOOConstType */',
130+
],
131+
'OO property type' => [
132+
'testMarker' => '/* testOOPropertyType */',
133+
],
125134
];
126135

127136
}//end dataArrayType()
@@ -167,13 +176,17 @@ public function testNotArrayKeyword($testMarker, $testContent='array')
167176
public static function dataNotArrayKeyword()
168177
{
169178
return [
170-
'class-constant-name' => [
179+
'class-constant-name' => [
171180
'testMarker' => '/* testClassConst */',
172181
'testContent' => 'ARRAY',
173182
],
174-
'class-method-name' => [
183+
'class-method-name' => [
175184
'testMarker' => '/* testClassMethod */',
176185
],
186+
'class-constant-name-after-type' => [
187+
'testMarker' => '/* testTypedOOConstName */',
188+
'testContent' => 'ARRAY',
189+
],
177190
];
178191

179192
}//end dataNotArrayKeyword()

‎tests/Core/Tokenizer/BitwiseOrTest.inc

+24
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,30 @@ $result = $value | $test /* testBitwiseOr2 */ | $another;
99

1010
class TypeUnion
1111
{
12+
/* testTypeUnionOOConstSimple */
13+
public const Foo|Bar SIMPLE = new Foo;
14+
15+
/* testTypeUnionOOConstReverseModifierOrder */
16+
protected final const int|float MODIFIERS_REVERSED /* testBitwiseOrOOConstDefaultValue */ = E_WARNING | E_NOTICE;
17+
18+
const
19+
/* testTypeUnionOOConstMulti1 */
20+
array |
21+
/* testTypeUnionOOConstMulti2 */
22+
Traversable | // phpcs:ignore Stnd.Cat.Sniff
23+
false
24+
/* testTypeUnionOOConstMulti3 */
25+
| null MULTI_UNION = false;
26+
27+
/* testTypeUnionOOConstNamespaceRelative */
28+
final protected const namespace\Sub\NameA|namespace\Sub\NameB NAMESPACE_RELATIVE = new namespace\Sub\NameB;
29+
30+
/* testTypeUnionOOConstPartiallyQualified */
31+
const Partially\Qualified\NameA|Partially\Qualified\NameB PARTIALLY_QUALIFIED = new Partially\Qualified\NameA;
32+
33+
/* testTypeUnionOOConstFullyQualified */
34+
const \Fully\Qualified\NameA|\Fully\Qualified\NameB FULLY_QUALIFIED = new \Fully\Qualified\NameB();
35+
1236
/* testTypeUnionPropertySimple */
1337
public static Foo|Bar $obj;
1438

‎tests/Core/Tokenizer/BitwiseOrTest.php

+9
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public static function dataBitwiseOr()
4747
return [
4848
'in simple assignment 1' => ['/* testBitwiseOr1 */'],
4949
'in simple assignment 2' => ['/* testBitwiseOr2 */'],
50+
'in OO constant default value' => ['/* testBitwiseOrOOConstDefaultValue */'],
5051
'in property default value' => ['/* testBitwiseOrPropertyDefaultValue */'],
5152
'in method parameter default value' => ['/* testBitwiseOrParamDefaultValue */'],
5253
'in return statement' => ['/* testBitwiseOr3 */'],
@@ -97,6 +98,14 @@ public function testTypeUnion($testMarker)
9798
public static function dataTypeUnion()
9899
{
99100
return [
101+
'type for OO constant' => ['/* testTypeUnionOOConstSimple */'],
102+
'type for OO constant, reversed modifier order' => ['/* testTypeUnionOOConstReverseModifierOrder */'],
103+
'type for OO constant, first of multi-union' => ['/* testTypeUnionOOConstMulti1 */'],
104+
'type for OO constant, middle of multi-union + comments' => ['/* testTypeUnionOOConstMulti2 */'],
105+
'type for OO constant, last of multi-union' => ['/* testTypeUnionOOConstMulti3 */'],
106+
'type for OO constant, using namespace relative names' => ['/* testTypeUnionOOConstNamespaceRelative */'],
107+
'type for OO constant, using partially qualified names' => ['/* testTypeUnionOOConstPartiallyQualified */'],
108+
'type for OO constant, using fully qualified names' => ['/* testTypeUnionOOConstFullyQualified */'],
100109
'type for static property' => ['/* testTypeUnionPropertySimple */'],
101110
'type for static property, reversed modifier order' => ['/* testTypeUnionPropertyReverseModifierOrder */'],
102111
'type for property, first of multi-union' => ['/* testTypeUnionPropertyMulti1 */'],

‎tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc

+6
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ class ContextSensitiveKeywords
7676
const /* testAnd */ AND = 'LOGICAL_AND';
7777
const /* testOr */ OR = 'LOGICAL_OR';
7878
const /* testXor */ XOR = 'LOGICAL_XOR';
79+
80+
const /* testArrayIsTstringInConstType */ array /* testArrayNameForTypedConstant */ ARRAY = /* testArrayIsKeywordInConstDefault */ array();
81+
const /* testStaticIsKeywordAsConstType */ static /* testStaticIsNameForTypedConstant */ STATIC = new /* testStaticIsKeywordAsConstDefault */ static;
82+
83+
const int|bool /* testPrivateNameForUnionTypedConstant */ PRIVATE = 'PRIVATE';
84+
const Foo&Bar /* testFinalNameForIntersectionTypedConstant */ FINAL = 'FINAL';
7985
}
8086

8187
namespace /* testKeywordAfterNamespaceShouldBeString */ class;

‎tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php

+19
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,12 @@ public static function dataStrings()
118118
'constant declaration: or' => ['/* testOr */'],
119119
'constant declaration: xor' => ['/* testXor */'],
120120

121+
'constant declaration: array in type' => ['/* testArrayIsTstringInConstType */'],
122+
'constant declaration: array, name after type' => ['/* testArrayNameForTypedConstant */'],
123+
'constant declaration: static, name after type' => ['/* testStaticIsNameForTypedConstant */'],
124+
'constant declaration: private, name after type' => ['/* testPrivateNameForUnionTypedConstant */'],
125+
'constant declaration: final, name after type' => ['/* testFinalNameForIntersectionTypedConstant */'],
126+
121127
'namespace declaration: class' => ['/* testKeywordAfterNamespaceShouldBeString */'],
122128
'namespace declaration (partial): my' => ['/* testNamespaceNameIsString1 */'],
123129
'namespace declaration (partial): class' => ['/* testNamespaceNameIsString2 */'],
@@ -182,6 +188,19 @@ public static function dataKeywords()
182188
'testMarker' => '/* testNamespaceIsKeyword */',
183189
'expectedTokenType' => 'T_NAMESPACE',
184190
],
191+
'array: default value in const decl' => [
192+
'testMarker' => '/* testArrayIsKeywordInConstDefault */',
193+
'expectedTokenType' => 'T_ARRAY',
194+
],
195+
'static: type in constant declaration' => [
196+
'testMarker' => '/* testStaticIsKeywordAsConstType */',
197+
'expectedTokenType' => 'T_STATIC',
198+
],
199+
'static: value in constant declaration' => [
200+
'testMarker' => '/* testStaticIsKeywordAsConstDefault */',
201+
'expectedTokenType' => 'T_STATIC',
202+
],
203+
185204
'abstract: class declaration' => [
186205
'testMarker' => '/* testAbstractIsKeyword */',
187206
'expectedTokenType' => 'T_ABSTRACT',

‎tests/Core/Tokenizer/OtherContextSensitiveKeywordsTest.inc

+14
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,17 @@ function standAloneFalseTrueNullTypesAndMore(
5151
|| $a === /* testNullIsKeywordInComparison */ null
5252
) {}
5353
}
54+
55+
class TypedConstProp {
56+
const /* testFalseIsKeywordAsConstType */ false /* testFalseIsNameForTypedConstant */ FALSE = /* testFalseIsKeywordAsConstDefault */ false;
57+
const /* testTrueIsKeywordAsConstType */ true /* testTrueIsNameForTypedConstant */ TRUE = /* testTrueIsKeywordAsConstDefault */ true;
58+
const /* testNullIsKeywordAsConstType */ null /* testNullIsNameForTypedConstant */ NULL = /* testNullIsKeywordAsConstDefault */ null;
59+
const /* testSelfIsKeywordAsConstType */ self /* testSelfIsNameForTypedConstant */ SELF = new /* testSelfIsKeywordAsConstDefault */ self;
60+
const /* testParentIsKeywordAsConstType */ parent /* testParentIsNameForTypedConstant */ PARENT = new /* testParentIsKeywordAsConstDefault */ parent;
61+
62+
public /* testFalseIsKeywordAsPropertyType */ false $false = /* testFalseIsKeywordAsPropertyDefault */ false;
63+
protected readonly /* testTrueIsKeywordAsPropertyType */ true $true = /* testTrueIsKeywordAsPropertyDefault */ true;
64+
static private /* testNullIsKeywordAsPropertyType */ null $null = /* testNullIsKeywordAsPropertyDefault */ null;
65+
var /* testSelfIsKeywordAsPropertyType */ self $self = new /* testSelfIsKeywordAsPropertyDefault */ self;
66+
protected /* testParentIsKeywordAsPropertyType */ parent $parent = new /* testParentIsKeywordAsPropertyDefault */ parent;
67+
}

0 commit comments

Comments
 (0)