diff --git a/.gitattributes b/.gitattributes index a0b89270..de5a88e7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,16 +1,16 @@ -*.abnf text eol=crlf *.php text eol=lf +*.pp3 text eol=lf .github export-ignore apigen export-ignore phpcs.xml export-ignore doc export-ignore tests export-ignore +tools export-ignore tmp export-ignore .editorconfig export-ignore .gitattributes export-ignore .gitignore export-ignore -build-abnfgen.sh export-ignore CLAUDE.md export-ignore CODE_OF_CONDUCT.md export-ignore Makefile export-ignore diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index efe034bc..84ba11c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,6 +140,44 @@ jobs: - name: "Tests" run: "make tests" + grammars: + name: "Grammars" + runs-on: "ubuntu-latest" + + strategy: + fail-fast: false + matrix: + php-version: + - "8.4" + - "8.5" + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + with: + egress-policy: audit + + - name: "Checkout" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: "Install PHP" + uses: "shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240" # v2 + with: + coverage: "none" + php-version: "${{ matrix.php-version }}" + tools: composer:v2 + + - name: "Install dependencies" + run: "composer update --no-interaction --no-progress" + + # The compiler reading doc/grammars asks for PHP 8.4, so the corpus is + # only written here. Everywhere else FuzzyTest skips itself. + - name: "Install the grammar toolchain" + run: "make grammars-install" + + - name: "Tests" + run: "make tests" + static-analysis: name: "PHPStan" runs-on: "ubuntu-latest" diff --git a/.gitignore b/.gitignore index 78981962..e7456d98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ /docs /temp -/tools +/tools/phplrt/vendor /tests/tmp /build-cs /vendor diff --git a/Makefile b/Makefile index 294ad0c5..40cf4820 100644 --- a/Makefile +++ b/Makefile @@ -34,3 +34,14 @@ phpstan: .PHONY: phpstan-generate-baseline phpstan-generate-baseline: php vendor/bin/phpstan --generate-baseline + +# --------------------------------------------------------------------------- +# The grammars +# --------------------------------------------------------------------------- + +# The tool writing a corpus out of doc/grammars is a development dependency of +# its own, because the grammar compiler asks for a PHP this library still runs +# without. +.PHONY: grammars-install +grammars-install: + composer install --no-interaction --working-dir tools/phplrt diff --git a/README.md b/README.md index 08e244b6..9118b121 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,19 @@ $newPhpDoc = $printer->printFormatPreserving($newPhpDocNode, $phpDocNode, $token echo $newPhpDoc; // '/** @param Ipsum $a */' ``` +## The grammars + +The language this library reads is written down as a grammar in +[`doc/grammars`](doc/grammars), in the format the [phplrt](https://phplrt.org) +compiler reads. A grammar says what a PHPDoc may be written as, so it can be +walked the other way round and asked for PHPDocs instead of being asked about +one: that is where `FuzzyTest` gets its corpus, and it covers a great deal more +of the language than a hand-written one does. + +Nothing in `src/` reads those files, and the library needs neither the toolchain +writing the corpus nor the PHP 8.4 it asks for. See +[`doc/grammars/README.md`](doc/grammars/README.md). + ## Code of Conduct This project adheres to a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this project and its community, you are expected to uphold this code. @@ -181,3 +194,11 @@ Afterwards you can either run the whole build including linting and coding stand or run only tests using make tests + +The grammars have a toolchain of their own, because the compiler reading them +asks for PHP 8.4. Without it the fuzzy tests skip themselves: + + make grammars-install # install it + +`FuzzyTest` then writes its own corpus out of `doc/grammars/*.pp3` every time it +runs, and leaves it in `temp/fuzzy` to be looked at afterwards. diff --git a/build-abnfgen.sh b/build-abnfgen.sh deleted file mode 100755 index 6e8752b4..00000000 --- a/build-abnfgen.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT_DIR="$DIR" - -if [[ ! -d "$ROOT_DIR/tools/abnfgen" ]]; then - rm -rf "$ROOT_DIR/temp/abnfgen" - mkdir -p "$ROOT_DIR/temp/abnfgen" - - tar xf "$ROOT_DIR/tests/abnfgen-0.20.tar.gz" \ - --directory "$ROOT_DIR/temp/abnfgen" \ - --strip-components 1 - - cd "$ROOT_DIR/temp/abnfgen" - ./configure - make - - mkdir -p "$ROOT_DIR/tools/abnfgen" - mv abnfgen "$ROOT_DIR/tools/abnfgen" - rm -rf "$ROOT_DIR/temp/abnfgen" "$ROOT_DIR/temp/abnfgen.tar.gz" -fi diff --git a/doc/grammars/README.md b/doc/grammars/README.md new file mode 100644 index 00000000..99960cca --- /dev/null +++ b/doc/grammars/README.md @@ -0,0 +1,130 @@ +# The grammars of the PHPDoc language + +The files in this directory describe the language `phpstan/phpdoc-parser` reads, +in the [PP3 format](https://phplrt.org/docs/basics/grammar) the +[phplrt](https://phplrt.org) compiler reads. They are the specification of the +language, and they are what `tests/PHPStan/Parser/FuzzyTest.php` writes its +corpus from. + +| File | What it holds | +|---------------------|-----------------------------------------------------------------| +| `lexemes.pp3` | Every token the language is read into | +| `common.pp3` | What the grammars share: names, brackets, line breaks | +| `types.pp3` | The type language of `TypeParser` | +| `const-expr.pp3` | The constant expressions of `ConstExprParser` | +| `phpdoc-block.pp3` | The PHPDoc itself: its tags, its text, its Doctrine annotations | +| `type.pp3` | The entry point starting at `Type` | +| `constant-expr.pp3` | The entry point starting at `ConstantExpr` | +| `phpdoc.pp3` | The entry point starting at `PhpDoc` | + +## What they are for + +A grammar says what a PHPDoc may be written as, so it can be walked the other +way round and asked for PHPDocs instead of being asked about one: + + make grammars-install # the toolchain, which asks for PHP 8.4 + php vendor/bin/phpunit --filter FuzzyTest + +`FuzzyTest` runs `tools/phplrt/fuzz.php` itself, once per grammar, every time it +runs: the tool compiles a grammar, walks its rules at random and writes down +what comes out, and the test then asks the parser to read every one of them in +full, and to read a type back as the very same type once it has been printed. +There is no corpus to keep, and none to refresh when a rule changes — what is +generated changes the moment the grammar is read again. The inputs of the last +run are left in `temp/fuzzy` to be looked at. + +That is a great deal more of the language than a hand-written corpus covers, and +it is what replaced the `abnfgen`-driven fuzzer this project used before. + +Nothing in `src/` reads these files, and the library needs neither the toolchain +nor PHP 8.4: where either is missing, `FuzzyTest` skips itself. Only the +`Grammars` job of `.github/workflows/build.yml` runs it for real. + +## How they are written + +The rules are named after the methods of `PhpDocParser`, `TypeParser` and +`ConstExprParser` they stand for and are written in the order those methods try +things in, so that a grammar and the parser it describes can be read side by +side. + +### What the grammars describe + +**A PHPDoc that is written correctly.** The parser reads a broken one as well, +by turning whatever it cannot read into an `InvalidTagValueNode` carrying the +very error it has raised, and a grammar has no way of writing that error down. +So what a broken PHPDoc means is left to the parser, and everything the grammars +describe is something the parser has to read in full. + +Two things follow from wanting that to hold for **every** input rather than for +most of them: + +- **A place the parser raises an error at is written as something the grammar + cannot recognize.** Most of them are written as a `!` predicate forbidding + whatever the error would have been raised on. For instance a name followed by + a `<` has to go on into a generic type or into a callable, because `Foo<` is + an error rather than the type `Foo` followed by something else: + + ``` + IdentifierAtomic + : ... + | !ShapeBrace() Identifier() ! ( IdentifierSuffix() | ! ) + ; + ``` + + The same predicate is what keeps a rule from **giving back** what it has read. + `@template T of` is an error rather than a template named `T` with the + description `of`, so the bound is written as "either a bound or no `of` at + all": + + ``` + TemplateUpperBound + : Type() + | Type() + | ! ! + ; + ``` + +- **A rule reads exactly the tokens its method reads**, down to the line breaks + around it. + +### The tokens are not read by phplrt + +A grammar of this directory is not read by the lexer it declares: it is read by +the very tokens `PHPStan\PhpDocParser\Lexer\Lexer` produces, handed over by +`tools/phplrt/Fuzzer/TokenStream.php`. + +The `%token` declarations therefore name the tokens and document the language +without being what reads it. Some of them describe something the lexer never +reads as a token of its own, and `TokenStream` is what tells those apart in the +stream: + +- a word the parser compares by value (`is`, `array`, `covariant`, `static`, …) + — every one of them is still an ordinary name as well, which is why they are + all listed among the alternatives of `Identifier`; +- a tag whose value a rule of its own reads (`@param`, `@return`, …), told apart + from the tags nothing reads the value of; +- a bracket or an asterisk whose neighbouring whitespace decides what it means, + which is what tells `array{a: int}` from the type `array` followed by a brace, + and `Foo[0]` from `Foo [0]`; +- a tag a space is written before, which is what tells the `@since` of + `@author Foo @since 1.0` from the `@baz` of `@author Foo `; +- a `<` opening what the parser recognizes as an HTML tag, so that + `@return Foo
see below
` keeps meaning the type `Foo` followed by a + description. + +Telling them apart there is what lets the grammars be written without semantic +predicates, which the PP3 format has none of. + +### What is left out + +Three corners of the language are left out on purpose, because a grammar cannot +say what the parser does there. Each of them is written up where the rule that +skirts it is written: + +- a description that ends at a tag written in the middle of a line, which the + parser decides by reading the tag and looking at what it turns out to be; +- the same, on a line after the first, where the parser reads that line twice: + once as part of the description and again as whatever comes next; +- a tag whose value a rule reads, written with a parenthesis after it, where + whether the description ends there depends on whether that value can be read + at all. diff --git a/doc/grammars/common.pp3 b/doc/grammars/common.pp3 new file mode 100644 index 00000000..a0ab39ea --- /dev/null +++ b/doc/grammars/common.pp3 @@ -0,0 +1,98 @@ +/** + * ----------------------------------------------------------------------------- + * What Both Grammars Are Written Of + * ----------------------------------------------------------------------------- + */ + +/** + * A name, which every keyword is as well. + * + * The lexer reads all of them as T_IDENTIFIER and they are only told apart in + * the stream so that the rules asking for a particular word can be written + * without a semantic predicate. A word is therefore still a perfectly ordinary + * name wherever a name is what is being read. + */ +Identifier + : + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + ; + +/** + * The word "array" written in any case at all. + */ +ArrayKeyword + : + | + ; + +/** + * An asterisk, whether or not whitespace follows it. + */ +Wildcard + : + | + ; + +/** + * A "[" whether or not whitespace precedes it. + */ +SquareBracketOpen + : + | + ; + +/** + * A "{" whether or not whitespace precedes it. + */ +CurlyBracketOpen + : + | + ; + +/** + * A "<" whether or not it opens what looks like an HTML tag. + */ +AngleBracketOpen + : + | + ; + +/** + * The line breaks and the line comments a type may be written across, which is + * what "TokenIterator::skipNewLineTokensAndConsumeComments()" walks over. + * + * That method reads "T_COMMENT? (T_PHPDOC_EOL T_COMMENT?)*", and this reads the + * very same thing written the shorter way: a comment runs to the end of its + * line, so the lexer never puts two of them next to each other and never puts + * one anywhere but at the end of a line. Writing it as one repetition rather + * than as three nested rules is what makes it cheap enough to be written in as + * many places as it is. + * + * The comments are kept rather than thrown away: each of them ends up on the + * node the reading reaches first after it, the way the hand-written parser + * flushes them. + * + * This recognizes an empty input as well, so a rule written with it says + * "a line break may be written here" rather than "a line break is written + * here". + */ +Trivia + : ( | )* + ; diff --git a/doc/grammars/const-expr.pp3 b/doc/grammars/const-expr.pp3 new file mode 100644 index 00000000..827edbbf --- /dev/null +++ b/doc/grammars/const-expr.pp3 @@ -0,0 +1,118 @@ +/** + * ----------------------------------------------------------------------------- + * Constant Expressions + * ----------------------------------------------------------------------------- + * + * The literals a PHPDoc may be written of, read the way + * "PHPStan\PhpDocParser\Parser\ConstExprParser" reads them. + * + * The alternatives are written in the order that parser tries them in, and the + * places it would raise an error at are written as a rule that cannot be + * recognized, so that the two agree on what is a constant expression and what + * is not. + */ + +ConstantExpr + : ConstantFloat() + | ConstantInteger() + | ConstantString() + | ConstantTrue() + | ConstantFalse() + | ConstantNull() + | ConstantArrayLiteral() + | ConstantFetch() + | ConstantArrayShorthand() + ; + +ConstantFloat + : + ; + +ConstantInteger + : + ; + +ConstantString + : + | + ; + +ConstantTrue + : + ; + +ConstantFalse + : + ; + +ConstantNull + : + ; + +/** + * An array written the long way, like: + * - array(1, 2) + * + * The word is compared without regard to case, so "ARRAY(1, 2)" is one as + * well. A word that is not followed by a parenthesis is an error rather than a + * constant of that name, which is what keeps it out of "ConstantFetch". + */ +ConstantArrayLiteral + : ArrayKeyword() ConstantArrayItems()? + ; + +/** + * An array written the short way, like: + * - [1, 2] + */ +ConstantArrayShorthand + : SquareBracketOpen() ConstantArrayItems()? + ; + +/** + * A trailing comma is written by the very rule the separating ones are, so an + * array of a single item may still end with one. + */ +ConstantArrayItems + : ConstantArrayItem() ( ConstantArrayItem() )* ? + ; + +ConstantArrayItem + : ConstantExpr() ( ConstantExpr() | ! ) + ; + +/** + * A constant, either of a class or of none, like: + * - Foo::BAR + * - Foo::BAR_* + * - BAR + */ +ConstantFetch + : !ArrayKeyword() Identifier() ClassConstantName() + | !ArrayKeyword() Identifier() + ; + +/** + * The name of a class constant, which may be written with asterisks standing + * for any part of it, like: + * - BAR + * - BAR_* + * - *_BAR_* + * + * A name is written of names and asterisks one after another, never two of a + * kind in a row, and it ends at the first asterisk that whitespace follows. + */ +ClassConstantName + : + | ClassConstantNameAfterWildcard()? + | Identifier() ClassConstantNameAfterIdentifier()? + ; + +ClassConstantNameAfterWildcard + : Identifier() ClassConstantNameAfterIdentifier()? + ; + +ClassConstantNameAfterIdentifier + : + | ClassConstantNameAfterWildcard()? + ; diff --git a/doc/grammars/constant-expr.pp3 b/doc/grammars/constant-expr.pp3 new file mode 100644 index 00000000..ba814bcf --- /dev/null +++ b/doc/grammars/constant-expr.pp3 @@ -0,0 +1,14 @@ +/** + * ----------------------------------------------------------------------------- + * Constant Expressions + * ----------------------------------------------------------------------------- + * + * The entry point describing what + * "PHPStan\PhpDocParser\Parser\ConstExprParser" reads. + */ + +%include lexemes +%include common +%include const-expr + +%pragma root ConstantExpr diff --git a/doc/grammars/lexemes.pp3 b/doc/grammars/lexemes.pp3 new file mode 100644 index 00000000..adedc4de --- /dev/null +++ b/doc/grammars/lexemes.pp3 @@ -0,0 +1,218 @@ +/** + * ----------------------------------------------------------------------------- + * The Lexemes Of The PHPDoc Language + * ----------------------------------------------------------------------------- + * + * Every token the PHPDoc language is read into, written the way + * "PHPStan\PhpDocParser\Lexer\Lexer" reads it. + * + * A grammar of this directory is not read by the lexer it declares: it is read + * by the very tokens "Lexer" produces, handed over by + * "tools/phplrt/Fuzzer/TokenStream.php". The declarations below therefore + * serve three purposes: + * + * - they name the tokens the rules are written in terms of; + * - they document the language down to its lexical level; + * - they are what the grammar-driven fuzzer writes its inputs from. + * + * A block of tokens marked "classified" is not read as a token of its own by + * the lexer. Such a token is a T_IDENTIFIER (or a bracket, or an asterisk) + * whose surroundings or whose very value the hand-written parser asks about, + * and telling those apart in the token stream is what lets a grammar written + * without semantic predicates say the same thing. + */ + +%pragma lexer.pcre.disable u +%pragma lexer.pcre.flag i + +// ----------------------------------------------------------------------------- +// Trivia +// ----------------------------------------------------------------------------- + +/** + * The whitespace "TokenIterator" walks over without ever reporting it. + * + * A place that cares whether it is there asks about it through one of the + * classified tokens below rather than through the token itself. + */ +%skip T_HORIZONTAL_WS [\x09\x20]++ + +// ----------------------------------------------------------------------------- +// Classified identifiers +// ----------------------------------------------------------------------------- + +/** + * The words the hand-written parser compares a T_IDENTIFIER against by value. + * + * Every one of them is a perfectly ordinary type name as well, so each is + * listed among the alternatives of "Identifier" and means a keyword only where + * a rule asks for it by name. + */ +%token T_KEYWORD_CONTRAVARIANT contravariant(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_COVARIANT covariant(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_NON_EMPTY_ARRAY non-empty-array(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_NON_EMPTY_LIST non-empty-list(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_SUPER super(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_OBJECT object(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_ARRAY array(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_LIST list(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_STATIC static(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_FROM from(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_NOT not(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_IS is(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_OF of(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_AS as(?![0-9a-z_\x80-\xFF-]) + +/** + * The words the constant expression parser compares against case-insensitively, + * which is why they are told apart from the ones above. + * + * "array" is spelled by both: a shape is only a shape when the word is written + * exactly as "array", while "ARRAY(1,2)" is still an array literal. + */ +%token T_KEYWORD_FALSE false(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_TRUE true(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_NULL null(?![0-9a-z_\x80-\xFF-]) +%token T_KEYWORD_ARRAY_ANY_CASE (?!array)[aA][rR][rR][aA][yY](?![0-9a-z_\x80-\xFF-]) + +// ----------------------------------------------------------------------------- +// Literals and names +// ----------------------------------------------------------------------------- + +%token T_FLOAT [+\-]?(?:(?:[0-9]++(_[0-9]++)*\.[0-9]*+(_[0-9]++)*(?:e[+\-]?[0-9]++(_[0-9]++)*)?)|(?:[0-9]*+(_[0-9]++)*\.[0-9]++(_[0-9]++)*(?:e[+\-]?[0-9]++(_[0-9]++)*)?)|(?:[0-9]++(_[0-9]++)*e[+\-]?[0-9]++(_[0-9]++)*)) +%token T_INTEGER [+\-]?(?:(?:0b[0-1]++(_[0-1]++)*)|(?:0o[0-7]++(_[0-7]++)*)|(?:0x[0-9a-f]++(_[0-9a-f]++)*)|(?:[0-9]++(_[0-9]++)*)) +%token T_SINGLE_QUOTED_STRING '(?:\\[^\r\n]|[^'\r\n\\])*+' +%token T_DOUBLE_QUOTED_STRING "(?:\\[^\r\n]|[^"\r\n\\])*+" +%token T_DOCTRINE_ANNOTATION_STRING "(?:""|[^"])*+" + +%token T_IDENTIFIER (?:[\\]?+[a-z_\x80-\xFF][0-9a-z_\x80-\xFF-]*+)++ +%token T_THIS_VARIABLE \$this(?![0-9a-z_\x80-\xFF]) +%token T_VARIABLE \$[a-z_\x80-\xFF][0-9a-z_\x80-\xFF]*+ + +// ----------------------------------------------------------------------------- +// Punctuation +// ----------------------------------------------------------------------------- + +/** + * An "&" standing before a variable, a variadic, a default value or a closing + * parenthesis is a by-reference marker rather than an intersection. + */ +%token T_REFERENCE &(?=\s*+(?:[,=)]|\.\.\.|(?:\$(?!this(?![0-9a-z_\x80-\xFF]))))) +%token T_UNION \| +%token T_INTERSECTION & +%token T_NULLABLE \? +%token T_NEGATED ! + +%token T_OPEN_PARENTHESES \( +%token T_CLOSE_PARENTHESES \) +%token T_OPEN_ANGLE_BRACKET < +%token T_CLOSE_ANGLE_BRACKET > +%token T_OPEN_SQUARE_BRACKET \[ +%token T_CLOSE_SQUARE_BRACKET \] +%token T_OPEN_CURLY_BRACKET \{ +%token T_CLOSE_CURLY_BRACKET \} + +%token T_COMMA , +%token T_COMMENT //[^\r\n]*(?=\n|\r|\*/) +%token T_VARIADIC \.\.\. +%token T_DOUBLE_COLON :: +%token T_DOUBLE_ARROW => +%token T_ARROW -> +%token T_EQUAL = +%token T_COLON : +%token T_WILDCARD \* + +// ----------------------------------------------------------------------------- +// PHPDoc structure +// ----------------------------------------------------------------------------- + +%token T_OPEN_PHPDOC /\*\*(?=\s)\x20?+ +%token T_CLOSE_PHPDOC \*/ + +/** + * The tags whose value is read by a rule of its own, told apart from one + * another so that a rule can ask for the one it reads. + * + * The lexer reads every one of them as T_PHPDOC_TAG and they are only told + * apart in the stream, the way a keyword is: a tag is compared by its whole + * name and by the case it is written in, so "@Param" names no tag at all and is + * read as any other unknown one. + */ +%token T_TAG_PARAM_IMMEDIATELY_INVOKED_CALLABLE @(?:phpstan-)?param-immediately-invoked-callable(?![a-z0-9-]) +%token T_TAG_PURE_UNLESS_CALLABLE_IS_IMPURE @(?:phpstan-)?pure-unless-callable-is-impure(?![a-z0-9-]) +%token T_TAG_PURE_UNLESS_PARAMETER_PASSED @(?:phpstan-)?pure-unless-parameter-passed(?![a-z0-9-]) +%token T_TAG_PARAM_LATER_INVOKED_CALLABLE @(?:phpstan-)?param-later-invoked-callable(?![a-z0-9-]) +%token T_TAG_PARAM_CLOSURE_THIS @(?:phpstan-)?param-closure-this(?![a-z0-9-]) +%token T_TAG_REQUIRE_IMPLEMENTS @(?:psalm|phpstan)-require-implements(?![a-z0-9-]) +%token T_TAG_REQUIRE_EXTENDS @(?:psalm|phpstan)-require-extends(?![a-z0-9-]) +%token T_TAG_TYPE_ALIAS_IMPORT @(?:phpstan|psalm)-import-type(?![a-z0-9-]) +%token T_TAG_ASSERT @(?:phpstan|psalm|phan)-assert(?:-if-(?:true|false))?(?![a-z0-9-]) +%token T_TAG_SELF_OUT @(?:phpstan|psalm)-(?:this|self)-out(?![a-z0-9-]) +%token T_TAG_SEALED @(?:psalm-inheritors|phpstan-sealed)(?![a-z0-9-]) +%token T_TAG_TEMPLATE @(?:(?:phpstan|psalm|phan)-)?template(?:-(?:co|contra)variant)?(?![a-z0-9-]) +%token T_TAG_PROPERTY @(?:(?:phpstan|psalm|phan)-)?property(?:-(?:read|write))?(?![a-z0-9-]) +%token T_TAG_IMPLEMENTS @(?:phpstan-|template-)?implements(?![a-z0-9-]) +%token T_TAG_DEPRECATED @deprecated(?![a-z0-9-]) +%token T_TAG_TYPE_ALIAS @(?:phpstan|psalm|phan)-type(?![a-z0-9-]) +%token T_TAG_PARAM_OUT @(?:(?:phpstan|psalm)-)?param-out(?![a-z0-9-]) +%token T_TAG_EXTENDS @(?:(?:phpstan|phan|template)-)?(?:extends|inherits)(?![a-z0-9-]) +%token T_TAG_METHOD @(?:(?:phpstan|psalm|phan)-)?method(?![a-z0-9-]) +%token T_TAG_RETURN @(?:(?:phpstan|psalm|phan|phan-real)-)?return(?![a-z0-9-]) +%token T_TAG_THROWS @(?:phpstan-)?throws(?![a-z0-9-]) +%token T_TAG_PARAM @(?:(?:phpstan|psalm|phan)-)?param(?![a-z0-9-]) +%token T_TAG_MIXIN @(?:phan-)?mixin(?![a-z0-9-]) +%token T_TAG_USE @(?:phpstan-|template-)?use(?![a-z0-9-]) +%token T_TAG_VAR @(?:(?:phpstan|psalm|phan)-)?var(?![a-z0-9-]) + +/** + * A tag whose value nothing reads: whatever is written after it is a + * description, or the arguments of a Doctrine annotation. + * + * Whether whitespace is written before such a tag is what says whether it is a + * tag at all: a description only ever ends at one that a space stands before, + * so that the "@baz" of "@author Foo " belongs to the description + * it is written in. + */ +%token T_PHPDOC_TAG_WS (?<=[\x09\x20])@(?:[a-z][a-z0-9-\\]+:)?[a-z][a-z0-9-\\]*+ +%token T_PHPDOC_TAG @(?:[a-z][a-z0-9-\\]+:)?[a-z][a-z0-9-\\]*+ +%token T_DOCTRINE_TAG_WS (?<=[\x09\x20])@[a-z_\\][a-z0-9_:\\]*[a-z_][a-z0-9_]* +%token T_DOCTRINE_TAG @[a-z_\\][a-z0-9_:\\]*[a-z_][a-z0-9_]* +%token T_PHPDOC_EOL \r?+\n[\x09\x20]*+(?:\*(?!/)\x20?+)? + +/** + * Whatever is neither whitespace nor the end of a PHPDoc, which is what the + * free-form text of a PHPDoc is read as. + */ +%token T_OTHER (?:(?!\*/)[^\s])++ + +// ----------------------------------------------------------------------------- +// Classified punctuation +// ----------------------------------------------------------------------------- + +/** + * The brackets and the asterisk whose neighbouring whitespace decides what they + * mean, told apart in the stream so that a rule can ask for one of them alone. + * + * - "array{a: int}" is a shape while "array {a: int}" is the type "array" + * followed by something the type says nothing about; + * - "Foo[0]" reads an offset while "Foo [0]" does not; + * - "Foo::BAR*" goes on reading the name of the constant while "Foo::BAR* " + * ends it. + */ +%token T_OPEN_CURLY_BRACKET_WS (?<=[\x09\x20])\{ +%token T_OPEN_SQUARE_BRACKET_WS (?<=[\x09\x20])\[ +%token T_WILDCARD_WS \*(?=[\x09\x20]) + +/** + * A "<" opening what the hand-written parser recognizes as an HTML tag, that is + * a "" with a matching "" somewhere after it. A generic type is + * never read out of one, so that "@return Foo
see below
" keeps meaning + * the type "Foo" followed by a description. + */ +%token T_OPEN_ANGLE_BRACKET_HTML <(?=[a-z][a-z0-9-]*+>) + +/** + * The end of the input is not declared here: a lexer of the phplrt runtime + * reports one of its own, which is what a stream ends with no matter who has + * written it. + */ diff --git a/doc/grammars/phpdoc-block.pp3 b/doc/grammars/phpdoc-block.pp3 new file mode 100644 index 00000000..48eacb6a --- /dev/null +++ b/doc/grammars/phpdoc-block.pp3 @@ -0,0 +1,684 @@ +/** + * ----------------------------------------------------------------------------- + * The PHPDoc Itself + * ----------------------------------------------------------------------------- + * + * Everything "PHPStan\PhpDocParser\Parser\PhpDocParser" reads: the shape of a + * PHPDoc, the tags it is written of and the free-form text between them. + * + * What this grammar describes is a PHPDoc that is written correctly. The + * parser reads a broken one as well, by turning whatever it cannot read into + * an "InvalidTagValueNode" carrying the very error it has raised, and a + * grammar has no way of writing that error down. So what a broken PHPDoc means + * is left to the parser, and everything written here is something the parser + * has to read in full. + */ + +/** + * A PHPDoc, like: + * - / ** @param Foo $a The first one * / + * + * The children are separated by line breaks, except after a tag nothing reads + * the value of: whatever follows such a tag has already been read as its + * description, up to the line break the description stops at. + */ +PhpDoc + : ? ( ! PhpDocChildren() )? + ; + +PhpDocChildren + : UnreadTagChild() ? MorePhpDocChildren()? + | PhpDocChild() ( MorePhpDocChildren()? )? + ; + +MorePhpDocChildren + : ! PhpDocChildren() + ; + +/** + * A tag whose value nothing reads, which is what makes the line break after it + * optional: its description has already been read up to wherever it ends. + */ +UnreadTagChild + : GenericTag() + | DoctrineTag() + ; + +/** + * A child of a PHPDoc that is not a tag nothing reads the value of. + * + * Free-form text is only read where no tag is written: a tag whose value cannot + * be read is an error rather than a line of text that happens to begin with an + * "@", which is what the predicate says. + */ +PhpDocChild + : KnownTag() + | !TagToken() TextChild() + ; + +/** + * The free-form text between the tags of a PHPDoc. + */ +TextChild + : Text() + ; + +/** + * A tag named by nothing this grammar knows, like: + * - @author John Doe + * + * Its value is whatever is written after it, read as a description that stops + * at the first tag rather than reading it as part of the text. + */ +GenericTag + : GenericTagToken() ! DoctrineDescription() + ; + +GenericTagToken + : + | + ; + +DoctrineTagToken + : + | + ; + +// ----------------------------------------------------------------------------- +// The tags whose value is read +// ----------------------------------------------------------------------------- + +/** + * Every tag this grammar knows the value of. + * + * The value of each of them is read by a rule of its own and the tag itself + * says which: which alternative is worth entering is decided by the very first + * token, so nothing is ever read twice. + */ +KnownTag + : ParamTagValue() + | VarTagValue() + | TypeAndDescription() + | TypeAndDescription() + | TypeAndDescription() + | TypeAndDescription() + | TypeAndDescription() + | TypeAndDescription() + | TypeAndDescription() + | TypeAndVariableName() + | TypeAndVariableName() + | TypeAndVariableName() + | VariableNameAndDescription() + | VariableNameAndDescription() + | VariableNameAndDescription() + | VariableNameAndDescription() + | Description() + | MethodTagValue() + | TemplateTagValue() + | ExtendsTagValue() + | ExtendsTagValue() + | ExtendsTagValue() + | TypeAliasTagValue() + | TypeAliasImportTagValue() + | AssertTagValue() + ; + +/** + * A type followed by a description, which is what most of the tags are written + * of: @return, @throws, @mixin and the rest of them. + */ +TypeAndDescription + : Type() DescriptionAfterType() + ; + +/** + * A type, the name of what it belongs to and a description, like: + * - @property Foo $bar The one that matters + */ +TypeAndVariableName + : Type() Description() + ; + +VariableNameAndDescription + : Description() + ; + +/** + * A parameter, which may be written with no type at all, like: + * - @param Foo $a + * - @param &...$a + */ +ParamTagValue + : &ParameterNameStart() ? ? Description() + | Type() ? ? Description() + ; + +ParameterNameStart + : + | + | + ; + +/** + * A type and the variable it belongs to, of which the variable may be left out. + * + * A description written with no variable before it has to be written after a + * space, which is what tells "@var Foo the one" from "@var Foo|the one". + */ +VarTagValue + : Type() VariableName() Description() + | Type() DescriptionAfterType() + ; + +VariableName + : + | + ; + +// ----------------------------------------------------------------------------- +// @method +// ----------------------------------------------------------------------------- + +/** + * A method, like: + * - @method static Foo bar(int $a = 1) + * - @method bar() + * + * The name of a method is read as a type first and is only told from a return + * type by what follows it, which is why a type is written where a name is + * meant. + */ +MethodTagValue + : StaticType() Type() Identifier() MethodTagValueTail() + | StaticType() Type() MethodTagValueTail() + | Type() Identifier() MethodTagValueTail() + | Type() MethodTagValueTail() + ; + +/** + * The word "static" written as a whole type of its own, which is what tells + * "@method static Foo bar()" from "@method static[] bar()". + */ +StaticType + : !StaticTypeContinues() + ; + +StaticTypeContinues + : Trivia() ( | ) + | + | + | + | SquareBracketOpen() + ; + +MethodTagValueTail + : ( MethodTemplates() | !AngleBracketOpen() ) + MethodParameters()? Description() + ; + +MethodTemplates + : AngleBracketOpen() CallableTemplateArgument() ( CallableTemplateArgument() )* + ; + +MethodParameters + : MethodParameter() ( MethodParameter() )* + ; + +MethodParameter + : &MethodParameterTypeStart() Type() ? ? MethodParameterDefault() + | ? ? MethodParameterDefault() + ; + +/** + * A parameter is only read with a type when one of these is written first, + * which is what keeps "&$a" from being read as a type. + */ +MethodParameterTypeStart + : Identifier() + | + | + ; + +MethodParameterDefault + : ConstantExpr() + | ! + ; + +// ----------------------------------------------------------------------------- +// @template, @extends, @phpstan-type and friends +// ----------------------------------------------------------------------------- + +TemplateTagValue + : Identifier() TemplateUpperBound() TemplateLowerBound() TemplateDefault() DescriptionAfterType() + ; + +/** + * What a class extends, implements or uses, which is always a generic type: + * - @extends Collection + */ +ExtendsTagValue + : Identifier() GenericInReturnType() DescriptionAfterType() + ; + +/** + * A type written under a name of its own, like: + * - @phpstan-type Foo = int|string + * + * Nothing may be written after the type but the end of its line, so that a + * description is never read as part of the type it follows. + */ +TypeAliasTagValue + : Identifier() ? Type() &EndOfLine() + ; + +/** + * A type read out of another PHPDoc, like: + * - @phpstan-import-type Foo from Bar as Baz + */ +TypeAliasImportTagValue + : Identifier() Identifier() ( Identifier() | ! ) + ; + +/** + * What a method says about its arguments once it gives back, like: + * - @phpstan-assert !Foo $a + * - @phpstan-assert Foo $a->bar + * - @phpstan-assert Foo $a->bar() + */ +AssertTagValue + : ? ? Type() AssertParameter() Description() + ; + +AssertParameter + : AssertParameterName() ( Identifier() AssertMethodParentheses() | ! ) + ; + +AssertMethodParentheses + : + | ! + ; + +AssertParameterName + : + | + ; + +// ----------------------------------------------------------------------------- +// Descriptions +// ----------------------------------------------------------------------------- + +/** + * Whatever is written after the value of a tag. + */ +Description + : Text() + ; + +/** + * The same, written right after a type. + * + * Such a description has to begin after a space and may not begin with a "|" or + * a "&", so that "@return Foo|null" is one type rather than a type followed by + * something else. Which of the two is written is decided by the reducer, which + * turns the tag down where the hand-written parser raises an error. + */ +DescriptionAfterType + : Text() + ; + +/** + * The description of a tag nothing reads the value of. + * + * It stops at the first tag rather than reading it as text, so that + * "@author John\n@since 1.0" is two tags. A tag written in the middle of a line + * is read by looking at what the tag turns out to mean, which a grammar cannot + * do, so a description that would stop at one is left out of this grammar. + */ +DoctrineDescription + : DoctrineText() + ; + +/** + * The text of a description or of a PHPDoc, read the way + * PhpDocParser::parseText() reads it. + * + * The reading goes past a line break as long as something is written on the + * next line, and stops before the line break that follows the last line with + * anything on it: a blank line written in the middle of a description belongs + * to it, while one written after it does not. + */ +Text + : TextLine() TextTail()? + ; + +TextTail + : !TextStop() TextLineWithSomethingOnIt() TextTail()? + | !TextStop() TextTail() + ; + +TextLine + : TextToken()* + ; + +TextLineWithSomethingOnIt + : TextToken()+ + ; + +/** + * Where the reading of a description stops for good: another tag, or the end of + * the PHPDoc. + */ +TextStop + : TagToken() + | + ; + +EndOfLine + : + | + ; + +/** + * A tag whose value a rule of its own reads. + */ +KnownTagToken + : + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + ; + +DoctrineText + : DoctrineTextLine() DoctrineTextRest() + ; + +/** + * Whatever a description of this kind goes on with once its first line has been + * read. + * + * It ends at the end of the line, and at a tag nothing reads the value of. A + * tag written with no space before it is no tag at all here, and neither is one + * whose value something does read, so the description goes on reading them the + * way any other text is read. + */ +/** + * Whatever the description goes on with once a line of it has been read. + * + * A description ends at the end of its line and at a tag a space is written + * before. The second of the two is where the parser reads part of the PHPDoc + * twice: the line the tag is written on belongs to the description and is then + * read again as whatever comes next. A grammar describes each token once, so a + * description that would end that way is left out of this one. + */ +DoctrineTextRest + : &EndOfLine() DoctrineTextMoreLines() + | &TagToken() !SpacedTag() !AmbiguousTagEnd() Text() + ; + +/** + * A tag a space is written before, which is what ends a description of this + * kind. + */ +SpacedTag + : + | + ; + +/** + * Whatever the description goes on with once the line it is on has ended. + * + * A description that ends at a tag written in the middle of a line is read + * twice by the hand-written parser: the line the tag is on belongs to the + * description, and is then read again as whatever comes next. Nothing a grammar + * builds can be read twice, so a description that would end that way is turned + * down and the PHPDoc is read by that parser instead. + */ +DoctrineTextMoreLines + : DoctrineTextTail() + | !MoreTextToRead() + ; + +/** + * Whether anything worth reading is written on the lines that follow. + * + * A description runs on past the blank lines written inside it and ends at the + * first line that begins with a tag, so this says the same thing the other way + * round: a description that ends while there is still something to read has + * ended somewhere this grammar does not describe. + */ +MoreTextToRead + : !TextStop() MoreTextToRead() + | !TextStop() TextToken() + ; + +/** + * A tag whose value something reads, written with a parenthesis after it. + * + * Whether such a tag ends the description depends on whether its value can be + * read at all, which is decided by reading it and looking at what it turns out + * to be. A grammar cannot do that, so a description that would stop at one is + * left out of this grammar. + */ +AmbiguousTagEnd + : KnownTagToken() + ; + +DoctrineTextTail + : !TextStop() DoctrineTextLineWithSomethingOnIt() DoctrineTextRest() + | !TextStop() DoctrineTextTail() + ; + +DoctrineTextLine + : NonTagTextToken()* + ; + +DoctrineTextLineWithSomethingOnIt + : NonTagTextToken()+ + ; + +/** + * Anything a line of a description may be written of, which is anything at all + * apart from what ends the line. + */ +TextToken + : NonTagTextToken() + | TagToken() + ; + +TagToken + : GenericTagToken() + | DoctrineTagToken() + | KnownTagToken() + ; + +NonTagTextToken + : Identifier() + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + ; + +// ----------------------------------------------------------------------------- +// Doctrine annotations +// ----------------------------------------------------------------------------- + +/** + * An annotation of the kind Doctrine reads, like: + * - @ORM\Entity(repositoryClass="FooRepository") + * - @Assert\NotBlank + * + * A tag written with a name no ordinary tag may be written with is always one + * of these; any other tag is one as soon as a parenthesis follows it. + * + * The line breaks written inside the arguments are walked over the way the + * whitespace is, which is why they may be written between any two things there. + * + * A parenthesis written after the tag is always the one its arguments open, + * never the first thing of a description: "@\Foo(" that is never closed is an + * error rather than a tag with no arguments at all, which is what the + * predicate says. + */ +DoctrineTag + : DoctrineTagToken() DoctrineArguments() DoctrineDescription() + | DoctrineTagToken() ! DoctrineDescription() + | GenericTagToken() DoctrineArguments() DoctrineDescription() + ; + +DoctrineArguments + : Eols() DoctrineArgumentList()? + ; + +DoctrineArgumentList + : DoctrineArgument() Eols() ( Eols() ( & | DoctrineArgument() Eols() ) )* + ; + +DoctrineArgument + : Identifier() Eols() Eols() DoctrineValue() + | DoctrineValue() + ; + +/** + * What an argument of an annotation may be written as. + * + * A name written with no "::" after it is a type rather than a constant, which + * is what makes "@Foo(bar=true)" say the name "true" instead of saying yes. + */ +DoctrineValue + : DoctrineNestedAnnotation() + | DoctrineArray() + | DoctrineIdentifier() + | DoctrineConstantFetch() + | DoctrineFloat() + | DoctrineInteger() + | DoctrineString() + ; + +DoctrineNestedAnnotation + : TagToken() DoctrineArguments()? + ; + +DoctrineArray + : CurlyBracketOpen() Eols() DoctrineArrayItemList()? + ; + +DoctrineArrayItemList + : DoctrineArrayItem() Eols() ( Eols() ( & | DoctrineArrayItem() Eols() ) )* + ; + +DoctrineArrayItem + : DoctrineArrayKey() Eols() ( | ) Eols() DoctrineValue() + | DoctrineValue() + ; + +DoctrineArrayKey + : + | + | DoctrineString() + | DoctrineConstantFetch() + | Identifier() ! + ; + +DoctrineIdentifier + : Identifier() ! + ; + +/** + * A constant of a class, like: + * - Foo::BAR + * + * A word that already means something to a constant expression is none of + * these: "null::BAR" says "null" and leaves the rest of it unread, which is an + * error rather than the name of a constant. + */ +DoctrineConstantFetch + : !DoctrineConstantWord() Identifier() ClassConstantName() + ; + +DoctrineConstantWord + : + | + | + | ArrayKeyword() + ; + +DoctrineFloat + : + ; + +DoctrineInteger + : + ; + +/** + * A string of a Doctrine annotation, which is written with doubled quotes + * instead of escapes. + * + * A string written across several lines is read as several tokens, because a + * quote written inside it opens one string and closes another as far as the + * lexer is concerned, so whatever follows one is read as part of it. + */ +DoctrineString + : ( | )* + | + ; + +/** + * The line breaks written inside the arguments of an annotation, which the + * reading walks over the way it walks over the whitespace. + */ +Eols + : * + ; diff --git a/doc/grammars/phpdoc-method.peg b/doc/grammars/phpdoc-method.peg deleted file mode 100644 index 3d16942f..00000000 --- a/doc/grammars/phpdoc-method.peg +++ /dev/null @@ -1,41 +0,0 @@ -PhpDocMethod - = AnnotationName IsStatic? MethodReturnType? MethodName MethodParameters? Description? - -AnnotationName - = '@method' - -IsStatic - = 'static' - -MethodReturnType - = Type - -MethodName - = [a-zA-Z_\127-\255][a-zA-Z0-9_\127-\255]* - -MethodParameters - = '(' MethodParametersInner? ')' - -MethodParametersInner - = MethodParameter (',' MethodParameter)* - -MethodParameter - = MethodParameterType? IsReference? IsVariadic? MethodParameterName MethodParameterDefaultValue? - -MethodParameterType - = Type - -IsReference - = '&' - -IsVariadic - = '...' - -MethodParameterName - = '$' [a-zA-Z_\127-\255][a-zA-Z0-9_\127-\255]* - -MethodParameterDefaultValue - = '=' PhpConstantExpr - -Description - = .+ # TODO: exclude EOL or another PhpDocTag start diff --git a/doc/grammars/phpdoc-param.peg b/doc/grammars/phpdoc-param.peg deleted file mode 100644 index f16cb948..00000000 --- a/doc/grammars/phpdoc-param.peg +++ /dev/null @@ -1,17 +0,0 @@ -PhpDocParam - = AnnotationName Type? IsReference? IsVariadic? ParameterName Description? - -AnnotationName - = '@param' - -IsReference - = '&' - -IsVariadic - = '...' - -ParameterName - = '$' [a-zA-Z_\127-\255][a-zA-Z0-9_\127-\255]* - -Description - = .+ # TODO: exclude EOL or another PhpDocTag start diff --git a/doc/grammars/phpdoc.pp3 b/doc/grammars/phpdoc.pp3 new file mode 100644 index 00000000..9018e67b --- /dev/null +++ b/doc/grammars/phpdoc.pp3 @@ -0,0 +1,16 @@ +/** + * ----------------------------------------------------------------------------- + * PHPDoc + * ----------------------------------------------------------------------------- + * + * The entry point describing what "PHPStan\PhpDocParser\Parser\PhpDocParser" + * reads. See "phpdoc-block.pp3" for the rules. + */ + +%include lexemes +%include common +%include const-expr +%include types +%include phpdoc-block + +%pragma root PhpDoc diff --git a/doc/grammars/type.abnf b/doc/grammars/type.abnf deleted file mode 100644 index ad955a9b..00000000 --- a/doc/grammars/type.abnf +++ /dev/null @@ -1,282 +0,0 @@ -; ---------------------------------------------------------------------------- ; -; Type ; -; ---------------------------------------------------------------------------- ; - -Type - = Atomic [Union / Intersection] - / Nullable - -ParenthesizedType - = Atomic [Union / Intersection / Conditional] - / Nullable - -Union - = 1*(TokenUnion Atomic) - -Intersection - = 1*(TokenIntersection Atomic) - -Conditional - = 1*ByteHorizontalWs TokenIs [TokenNot] Atomic TokenNullable Type TokenColon ParenthesizedType - -Nullable - = TokenNullable Atomic - -Atomic - = TokenIdentifier [Generic / Callable / Array] - / TokenThisVariable - / TokenParenthesesOpen ParenthesizedType TokenParenthesesClose [Array] - -Generic - = TokenAngleBracketOpen GenericTypeArgument *(TokenComma GenericTypeArgument) TokenAngleBracketClose - -GenericTypeArgument - = [TokenContravariant / TokenCovariant] Type - / TokenWildcard - -Callable - = [CallableTemplate] TokenParenthesesOpen [CallableParameters] TokenParenthesesClose TokenColon CallableReturnType - -CallableTemplate - = TokenAngleBracketOpen CallableTemplateArgument *(TokenComma CallableTemplateArgument) TokenAngleBracketClose - -CallableTemplateArgument - = TokenIdentifier [1*ByteHorizontalWs TokenOf Type] [1*ByteHorizontalWs TokenSuper Type] ["=" Type] - -CallableParameters - = CallableParameter *(TokenComma CallableParameter) - -CallableParameter - = Type [CallableParameterIsReference] [CallableParameterIsVariadic] [CallableParameterName] [CallableParameterIsOptional] - -CallableParameterIsReference - = TokenIntersection - -CallableParameterIsVariadic - = TokenVariadic - -CallableParameterName - = TokenVariable - -CallableParameterIsOptional - = TokenEqualSign - -CallableReturnType - = TokenIdentifier [Generic] - / Nullable - / TokenParenthesesOpen Type TokenParenthesesClose - -Array - = 1*(TokenSquareBracketOpen TokenSquareBracketClose) - -ArrayShape - = TokenCurlyBracketOpen ArrayShapeItem *(TokenComma ArrayShapeItem) TokenCurlyBracketClose - -ArrayShapeItem - = (ConstantString / ConstantInt / TokenIdentifier) TokenNullable TokenColon Type - / Type - -; ---------------------------------------------------------------------------- ; -; ConstantExpr ; -; ---------------------------------------------------------------------------- ; - -ConstantExpr - = ConstantFloat *ByteHorizontalWs - / ConstantInt *ByteHorizontalWs - / ConstantTrue *ByteHorizontalWs - / ConstantFalse *ByteHorizontalWs - / ConstantNull *ByteHorizontalWs - / ConstantString *ByteHorizontalWs - / ConstantArray *ByteHorizontalWs - / ConstantFetch *ByteHorizontalWs - -ConstantFloat - = [ByteNumberSign] 1*ByteDecDigit *("_" 1*ByteDecDigit) "." [1*ByteDecDigit *("_" 1*ByteDecDigit)] [ConstantFloatExp] - / [ByteNumberSign] 1*ByteDecDigit *("_" 1*ByteDecDigit) ConstantFloatExp - / [ByteNumberSign] "." 1*ByteDecDigit *("_" 1*ByteDecDigit) [ConstantFloatExp] - -ConstantFloatExp - = "e" [ByteNumberSign] 1*ByteDecDigit *("_" 1*ByteDecDigit) - -ConstantInt - = [ByteNumberSign] "0b" 1*ByteBinDigit *("_" 1*ByteBinDigit) - / [ByteNumberSign] "0o" 1*ByteOctDigit *("_" 1*ByteOctDigit) - / [ByteNumberSign] "0x" 1*ByteHexDigit *("_" 1*ByteHexDigit) - / [ByteNumberSign] 1*ByteDecDigit *("_" 1*ByteDecDigit) - -ConstantTrue - = "true" - -ConstantFalse - = "false" - -ConstantNull - = "null" - -ConstantString - = ByteSingleQuote *(ByteBackslash ByteNotEol / ByteNotEolAndNotBackslashAndNotSingleQuote) ByteSingleQuote - / ByteDoubleQuote *(ByteBackslash ByteNotEol / ByteNotEolAndNotBackslashAndNotDoubleQuote) ByteDoubleQuote - -ConstantArray - = TokenSquareBracketOpen [ConstantArrayItems] TokenSquareBracketClose - / "array" TokenParenthesesOpen [ConstantArrayItems] TokenParenthesesClose - -ConstantArrayItems - = ConstantArrayItem *(TokenComma ConstantArrayItem) [TokenComma] - -ConstantArrayItem - = ConstantExpr [TokenDoubleArrow ConstantExpr] - -ConstantFetch - = TokenIdentifier [TokenDoubleColon ByteIdentifierFirst *ByteIdentifierSecond *ByteHorizontalWs] - - -; ---------------------------------------------------------------------------- ; -; Tokens ; -; ---------------------------------------------------------------------------- ; - -TokenUnion - = "|" *ByteHorizontalWs - -TokenIntersection - = "&" *ByteHorizontalWs - -TokenNullable - = "?" *ByteHorizontalWs - -TokenParenthesesOpen - = "(" *ByteHorizontalWs - -TokenParenthesesClose - = ")" *ByteHorizontalWs - -TokenAngleBracketOpen - = "<" *ByteHorizontalWs - -TokenAngleBracketClose - = ">" *ByteHorizontalWs - -TokenSquareBracketOpen - = "[" *ByteHorizontalWs - -TokenSquareBracketClose - = "]" *ByteHorizontalWs - -TokenCurlyBracketOpen - = "{" *ByteHorizontalWs - -TokenCurlyBracketClose - = "}" *ByteHorizontalWs - -TokenComma - = "," *ByteHorizontalWs - -TokenColon - = ":" *ByteHorizontalWs - -TokenVariadic - = "..." *ByteHorizontalWs - -TokenEqualSign - = "=" *ByteHorizontalWs - -TokenVariable - = "$" ByteIdentifierFirst *ByteIdentifierSecond *ByteHorizontalWs - -TokenDoubleArrow - = "=>" *ByteHorizontalWs - -TokenDoubleColon - = "::" *ByteHorizontalWs - -TokenThisVariable - = %s"$this" *ByteHorizontalWs - -TokenIs - = %s"is" 1*ByteHorizontalWs - -TokenNot - = %s"not" 1*ByteHorizontalWs - -TokenOf - = %s"of" 1*ByteHorizontalWs - -TokenSuper - = %s"super" 1*ByteHorizontalWs - -TokenContravariant - = %s"contravariant" 1*ByteHorizontalWs - -TokenCovariant - = %s"covariant" 1*ByteHorizontalWs - -TokenWildcard - = "*" *ByteHorizontalWs - -TokenIdentifier - = [ByteBackslash] ByteIdentifierFirst *ByteIdentifierSecond *(ByteBackslash ByteIdentifierFirst *ByteIdentifierSecond) *ByteHorizontalWs - - -; ---------------------------------------------------------------------------- ; -; Bytes ; -; ---------------------------------------------------------------------------- ; - -ByteHorizontalWs - = %x09 ; horizontal tab - / " " - -ByteNumberSign - = "+" - / "-" - -ByteBinDigit - = %x30-31 ; 0-1 - -ByteOctDigit - = %x30-37 ; 0-7 - -ByteDecDigit - = %x30-39 ; 0-9 - -ByteHexDigit - = %x30-39 ; 0-9 - / %x41-46 ; A-F - / %x61-66 ; a-f - -ByteIdentifierFirst - = %x41-5A ; A-Z - / "_" - / %x61-7A ; a-z - / %x80-FF - -ByteIdentifierSecond - = ByteIdentifierFirst - / %x30-39 ; 0-9 - -ByteSingleQuote - = %x27 ; ' - -ByteDoubleQuote - = %x22 ; " - -ByteBackslash - = %x5C ; \ - -ByteNotEol - = %x00-09 ; skip LF - / %x0B-0C ; skip CR - / %x0E-FF - -ByteNotEolAndNotBackslashAndNotSingleQuote - = %x00-09 ; skip LF - / %x0B-0C ; skip CR - / %x0E-26 ; skip single quote - / %x28-5B ; skip backslash - / %x5D-FF - -ByteNotEolAndNotBackslashAndNotDoubleQuote - = %x00-09 ; skip LF - / %x0B-0C ; skip CR - / %x0E-21 ; skip double quote - / %x23-5B ; skip backslash - / %x5D-FF diff --git a/doc/grammars/type.pp3 b/doc/grammars/type.pp3 new file mode 100644 index 00000000..3ecf779c --- /dev/null +++ b/doc/grammars/type.pp3 @@ -0,0 +1,16 @@ +/** + * ----------------------------------------------------------------------------- + * The PHPDoc Type Language + * ----------------------------------------------------------------------------- + * + * The entry point describing what "PHPStan\PhpDocParser\Parser\TypeParser" + * reads. See "types.pp3" for the rules and "lexemes.pp3" for the tokens they + * are written in terms of. + */ + +%include lexemes +%include common +%include const-expr +%include types + +%pragma root Type diff --git a/doc/grammars/types.pp3 b/doc/grammars/types.pp3 new file mode 100644 index 00000000..bf5153d1 --- /dev/null +++ b/doc/grammars/types.pp3 @@ -0,0 +1,474 @@ +/** + * ----------------------------------------------------------------------------- + * The PHPDoc Type Language + * ----------------------------------------------------------------------------- + * + * Every type "PHPStan\PhpDocParser\Parser\TypeParser" reads, written as the + * grammar it reads them by. + * + * The rules are named after the methods of that parser and are written in the + * order those methods try things in, so that the two can be read side by side. + * Two things follow from wanting them to agree on every input rather than on + * most of them: + * + * - a place the parser raises an error at is written as something the grammar + * cannot recognize, most often as a "!" predicate forbidding what the error + * would have been raised on, so that everything this grammar describes is + * something the parser reads in full; + * + * - a rule reads exactly the tokens its method reads, down to the line breaks + * around it, because the tokens a rule has read are what the start and the + * end of a node are counted from. + */ + +/** + * A type, which is what a PHPDoc tag is written of. + * + * A union or an intersection may be written across several lines, and the + * reading only goes past a line break when the next line goes on with the type: + * "int\n|string" is one type while "int\nstring" is a type followed by a + * description. Whichever of the two is written, a "|" or a "&" left unread is + * an error rather than the end of the type, which is what the predicates say. + */ +Type + : Nullable() + | Atomic() ( TypeTail() | ! ! ) + ; + +TypeTail + : Trivia() ( Union() | Intersection() ) + ; + +Union + : ( Atomic() ( Trivia() & )? )+ ! + ; + +Intersection + : ( Atomic() ( Trivia() & )? )+ ! + ; + +Nullable + : Atomic() + ; + +/** + * A type written inside parentheses, which is the only place a conditional type + * may be written and the only place a line break needs no "|" after it. + */ +SubType + : Nullable() + | ConditionalForParameter() + | Atomic() ( Conditional() | SubTypeTail() ) + ; + +SubTypeTail + : Trivia() ( SubUnion() | SubIntersection() )? + ; + +SubUnion + : ( Trivia() Atomic() Trivia() )+ ! + ; + +SubIntersection + : ( Trivia() Atomic() Trivia() )+ ! + ; + +/** + * A type chosen by what another type is, like: + * - ($value is int ? positive-int : string) + */ +Conditional + : ? Type() + Trivia() Trivia() Type() + Trivia() Trivia() SubType() + ; + +ConditionalForParameter + : ? Type() + Trivia() Trivia() Type() + Trivia() Trivia() SubType() + ; + +// ----------------------------------------------------------------------------- +// Atomic types +// ----------------------------------------------------------------------------- + +Atomic + : ParenthesizedAtomic() + | ThisAtomic() + | IdentifierAtomic() + | !PlainName() ConstantAtomic() + ; + +/** + * A name standing for a type rather than for the class of a constant. + * + * A name is only ever read as a constant expression when a "::" follows it, so + * a name written without one is a type and stays a type: whatever the reading + * fails at after it is an error rather than a reason to read the name again as + * something else. + */ +PlainName + : Identifier() ! + ; + +ParenthesizedAtomic + : Trivia() SubType() Trivia() ArrayOrOffsetAccess()? + ; + +ThisAtomic + : ArrayOrOffsetAccess()? + ; + +/** + * A named type, along with whatever may be written right after the name. + * + * A shape is only read when the name is one of the words that names a shape and + * the brace follows the name with no space in between, which is what tells + * "array{a: int}" from the type "array" followed by a "{a: int}" the type says + * nothing about. + */ +IdentifierAtomic + : ! ArrayShape() ArrayOrOffsetAccess()? + | ListShapeIdentifier() ! ListShape() ArrayOrOffsetAccess()? + | ! ObjectShape() ArrayOrOffsetAccess()? + | ShapeIdentifier() ! ! IdentifierTail() + | PlainIdentifier() ! IdentifierTail() + ; + +/** + * A name that names no shape, told apart from one that does so that a name the + * parser reads every day is read without ever asking whether a shape follows + * it: which of the alternatives above is worth entering is then decided by the + * very first token, and only one of them ever is. + */ +PlainIdentifier + : + | + | + | + | + | + | + | + | + | + | + | + | + | + ; + +/** + * The words naming a shape that is written with no key type after the "...". + * + * Only "array" is written with one, so "array{...}" says what the + * keys are while "non-empty-array{...}" says nothing at all: it is + * an error. + */ +ListShapeIdentifier + : + | + | + ; + +ArrayShapeIdentifier + : + | ListShapeIdentifier() + ; + +/** + * A word naming a shape. Once one is written right before a brace the shape has + * to be read, because "array{" is an error rather than the type "array" + * followed by a brace. + */ +ShapeIdentifier + : ArrayShapeIdentifier() + | + ; + +/** + * What may be written after a name, along with the one thing that may not: a + * "<" the reading stops in front of, which is an error rather than the end of + * the name. + */ +IdentifierTail + : IdentifierSuffix() + | ! + ; + +/** + * A "<" after a name opens either the templates of a callable or the arguments + * of a generic type, and one of the two has to be written: a name the reading + * stops in the middle of is an error rather than a name of its own. + */ +IdentifierSuffix + : & ( CallableWithTemplates() | Generic() ArrayOrOffsetAccess()? ) + | Callable() + | ArrayOrOffsetAccess() + ; + +/** + * A constant expression standing for a type, like: + * - Foo::BAR + * - 1.0 + * - 'foo' + * + * An array is a constant expression but never a type, which is what the + * predicate forbids: "[1, 2]" is an error rather than a type. + */ +ConstantAtomic + : !ArrayLiteral() ConstantExpr() ArrayOrOffsetAccess()? + ; + +ArrayLiteral + : SquareBracketOpen() + | ArrayKeyword() + ; + +// ----------------------------------------------------------------------------- +// Arrays and offsets +// ----------------------------------------------------------------------------- + +/** + * The brackets a type may be written with after it, like: + * - int[] + * - Foo[Bar] + * + * An offset is only read when the bracket follows the type with no space in + * between, so "int [0]" is the type "int" followed by something else. + */ +ArrayOrOffsetAccess + : ArrayOrOffsetAccessItem()+ + ; + +ArrayOrOffsetAccessItem + : ! Type() + | SquareBracketOpen() + ; + +// ----------------------------------------------------------------------------- +// Generic types +// ----------------------------------------------------------------------------- + +Generic + : Trivia() GenericArgument() Trivia() GenericArgumentTail()* + ; + +/** + * The very same rule, written for the one place a "<" opening what looks like + * an HTML tag still opens a generic type: the return type of a callable, which + * the hand-written parser never asks the question about. + */ +GenericInReturnType + : AngleBracketOpen() Trivia() GenericArgument() Trivia() GenericArgumentTail()* + ; + +/** + * A trailing comma is allowed, so a comma may be followed by the closing + * bracket instead of by another argument. + */ +GenericArgumentTail + : Trivia() ( & | GenericArgument() Trivia() ) + ; + +GenericArgument + : Wildcard() + | Variance() Type() + | !Variance() Type() + ; + +Variance + : + | + ; + +// ----------------------------------------------------------------------------- +// Callable types +// ----------------------------------------------------------------------------- + +Callable + : Trivia() CallableParameters()? + CallableReturnType() + ; + +CallableWithTemplates + : CallableTemplates() + Trivia() CallableParameters()? + CallableReturnType() + ; + +CallableTemplates + : Trivia() CallableTemplateArgument() Trivia() + CallableTemplateArgumentTail()* + ; + +CallableTemplateArgumentTail + : Trivia() ( & | CallableTemplateArgument() Trivia() ) + ; + +CallableTemplateArgument + : Identifier() TemplateUpperBound() TemplateLowerBound() TemplateDefault() + ; + +/** + * What a template may be bound by and what it stands for where nothing is + * written for it. + * + * Each of them is read as soon as the word opening it is written, and once it + * is written a type has to follow: "T of" is an error rather than a template + * named "T" followed by the description "of". + */ +TemplateUpperBound + : Type() + | Type() + | ! ! + ; + +TemplateLowerBound + : Type() + | ! + ; + +TemplateDefault + : Type() + | ! + ; + +CallableParameters + : CallableParameter() Trivia() CallableParameterTail()* + ; + +CallableParameterTail + : Trivia() ( & | CallableParameter() Trivia() ) + ; + +CallableParameter + : Type() ? ? ? ? + ; + +/** + * What a callable gives back, which is read without ever looking for a callable + * inside it and without ever asking whether a "<" opens an HTML tag. + */ +CallableReturnType + : Nullable() + | SubType() ArrayOrOffsetAccess()? + | ArrayOrOffsetAccess()? + | ! ArrayShape() ArrayOrOffsetAccess()? + | ListShapeIdentifier() ! ListShape() ArrayOrOffsetAccess()? + | ! ObjectShape() ArrayOrOffsetAccess()? + | ShapeIdentifier() ! ! CallableReturnTypeTail() + | PlainIdentifier() ! CallableReturnTypeTail() + | !PlainName() ConstantAtomic() + ; + +CallableReturnTypeTail + : GenericInReturnType() ArrayOrOffsetAccess()? + | ArrayOrOffsetAccess() + | !AngleBracketOpen() + ; + +// ----------------------------------------------------------------------------- +// Array shapes +// ----------------------------------------------------------------------------- + +/** + * The items an array is written to be made of, like: + * - array{a: int, b?: string} + * - array{int, ...} + */ +ArrayShape + : Trivia() ArrayShapeRest()? Trivia() + ; + +ArrayShapeRest + : ArrayShapeUnsealed() + | ArrayShapeItem() Trivia() ArrayShapeItemTail()? + ; + +/** + * A comment written right after the separating comma is read and thrown away, + * which is the one place a comment ends up on no node at all. + */ +ArrayShapeItemTail + : DiscardedComment()? Trivia() ArrayShapeRest()? + ; + +DiscardedComment + : + ; + +ArrayShapeUnsealed + : Trivia() ( ArrayShapeUnsealedType() Trivia() | !AngleBracketOpen() ) ? + ; + +/** + * What the keys and the values of whatever is left over are written as. + */ +ArrayShapeUnsealedType + : AngleBracketOpen() Trivia() Type() Trivia() + ( Trivia() Type() Trivia() )? + ; + +/** + * A shape of every other kind, which is written exactly like an array shape + * save for the one thing above: what is left over is written with a value type + * and nothing else. + */ +ListShape + : Trivia() ListShapeRest()? Trivia() + ; + +ListShapeRest + : ListShapeUnsealed() + | ArrayShapeItem() Trivia() ListShapeItemTail()? + ; + +ListShapeItemTail + : DiscardedComment()? Trivia() ListShapeRest()? + ; + +ListShapeUnsealed + : Trivia() ( ListShapeUnsealedType() Trivia() | !AngleBracketOpen() ) ? + ; + +ListShapeUnsealedType + : AngleBracketOpen() Trivia() Type() Trivia() + ; + +ArrayShapeItem + : ArrayShapeKey() ? Type() + | Type() + ; + +ArrayShapeKey + : + | + | + | Identifier() ( Identifier() )? + ; + +// ----------------------------------------------------------------------------- +// Object shapes +// ----------------------------------------------------------------------------- + +ObjectShape + : Trivia() + ( ObjectShapeItem() Trivia() ObjectShapeItemTail()* )? + + ; + +ObjectShapeItemTail + : Trivia() ( & | ObjectShapeItem() Trivia() ) + ; + +ObjectShapeItem + : ObjectShapeKey() ? Type() + ; + +ObjectShapeKey + : + | + | Identifier() + ; diff --git a/src/Lexer/Lexer.php b/src/Lexer/Lexer.php index e2e0e576..27ac5dcd 100644 --- a/src/Lexer/Lexer.php +++ b/src/Lexer/Lexer.php @@ -145,7 +145,7 @@ private function generateRegexp(): string self::TOKEN_VARIABLE => '\\$[a-z_\\x80-\\xFF][0-9a-z_\\x80-\\xFF]*+', // '&' followed by TOKEN_VARIADIC, TOKEN_VARIABLE, TOKEN_EQUAL, TOKEN_EQUAL or TOKEN_CLOSE_PARENTHESES - self::TOKEN_REFERENCE => '&(?=\\s*+(?:[.,=)]|(?:\\$(?!this(?![0-9a-z_\\x80-\\xFF])))))', + self::TOKEN_REFERENCE => '&(?=\\s*+(?:[,=)]|\\.\\.\\.|(?:\\$(?!this(?![0-9a-z_\\x80-\\xFF])))))', self::TOKEN_UNION => '\\|', self::TOKEN_INTERSECTION => '&', self::TOKEN_NULLABLE => '\\?', diff --git a/tests/PHPStan/Parser/FuzzyTest.php b/tests/PHPStan/Parser/FuzzyTest.php index 9491bf69..2ef4ec26 100644 --- a/tests/PHPStan/Parser/FuzzyTest.php +++ b/tests/PHPStan/Parser/FuzzyTest.php @@ -3,117 +3,211 @@ namespace PHPStan\PhpDocParser\Parser; use Iterator; +use PHPStan\PhpDocParser\Ast\Node; use PHPStan\PhpDocParser\Lexer\Lexer; use PHPStan\PhpDocParser\ParserConfig; +use PHPStan\PhpDocParser\Printer\Printer; use PHPUnit\Framework\TestCase; use Symfony\Component\Process\Process; use function file_get_contents; use function glob; use function is_dir; -use function mkdir; +use function is_file; use function sprintf; -use function unlink; +use const PHP_BINARY; +use const PHP_VERSION_ID; /** - * @requires OS ^(?!win) + * Reads what the grammars in "doc/grammars" say is a well-formed input. + * + * A grammar says what a PHPDoc may be written as, so "tools/phplrt/fuzz.php" + * walks it the other way round and writes PHPDocs down instead of reading them. + * Every one of them then has to be read in full and has to survive being + * printed and read again, which is a great deal more of the language than a + * hand-written corpus ever covers. + * + * The tool writing the corpus needs PHP 8.4 and a toolchain of its own, so + * where either is missing the test is skipped rather than failed. */ class FuzzyTest extends TestCase { + /** + * How many inputs are written from each grammar. + * + * Enough of them that a run reaches the corners of the language rather than + * only what a short walk over the rules comes out with: at a thousand the + * rules reading a "@method" signature are only reached every other run. + */ + private const INPUTS = 2000; + + /** + * The oldest PHP the tool writing the corpus runs on, which is the one the + * grammar compiler asks for. + */ + private const REQUIRED_PHP_VERSION = 80400; + private Lexer $lexer; + private Printer $printer; + private TypeParser $typeParser; private ConstExprParser $constExprParser; + private PhpDocParser $phpDocParser; + protected function setUp(): void { parent::setUp(); $config = new ParserConfig([]); $this->lexer = new Lexer($config); - $this->typeParser = new TypeParser($config, new ConstExprParser($config)); + $this->printer = new Printer(); $this->constExprParser = new ConstExprParser($config); + $this->typeParser = new TypeParser($config, $this->constExprParser); + $this->phpDocParser = new PhpDocParser($config, $this->typeParser, $this->constExprParser); } /** * @dataProvider provideTypeParserData */ - public function testTypeParser(string $input): void + public function testTypeParser(?string $input): void { - $tokens = new TokenIterator($this->lexer->tokenize($input)); - $this->typeParser->parse($tokens); - - $this->assertSame( - Lexer::TOKEN_END, - $tokens->currentTokenType(), - sprintf('Failed to parse input %s', $input), - ); + $this->assertReadInFull($input, fn (TokenIterator $tokens): Node => $this->typeParser->parse($tokens)); } public function provideTypeParserData(): Iterator { - return $this->provideFuzzyInputsData('Type'); + return $this->provideFuzzyInputsData('type.pp3', 'Type'); } /** * @dataProvider provideConstExprParserData */ - public function testConstExprParser(string $input): void + public function testConstExprParser(?string $input): void { + $this->assertReadInFull($input, fn (TokenIterator $tokens): Node => $this->constExprParser->parse($tokens)); + } + + public function provideConstExprParserData(): Iterator + { + return $this->provideFuzzyInputsData('constant-expr.pp3', 'ConstantExpr'); + } + + /** + * @dataProvider providePhpDocParserData + */ + public function testPhpDocParser(?string $input): void + { + // A description written across several lines is printed as it was read + // rather than as the lines of a PHPDoc, so a whole PHPDoc is not asked + // to survive being printed and read again + $this->assertReadInFull( + $input, + fn (TokenIterator $tokens): Node => $this->phpDocParser->parse($tokens), + false, + ); + } + + public function providePhpDocParserData(): Iterator + { + return $this->provideFuzzyInputsData('phpdoc.pp3', 'PhpDoc'); + } + + /** + * Asks the parser to read the whole input, and to read what it prints back + * as the very same thing. + * + * @param callable(TokenIterator): Node $parse + * @param bool $roundTrips whether what is printed has to be read back as + * the very same thing + */ + private function assertReadInFull(?string $input, callable $parse, bool $roundTrips = true): void + { + if ($input === null) { + self::markTestSkipped('The corpus needs PHP 8.4 and the toolchain of "make grammars-install"'); + } + $tokens = new TokenIterator($this->lexer->tokenize($input)); - $this->constExprParser->parse($tokens); + $node = $parse($tokens); $this->assertSame( Lexer::TOKEN_END, $tokens->currentTokenType(), sprintf('Failed to parse input %s', $input), ); - } - public function provideConstExprParserData(): Iterator - { - return $this->provideFuzzyInputsData('ConstantExpr'); + if (!$roundTrips) { + return; + } + + $printed = $this->printer->print($node); + $printedTokens = new TokenIterator($this->lexer->tokenize($printed)); + $reparsed = $parse($printedTokens); + + $this->assertSame( + Lexer::TOKEN_END, + $printedTokens->currentTokenType(), + sprintf('Failed to parse printed input %s', $printed), + ); + + $this->assertSame( + (string) $node, + (string) $reparsed, + sprintf('Printing and reading back changed the input %s', $input), + ); } - private function provideFuzzyInputsData(string $startSymbol): Iterator + /** + * Writes down a corpus of inputs the given grammar says are well-formed. + * + * @return Iterator + */ + private function provideFuzzyInputsData(string $grammar, string $startSymbol): Iterator { - $inputsDirectory = sprintf('%s/fuzzy/%s', __DIR__ . '/../../../temp', $startSymbol); + $root = __DIR__ . '/../../..'; + $inputsDirectory = sprintf('%s/temp/fuzzy/%s', $root, $startSymbol); - if (is_dir($inputsDirectory)) { - $glob = glob(sprintf('%s/*.tst', $inputsDirectory)); + if ( + PHP_VERSION_ID < self::REQUIRED_PHP_VERSION + || !is_file(sprintf('%s/tools/phplrt/vendor/autoload.php', $root)) + ) { + yield [null]; - if ($glob !== false) { - foreach ($glob as $file) { - unlink($file); - } - } - - } else { - mkdir($inputsDirectory, 0777, true); + return; } $process = new Process([ - __DIR__ . '/../../../tools/abnfgen/abnfgen', - '-lx', - '-n', - '1000', - '-d', + PHP_BINARY, + sprintf('%s/tools/phplrt/fuzz.php', $root), + sprintf('%s/doc/grammars/%s', $root, $grammar), $inputsDirectory, - '-s', - $startSymbol, - __DIR__ . '/../../../doc/grammars/type.abnf', + (string) self::INPUTS, ]); $process->mustRun(); + if (!is_dir($inputsDirectory)) { + yield [null]; + + return; + } + $glob = glob(sprintf('%s/*.tst', $inputsDirectory)); - if ($glob === false) { + if ($glob === false || $glob === []) { + yield [null]; + return; } foreach ($glob as $file) { $input = file_get_contents($file); + + if ($input === false) { + continue; + } + yield [$input]; } } diff --git a/tests/PHPStan/Parser/TypeParserTest.php b/tests/PHPStan/Parser/TypeParserTest.php index 3a9d67e7..1f97651d 100644 --- a/tests/PHPStan/Parser/TypeParserTest.php +++ b/tests/PHPStan/Parser/TypeParserTest.php @@ -421,6 +421,29 @@ public function provideParseData(): array new IdentifierTypeNode('float'), ]), ], + [ + // "&" only means a reference where a variadic really follows it + 'string & .5', + new IntersectionTypeNode([ + new IdentifierTypeNode('string'), + new ConstTypeNode(new ConstExprFloatNode('.5')), + ]), + ], + [ + 'string&.5', + new IntersectionTypeNode([ + new IdentifierTypeNode('string'), + new ConstTypeNode(new ConstExprFloatNode('.5')), + ]), + ], + [ + 'string & .5 & int', + new IntersectionTypeNode([ + new IdentifierTypeNode('string'), + new ConstTypeNode(new ConstExprFloatNode('.5')), + new IdentifierTypeNode('int'), + ]), + ], [ 'string & (int | float)', new IntersectionTypeNode([ diff --git a/tests/abnfgen-0.20.tar.gz b/tests/abnfgen-0.20.tar.gz deleted file mode 100644 index e84e968a..00000000 Binary files a/tests/abnfgen-0.20.tar.gz and /dev/null differ diff --git a/tests/bootstrap.php b/tests/bootstrap.php index f5fb2f9b..558d6ec4 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -1,7 +1,3 @@ > + */ + private const LITERALS = [ + 'T_KEYWORD_CONTRAVARIANT' => ['contravariant'], + 'T_KEYWORD_COVARIANT' => ['covariant'], + 'T_KEYWORD_NON_EMPTY_ARRAY' => ['non-empty-array'], + 'T_KEYWORD_NON_EMPTY_LIST' => ['non-empty-list'], + 'T_KEYWORD_SUPER' => ['super'], + 'T_KEYWORD_STATIC' => ['static'], + 'T_KEYWORD_FROM' => ['from'], + 'T_KEYWORD_OBJECT' => ['object'], + 'T_KEYWORD_ARRAY' => ['array'], + 'T_KEYWORD_LIST' => ['list'], + 'T_KEYWORD_NOT' => ['not'], + 'T_KEYWORD_IS' => ['is'], + 'T_KEYWORD_OF' => ['of'], + 'T_KEYWORD_AS' => ['as'], + 'T_KEYWORD_FALSE' => ['false', 'FALSE'], + 'T_KEYWORD_TRUE' => ['true', 'True'], + 'T_KEYWORD_NULL' => ['null', 'NULL'], + 'T_KEYWORD_ARRAY_ANY_CASE' => ['ARRAY', 'Array'], + + 'T_FLOAT' => ['1.0', '0.5', '-1.5e3', '1_000.5', '.5'], + 'T_INTEGER' => ['0', '1', '123', '-5', '0x1F', '0b101', '0o17', '1_000'], + 'T_SINGLE_QUOTED_STRING' => ["'foo'", "'a b'", "''"], + 'T_DOUBLE_QUOTED_STRING' => ['"foo"', '"a b"', '""'], + + 'T_IDENTIFIER' => ['Foo', 'int', 'string', 'positive-int', 'Bar\\Baz', '\\Foo', 'a'], + 'T_THIS_VARIABLE' => ['$this'], + 'T_VARIABLE' => ['$foo', '$value'], + + 'T_REFERENCE' => ['&'], + 'T_UNION' => ['|'], + 'T_INTERSECTION' => ['&'], + 'T_NULLABLE' => ['?'], + 'T_NEGATED' => ['!'], + + 'T_OPEN_PARENTHESES' => ['('], + 'T_CLOSE_PARENTHESES' => [')'], + 'T_OPEN_ANGLE_BRACKET' => ['<'], + 'T_CLOSE_ANGLE_BRACKET' => ['>'], + 'T_OPEN_SQUARE_BRACKET' => ['['], + 'T_CLOSE_SQUARE_BRACKET' => [']'], + 'T_OPEN_CURLY_BRACKET' => ['{'], + 'T_CLOSE_CURLY_BRACKET' => ['}'], + + 'T_COMMA' => [','], + 'T_COMMENT' => ['// a comment'], + 'T_VARIADIC' => ['...'], + 'T_DOUBLE_COLON' => ['::'], + 'T_DOUBLE_ARROW' => ['=>'], + 'T_ARROW' => ['->'], + 'T_EQUAL' => ['='], + 'T_COLON' => [':'], + 'T_WILDCARD' => ['*'], + + 'T_PHPDOC_EOL' => ["\n * "], + 'T_OTHER' => ['#other'], + + 'T_OPEN_PHPDOC' => ['/** '], + 'T_CLOSE_PHPDOC' => ['*/'], + 'T_PHPDOC_TAG' => ['@author'], + 'T_PHPDOC_TAG_WS' => ['@author'], + // A tag only reads as a Doctrine one where a "\\" or a "_" follows the + // "@" straight away: anything else is read as an ordinary tag first, + // and "@Foo_Bar" comes back as "@Foo" followed by the name "_Bar" + 'T_DOCTRINE_TAG' => ['@\\Foo'], + 'T_DOCTRINE_TAG_WS' => ['@\\Foo'], + + 'T_OPEN_CURLY_BRACKET_WS' => ['{'], + 'T_OPEN_SQUARE_BRACKET_WS' => ['['], + 'T_WILDCARD_WS' => ['*'], + 'T_OPEN_ANGLE_BRACKET_HTML' => ['<'], + + 'T_TAG_PARAM' => ['@param', '@phpstan-param'], + 'T_TAG_PARAM_IMMEDIATELY_INVOKED_CALLABLE' => ['@param-immediately-invoked-callable'], + 'T_TAG_PARAM_LATER_INVOKED_CALLABLE' => ['@param-later-invoked-callable'], + 'T_TAG_PARAM_CLOSURE_THIS' => ['@param-closure-this'], + 'T_TAG_PURE_UNLESS_CALLABLE_IS_IMPURE' => ['@pure-unless-callable-is-impure'], + 'T_TAG_PURE_UNLESS_PARAMETER_PASSED' => ['@pure-unless-parameter-passed'], + 'T_TAG_VAR' => ['@var'], + 'T_TAG_RETURN' => ['@return'], + 'T_TAG_THROWS' => ['@throws'], + 'T_TAG_MIXIN' => ['@mixin'], + 'T_TAG_REQUIRE_EXTENDS' => ['@phpstan-require-extends'], + 'T_TAG_REQUIRE_IMPLEMENTS' => ['@phpstan-require-implements'], + 'T_TAG_SEALED' => ['@phpstan-sealed'], + 'T_TAG_DEPRECATED' => ['@deprecated'], + 'T_TAG_PROPERTY' => ['@property', '@property-read'], + 'T_TAG_METHOD' => ['@method'], + 'T_TAG_TEMPLATE' => ['@template'], + 'T_TAG_EXTENDS' => ['@extends'], + 'T_TAG_IMPLEMENTS' => ['@implements'], + 'T_TAG_USE' => ['@use'], + 'T_TAG_TYPE_ALIAS' => ['@phpstan-type'], + 'T_TAG_TYPE_ALIAS_IMPORT' => ['@phpstan-import-type'], + 'T_TAG_ASSERT' => ['@phpstan-assert'], + 'T_TAG_SELF_OUT' => ['@phpstan-self-out'], + 'T_TAG_PARAM_OUT' => ['@param-out'], + ]; + + /** + * The tokens whose meaning the whitespace around them decides: what such a + * token is called is exactly what has to be written around it. + * + * @var array + */ + private const SPACING_BEFORE = [ + 'T_PHPDOC_TAG' => '', + 'T_DOCTRINE_TAG' => '', + 'T_OPEN_CURLY_BRACKET' => '', + 'T_OPEN_SQUARE_BRACKET' => '', + 'T_OPEN_CURLY_BRACKET_WS' => ' ', + 'T_OPEN_SQUARE_BRACKET_WS' => ' ', + 'T_PHPDOC_EOL' => '', + ]; + + /** + * @var array + */ + private const SPACING_AFTER = [ + 'T_WILDCARD' => '', + 'T_WILDCARD_WS' => ' ', + 'T_PHPDOC_EOL' => '', + ]; + + /** + * How many tokens an input may be written of before the walk starts looking + * for the shortest way out of the rule it is in. + */ + private const SIZE_LIMIT = 24; + + /** + * The fewest tokens each rule can be written of, which is what the walk + * takes the shortest way out of a rule by. + * + * @var array + */ + private array $sizes; + + /** + * @param list $grammar + * @param array $names + */ + public function __construct( + private array $grammar, + private array $names, + private int $initial, + ) { + $this->sizes = $this->calculateSizes(); + } + + /** + * @return list|null the tokens of an input, or "null" for a walk that + * has written nothing + */ + public function generate(): ?array + { + $tokens = []; + $this->walk($this->initial, $tokens); + + return $tokens === [] ? null : $tokens; + } + + /** + * @param list $tokens + */ + private function walk(int $rule, array &$tokens): void + { + $definition = $this->grammar[$rule]; + + if ($definition instanceof Lexeme) { + $tokens[] = $definition->tokenId; + + return; + } + + if ($definition instanceof Predicate) { + // A predicate reads nothing at all, and what it looks ahead at is + // written by whatever follows it + return; + } + + if ($definition instanceof Concatenation) { + foreach ($definition->ruleIds as $inner) { + $this->walk($inner, $tokens); + } + + return; + } + + if ($definition instanceof Alternation) { + $this->walk($this->choose($definition->ruleIds, $tokens), $tokens); + + return; + } + + if ($definition instanceof Optional) { + if ($this->isLongEnough($tokens) || \mt_rand(0, 2) === 0) { + return; + } + + $this->walk($definition->ruleId, $tokens); + + return; + } + + \assert($definition instanceof Repetition); + + $max = $definition->max === \INF ? $definition->min + 2 : (int) $definition->max; + $times = $this->isLongEnough($tokens) + ? $definition->min + : \mt_rand($definition->min, \max($definition->min, \min($max, $definition->min + 2))); + + for ($i = 0; $i < $times; $i++) { + $this->walk($definition->ruleId, $tokens); + } + } + + /** + * @param list $alternatives + * @param list $tokens + */ + private function choose(array $alternatives, array $tokens): int + { + if (!$this->isLongEnough($tokens)) { + return $alternatives[\mt_rand(0, \count($alternatives) - 1)]; + } + + // An input that has grown long enough goes on with the alternative that + // finishes it soonest, so that a grammar written of itself still stops + $shortest = $alternatives[0]; + + foreach ($alternatives as $alternative) { + if ($this->sizes[$alternative] >= $this->sizes[$shortest]) { + continue; + } + + $shortest = $alternative; + } + + return $shortest; + } + + /** + * @param list $tokens + */ + private function isLongEnough(array $tokens): bool + { + return \count($tokens) >= self::SIZE_LIMIT; + } + + /** + * Writes the given tokens down as the text they are read from. + * + * @param list $tokens + */ + public function render(array $tokens): string + { + $text = ''; + + foreach ($tokens as $index => $token) { + $name = $this->names[$token] ?? null; + $literals = $name === null ? null : (self::LITERALS[$name] ?? null); + + if ($literals === null) { + // The end of the input is written by the text simply stopping + continue; + } + + if ($index > 0) { + $after = self::SPACING_AFTER[$this->names[$tokens[$index - 1]] ?? ''] ?? null; + + $text .= $after ?? (self::SPACING_BEFORE[$name] ?? ' '); + } + + $text .= $literals[\mt_rand(0, \count($literals) - 1)]; + } + + return $text; + } + + /** + * The tokens of an input, leaving out the ones nothing is written for. + * + * @param list $tokens + * @return list + */ + public function written(array $tokens): array + { + $result = []; + + foreach ($tokens as $token) { + if (!isset(self::LITERALS[$this->names[$token] ?? ''])) { + continue; + } + + $result[] = $token; + } + + return $result; + } + + /** + * @return array + */ + private function calculateSizes(): array + { + $sizes = []; + + foreach ($this->grammar as $rule => $definition) { + $sizes[$rule] = $definition instanceof Lexeme ? 1 : \PHP_INT_MAX; + } + + // The fewest tokens a rule is written of depends on the rules it is + // written of, so the answer is looked for until it stops changing + do { + $changed = false; + + foreach ($this->grammar as $rule => $definition) { + $size = $this->calculateSize($definition, $sizes); + + if ($size >= $sizes[$rule]) { + continue; + } + + $sizes[$rule] = $size; + $changed = true; + } + } while ($changed); + + return $sizes; + } + + /** + * @param array $sizes + */ + private function calculateSize(RuleInterface $definition, array $sizes): int + { + if ($definition instanceof Lexeme) { + return 1; + } + + if ($definition instanceof Predicate || $definition instanceof Optional) { + return 0; + } + + if ($definition instanceof Repetition) { + return $definition->min === 0 ? 0 : $definition->min * $sizes[$definition->ruleId]; + } + + if ($definition instanceof Alternation) { + $size = \PHP_INT_MAX; + + foreach ($definition->ruleIds as $inner) { + $size = \min($size, $sizes[$inner]); + } + + return $size; + } + + \assert($definition instanceof Concatenation); + + $size = 0; + + foreach ($definition->ruleIds as $inner) { + if ($sizes[$inner] === \PHP_INT_MAX) { + return \PHP_INT_MAX; + } + + $size += $sizes[$inner]; + } + + return $size; + } +} diff --git a/tools/phplrt/Fuzzer/TokenStream.php b/tools/phplrt/Fuzzer/TokenStream.php new file mode 100644 index 00000000..ace6ebab --- /dev/null +++ b/tools/phplrt/Fuzzer/TokenStream.php @@ -0,0 +1,442 @@ +channel = $isEndOfInput ? Channel::EndOfInput : Channel::Default; + } + + public function __toString(): string + { + return $this->value; + } +} + +/** + * A source that is a stream of tokens rather than a text. + */ +final class TokenSource implements ReadableInterface +{ + public string $content { + get => ''; + } + + /** + * @param list $tokens + */ + public function __construct(public array $tokens) {} + + public function read(int $offset, int $bytes): string + { + return ''; + } +} + +final class TokenStreamLexer implements LexerInterface +{ + public function lex(ReadableInterface $source, int $offset = 0): iterable + { + \assert($source instanceof TokenSource); + + return $source->tokens; + } +} + +final class TokenStream +{ + /** + * The words the hand-written parser compares a T_IDENTIFIER against by + * value, written exactly as it compares them. + * + * @var array + */ + private const KEYWORDS = [ + 'is' => 'T_KEYWORD_IS', + 'not' => 'T_KEYWORD_NOT', + 'of' => 'T_KEYWORD_OF', + 'as' => 'T_KEYWORD_AS', + 'super' => 'T_KEYWORD_SUPER', + 'static' => 'T_KEYWORD_STATIC', + 'from' => 'T_KEYWORD_FROM', + 'covariant' => 'T_KEYWORD_COVARIANT', + 'contravariant' => 'T_KEYWORD_CONTRAVARIANT', + 'array' => 'T_KEYWORD_ARRAY', + 'list' => 'T_KEYWORD_LIST', + 'non-empty-array' => 'T_KEYWORD_NON_EMPTY_ARRAY', + 'non-empty-list' => 'T_KEYWORD_NON_EMPTY_LIST', + 'object' => 'T_KEYWORD_OBJECT', + 'true' => 'T_KEYWORD_TRUE', + 'false' => 'T_KEYWORD_FALSE', + 'null' => 'T_KEYWORD_NULL', + ]; + + /** + * The words the constant expression parser compares without regard to case. + * + * @var array + */ + private const KEYWORDS_ANY_CASE = [ + 'true' => 'T_KEYWORD_TRUE', + 'false' => 'T_KEYWORD_FALSE', + 'null' => 'T_KEYWORD_NULL', + 'array' => 'T_KEYWORD_ARRAY_ANY_CASE', + ]; + + private const KEYWORDS_ANY_CASE_LENGTH = 5; + + /** + * The tags whose value PhpDocParser::parseTagValue() reads by a rule of its + * own, told apart by the whole name written. + * + * @var array + */ + private const TAGS = [ + '@param' => 'T_TAG_PARAM', + '@phpstan-param' => 'T_TAG_PARAM', + '@psalm-param' => 'T_TAG_PARAM', + '@phan-param' => 'T_TAG_PARAM', + '@param-immediately-invoked-callable' => 'T_TAG_PARAM_IMMEDIATELY_INVOKED_CALLABLE', + '@phpstan-param-immediately-invoked-callable' => 'T_TAG_PARAM_IMMEDIATELY_INVOKED_CALLABLE', + '@param-later-invoked-callable' => 'T_TAG_PARAM_LATER_INVOKED_CALLABLE', + '@phpstan-param-later-invoked-callable' => 'T_TAG_PARAM_LATER_INVOKED_CALLABLE', + '@param-closure-this' => 'T_TAG_PARAM_CLOSURE_THIS', + '@phpstan-param-closure-this' => 'T_TAG_PARAM_CLOSURE_THIS', + '@pure-unless-callable-is-impure' => 'T_TAG_PURE_UNLESS_CALLABLE_IS_IMPURE', + '@phpstan-pure-unless-callable-is-impure' => 'T_TAG_PURE_UNLESS_CALLABLE_IS_IMPURE', + '@pure-unless-parameter-passed' => 'T_TAG_PURE_UNLESS_PARAMETER_PASSED', + '@phpstan-pure-unless-parameter-passed' => 'T_TAG_PURE_UNLESS_PARAMETER_PASSED', + '@var' => 'T_TAG_VAR', + '@phpstan-var' => 'T_TAG_VAR', + '@psalm-var' => 'T_TAG_VAR', + '@phan-var' => 'T_TAG_VAR', + '@return' => 'T_TAG_RETURN', + '@phpstan-return' => 'T_TAG_RETURN', + '@psalm-return' => 'T_TAG_RETURN', + '@phan-return' => 'T_TAG_RETURN', + '@phan-real-return' => 'T_TAG_RETURN', + '@throws' => 'T_TAG_THROWS', + '@phpstan-throws' => 'T_TAG_THROWS', + '@mixin' => 'T_TAG_MIXIN', + '@phan-mixin' => 'T_TAG_MIXIN', + '@psalm-require-extends' => 'T_TAG_REQUIRE_EXTENDS', + '@phpstan-require-extends' => 'T_TAG_REQUIRE_EXTENDS', + '@psalm-require-implements' => 'T_TAG_REQUIRE_IMPLEMENTS', + '@phpstan-require-implements' => 'T_TAG_REQUIRE_IMPLEMENTS', + '@psalm-inheritors' => 'T_TAG_SEALED', + '@phpstan-sealed' => 'T_TAG_SEALED', + '@deprecated' => 'T_TAG_DEPRECATED', + '@property' => 'T_TAG_PROPERTY', + '@property-read' => 'T_TAG_PROPERTY', + '@property-write' => 'T_TAG_PROPERTY', + '@phpstan-property' => 'T_TAG_PROPERTY', + '@phpstan-property-read' => 'T_TAG_PROPERTY', + '@phpstan-property-write' => 'T_TAG_PROPERTY', + '@psalm-property' => 'T_TAG_PROPERTY', + '@psalm-property-read' => 'T_TAG_PROPERTY', + '@psalm-property-write' => 'T_TAG_PROPERTY', + '@phan-property' => 'T_TAG_PROPERTY', + '@phan-property-read' => 'T_TAG_PROPERTY', + '@phan-property-write' => 'T_TAG_PROPERTY', + '@method' => 'T_TAG_METHOD', + '@phpstan-method' => 'T_TAG_METHOD', + '@psalm-method' => 'T_TAG_METHOD', + '@phan-method' => 'T_TAG_METHOD', + '@template' => 'T_TAG_TEMPLATE', + '@phpstan-template' => 'T_TAG_TEMPLATE', + '@psalm-template' => 'T_TAG_TEMPLATE', + '@phan-template' => 'T_TAG_TEMPLATE', + '@template-covariant' => 'T_TAG_TEMPLATE', + '@phpstan-template-covariant' => 'T_TAG_TEMPLATE', + '@psalm-template-covariant' => 'T_TAG_TEMPLATE', + '@template-contravariant' => 'T_TAG_TEMPLATE', + '@phpstan-template-contravariant' => 'T_TAG_TEMPLATE', + '@psalm-template-contravariant' => 'T_TAG_TEMPLATE', + '@extends' => 'T_TAG_EXTENDS', + '@phpstan-extends' => 'T_TAG_EXTENDS', + '@phan-extends' => 'T_TAG_EXTENDS', + '@phan-inherits' => 'T_TAG_EXTENDS', + '@template-extends' => 'T_TAG_EXTENDS', + '@implements' => 'T_TAG_IMPLEMENTS', + '@phpstan-implements' => 'T_TAG_IMPLEMENTS', + '@template-implements' => 'T_TAG_IMPLEMENTS', + '@use' => 'T_TAG_USE', + '@phpstan-use' => 'T_TAG_USE', + '@template-use' => 'T_TAG_USE', + '@phpstan-type' => 'T_TAG_TYPE_ALIAS', + '@psalm-type' => 'T_TAG_TYPE_ALIAS', + '@phan-type' => 'T_TAG_TYPE_ALIAS', + '@phpstan-import-type' => 'T_TAG_TYPE_ALIAS_IMPORT', + '@psalm-import-type' => 'T_TAG_TYPE_ALIAS_IMPORT', + '@phpstan-assert' => 'T_TAG_ASSERT', + '@phpstan-assert-if-true' => 'T_TAG_ASSERT', + '@phpstan-assert-if-false' => 'T_TAG_ASSERT', + '@psalm-assert' => 'T_TAG_ASSERT', + '@psalm-assert-if-true' => 'T_TAG_ASSERT', + '@psalm-assert-if-false' => 'T_TAG_ASSERT', + '@phan-assert' => 'T_TAG_ASSERT', + '@phan-assert-if-true' => 'T_TAG_ASSERT', + '@phan-assert-if-false' => 'T_TAG_ASSERT', + '@phpstan-this-out' => 'T_TAG_SELF_OUT', + '@phpstan-self-out' => 'T_TAG_SELF_OUT', + '@psalm-this-out' => 'T_TAG_SELF_OUT', + '@psalm-self-out' => 'T_TAG_SELF_OUT', + '@param-out' => 'T_TAG_PARAM_OUT', + '@phpstan-param-out' => 'T_TAG_PARAM_OUT', + '@psalm-param-out' => 'T_TAG_PARAM_OUT', + ]; + + /** + * The number each token of the lexer is named by, for the ones the grammar + * does not tell apart any further. + * + * @var array + */ + private array $types; + + /** + * @param array $ids the number the grammar names + * each of its tokens by + */ + public function __construct(private array $ids) + { + $this->types = [ + Lexer::TOKEN_REFERENCE => $this->id('T_REFERENCE'), + Lexer::TOKEN_UNION => $this->id('T_UNION'), + Lexer::TOKEN_INTERSECTION => $this->id('T_INTERSECTION'), + Lexer::TOKEN_NULLABLE => $this->id('T_NULLABLE'), + Lexer::TOKEN_NEGATED => $this->id('T_NEGATED'), + Lexer::TOKEN_OPEN_PARENTHESES => $this->id('T_OPEN_PARENTHESES'), + Lexer::TOKEN_CLOSE_PARENTHESES => $this->id('T_CLOSE_PARENTHESES'), + Lexer::TOKEN_CLOSE_ANGLE_BRACKET => $this->id('T_CLOSE_ANGLE_BRACKET'), + Lexer::TOKEN_CLOSE_SQUARE_BRACKET => $this->id('T_CLOSE_SQUARE_BRACKET'), + Lexer::TOKEN_CLOSE_CURLY_BRACKET => $this->id('T_CLOSE_CURLY_BRACKET'), + Lexer::TOKEN_COMMA => $this->id('T_COMMA'), + Lexer::TOKEN_COMMENT => $this->id('T_COMMENT'), + Lexer::TOKEN_VARIADIC => $this->id('T_VARIADIC'), + Lexer::TOKEN_DOUBLE_COLON => $this->id('T_DOUBLE_COLON'), + Lexer::TOKEN_DOUBLE_ARROW => $this->id('T_DOUBLE_ARROW'), + Lexer::TOKEN_ARROW => $this->id('T_ARROW'), + Lexer::TOKEN_EQUAL => $this->id('T_EQUAL'), + Lexer::TOKEN_COLON => $this->id('T_COLON'), + Lexer::TOKEN_FLOAT => $this->id('T_FLOAT'), + Lexer::TOKEN_INTEGER => $this->id('T_INTEGER'), + Lexer::TOKEN_SINGLE_QUOTED_STRING => $this->id('T_SINGLE_QUOTED_STRING'), + Lexer::TOKEN_DOUBLE_QUOTED_STRING => $this->id('T_DOUBLE_QUOTED_STRING'), + Lexer::TOKEN_DOCTRINE_ANNOTATION_STRING => $this->id('T_DOCTRINE_ANNOTATION_STRING'), + Lexer::TOKEN_THIS_VARIABLE => $this->id('T_THIS_VARIABLE'), + Lexer::TOKEN_VARIABLE => $this->id('T_VARIABLE'), + Lexer::TOKEN_OPEN_PHPDOC => $this->id('T_OPEN_PHPDOC'), + Lexer::TOKEN_CLOSE_PHPDOC => $this->id('T_CLOSE_PHPDOC'), + Lexer::TOKEN_PHPDOC_EOL => $this->id('T_PHPDOC_EOL'), + Lexer::TOKEN_OTHER => $this->id('T_OTHER'), + Lexer::TOKEN_END => EndOfInputToken::TOKEN_ID, + ]; + } + + /** + * @param non-empty-string $name + */ + private function id(string $name): int + { + return $this->ids[$name] ?? throw new \RuntimeException(\sprintf('The grammar declares no %s', $name)); + } + + /** + * @param list $tokens + * @return list + */ + public function create(array $tokens): array + { + $result = []; + $offset = 0; + + foreach ($tokens as $index => $token) { + $type = $token[Lexer::TYPE_OFFSET]; + $value = $token[Lexer::VALUE_OFFSET]; + + if ($type === Lexer::TOKEN_HORIZONTAL_WS) { + $offset += \strlen($value); + + continue; + } + + $result[] = new Token( + $this->identify($tokens, $index, $type, $value), + $value, + $offset, + \strlen($value), + $type === Lexer::TOKEN_END, + ); + + $offset += \strlen($value); + } + + return $result; + } + + /** + * @param list $tokens + */ + private function identify(array $tokens, int $index, int $type, string $value): int + { + switch ($type) { + case Lexer::TOKEN_IDENTIFIER: + $keyword = self::KEYWORDS[$value] ?? null; + + if ($keyword !== null) { + return $this->id($keyword); + } + + if (\strlen($value) <= self::KEYWORDS_ANY_CASE_LENGTH) { + $keyword = self::KEYWORDS_ANY_CASE[\strtolower($value)] ?? null; + } + + return $this->id($keyword ?? 'T_IDENTIFIER'); + + case Lexer::TOKEN_PHPDOC_TAG: + $tag = self::TAGS[$value] ?? null; + + if ($tag !== null) { + return $this->id($tag); + } + + return $this->id(self::isPrecededByWhitespace($tokens, $index) ? 'T_PHPDOC_TAG_WS' : 'T_PHPDOC_TAG'); + + case Lexer::TOKEN_DOCTRINE_TAG: + return $this->id(self::isPrecededByWhitespace($tokens, $index) ? 'T_DOCTRINE_TAG_WS' : 'T_DOCTRINE_TAG'); + + case Lexer::TOKEN_OPEN_CURLY_BRACKET: + return $this->id(self::isPrecededByWhitespace($tokens, $index) ? 'T_OPEN_CURLY_BRACKET_WS' : 'T_OPEN_CURLY_BRACKET'); + + case Lexer::TOKEN_OPEN_SQUARE_BRACKET: + return $this->id(self::isPrecededByWhitespace($tokens, $index) ? 'T_OPEN_SQUARE_BRACKET_WS' : 'T_OPEN_SQUARE_BRACKET'); + + case Lexer::TOKEN_WILDCARD: + return $this->id(self::isFollowedByWhitespace($tokens, $index) ? 'T_WILDCARD_WS' : 'T_WILDCARD'); + + case Lexer::TOKEN_OPEN_ANGLE_BRACKET: + return $this->id(self::isHtml($tokens, $index) ? 'T_OPEN_ANGLE_BRACKET_HTML' : 'T_OPEN_ANGLE_BRACKET'); + + default: + return $this->types[$type]; + } + } + + /** + * @param list $tokens + */ + private static function isPrecededByWhitespace(array $tokens, int $index): bool + { + return ($tokens[$index - 1][Lexer::TYPE_OFFSET] ?? -1) === Lexer::TOKEN_HORIZONTAL_WS; + } + + /** + * @param list $tokens + */ + private static function isFollowedByWhitespace(array $tokens, int $index): bool + { + return ($tokens[$index + 1][Lexer::TYPE_OFFSET] ?? -1) === Lexer::TOKEN_HORIZONTAL_WS; + } + + /** + * Whether the "<" at the given position opens what + * PHPStan\PhpDocParser\Parser\TypeParser::isHtml() recognizes as an HTML + * tag, read the very same way it reads it. + * + * @param list $tokens + */ + private static function isHtml(array $tokens, int $index): bool + { + $count = \count($tokens); + + $index = self::skipWhitespace($tokens, $index + 1, $count); + if ($index >= $count || $tokens[$index][Lexer::TYPE_OFFSET] !== Lexer::TOKEN_IDENTIFIER) { + return false; + } + + $name = $tokens[$index][Lexer::VALUE_OFFSET]; + + $index = self::skipWhitespace($tokens, $index + 1, $count); + if ($index >= $count || $tokens[$index][Lexer::TYPE_OFFSET] !== Lexer::TOKEN_CLOSE_ANGLE_BRACKET) { + return false; + } + + $index = self::skipWhitespace($tokens, $index + 1, $count); + + $endTag = ''; + $length = \strlen($endTag); + + while ($index < $count && $tokens[$index][Lexer::TYPE_OFFSET] !== Lexer::TOKEN_END) { + if ($tokens[$index][Lexer::TYPE_OFFSET] === Lexer::TOKEN_OPEN_ANGLE_BRACKET) { + $index = self::skipWhitespace($tokens, $index + 1, $count); + + if ($index < $count && \strpos($tokens[$index][Lexer::VALUE_OFFSET], '/' . $name . '>') !== false) { + return true; + } + } + + if ($index < $count) { + $value = $tokens[$index][Lexer::VALUE_OFFSET]; + + if (\strlen($value) >= $length && \substr_compare($value, $endTag, -$length) === 0) { + return true; + } + } + + $index = self::skipWhitespace($tokens, $index + 1, $count); + } + + return false; + } + + /** + * @param list $tokens + */ + private static function skipWhitespace(array $tokens, int $index, int $count): int + { + while ($index < $count && $tokens[$index][Lexer::TYPE_OFFSET] === Lexer::TOKEN_HORIZONTAL_WS) { + $index++; + } + + return $index; + } +} diff --git a/tools/phplrt/composer.json b/tools/phplrt/composer.json new file mode 100644 index 00000000..75edac07 --- /dev/null +++ b/tools/phplrt/composer.json @@ -0,0 +1,14 @@ +{ + "description": "Dev-only toolchain for compiling the phplrt grammars in doc/grammars into generated parsers.", + "require": { + "phplrt/phplrt": "4.0.0-rc1" + }, + "minimum-stability": "dev", + "prefer-stable": true, + "config": { + "platform": { + "php": "8.4.0" + }, + "sort-packages": true + } +} diff --git a/tools/phplrt/composer.lock b/tools/phplrt/composer.lock new file mode 100644 index 00000000..8101c845 --- /dev/null +++ b/tools/phplrt/composer.lock @@ -0,0 +1,996 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "31e0382729bc7f44d23ee49bea1427eb", + "packages": [ + { + "name": "laminas/laminas-code", + "version": "4.17.0", + "source": { + "type": "git", + "url": "https://github.com/laminas/laminas-code.git", + "reference": "40d61e2899ec17c5d08bbc0a2d586b3ca17ab9bd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laminas/laminas-code/zipball/40d61e2899ec17c5d08bbc0a2d586b3ca17ab9bd", + "reference": "40d61e2899ec17c5d08bbc0a2d586b3ca17ab9bd", + "shasum": "" + }, + "require": { + "php": "~8.2.0 || ~8.3.0 || ~8.4.0 || ~8.5.0" + }, + "require-dev": { + "doctrine/annotations": "^2.0.1", + "ext-phar": "*", + "laminas/laminas-coding-standard": "^3.0.0", + "laminas/laminas-stdlib": "^3.18.0", + "phpunit/phpunit": "^10.5.58", + "psalm/plugin-phpunit": "^0.19.0", + "vimeo/psalm": "^5.15.0" + }, + "suggest": { + "doctrine/annotations": "Doctrine\\Common\\Annotations >=1.0 for annotation features", + "laminas/laminas-stdlib": "Laminas\\Stdlib component" + }, + "type": "library", + "autoload": { + "psr-4": { + "Laminas\\Code\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "description": "Extensions to the PHP Reflection API, static code scanning, and code generation", + "homepage": "https://laminas.dev", + "keywords": [ + "code", + "laminas", + "laminasframework" + ], + "support": { + "chat": "https://laminas.dev/chat", + "docs": "https://docs.laminas.dev/laminas-code/", + "forum": "https://discourse.laminas.dev", + "issues": "https://github.com/laminas/laminas-code/issues", + "rss": "https://github.com/laminas/laminas-code/releases.atom", + "source": "https://github.com/laminas/laminas-code" + }, + "funding": [ + { + "url": "https://funding.communitybridge.org/projects/laminas-project", + "type": "community_bridge" + } + ], + "time": "2025-11-01T09:38:14+00:00" + }, + { + "name": "phplrt/phplrt", + "version": "4.0.0-rc1", + "source": { + "type": "git", + "url": "https://github.com/phplrt/phplrt.git", + "reference": "348cd166f750d0021aa95e3f87ea6f17e29b2147" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phplrt/phplrt/zipball/348cd166f750d0021aa95e3f87ea6f17e29b2147", + "reference": "348cd166f750d0021aa95e3f87ea6f17e29b2147", + "shasum": "" + }, + "require": { + "composer-runtime-api": "^2.2", + "ext-json": "*", + "ext-pcre": "*", + "laminas/laminas-code": "^4.17", + "php": "^8.4", + "symfony/console": "^7.4|^8.0", + "twig/twig": "^3.28" + }, + "provide": { + "phplrt/lexer-contracts-implementation": "^4.0", + "phplrt/parser-contracts-implementation": "^4.0", + "phplrt/position-contracts-implementation": "^4.0", + "phplrt/position-factory-contracts-implementation": "^4.0", + "phplrt/source-contracts-implementation": "^4.0", + "phplrt/source-factory-contracts-implementation": "^4.0" + }, + "replace": { + "phplrt/compiler": "*", + "phplrt/exception": "*", + "phplrt/lexer": "*", + "phplrt/lexer-builder": "*", + "phplrt/lexer-contracts": "*", + "phplrt/parser": "*", + "phplrt/parser-builder": "*", + "phplrt/parser-contracts": "*", + "phplrt/position": "*", + "phplrt/position-contracts": "*", + "phplrt/position-factory-contracts": "*", + "phplrt/runtime": "*", + "phplrt/source": "*", + "phplrt/source-contracts": "*", + "phplrt/source-factory-contracts": "*" + }, + "require-dev": { + "ext-intl": "*", + "ext-mbstring": "*", + "friendsofphp/php-cs-fixer": "^3.95", + "phpstan/phpstan": "^2.2", + "phpstan/phpstan-strict-rules": "^2.0", + "phpunit/phpunit": "^13.2", + "symfony/var-dumper": "^6.4|^7.0|^8.0", + "symplify/monorepo-builder": "^12.7" + }, + "bin": [ + "libs/components/compiler/bin/phplrt" + ], + "type": "library", + "autoload": { + "files": [ + "libs/components/source/src/polyfill.php" + ], + "psr-4": { + "Phplrt\\Lexer\\": "libs/components/lexer/src", + "Phplrt\\Parser\\": "libs/components/parser/src", + "Phplrt\\Source\\": "libs/components/source/src", + "Phplrt\\Compiler\\": "libs/components/compiler/src", + "Phplrt\\Position\\": "libs/components/position/src", + "Phplrt\\Exception\\": "libs/components/exception/src", + "Phplrt\\Lexer\\Builder\\": "libs/components/lexer-builder/src", + "Phplrt\\Parser\\Builder\\": "libs/components/parser-builder/src", + "Phplrt\\Contracts\\Lexer\\": "libs/contracts/lexer/src", + "Phplrt\\Contracts\\Parser\\": "libs/contracts/parser/src", + "Phplrt\\Contracts\\Source\\": [ + "libs/contracts/source/src", + "libs/contracts/source-factory/src" + ], + "Phplrt\\Contracts\\Position\\": [ + "libs/contracts/position/src", + "libs/contracts/position-factory/src" + ] + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Kirill Nesmeyanov", + "email": "nesk@xakep.ru" + } + ], + "description": "PHP Language Recognition Tool", + "homepage": "https://phplrt.org", + "support": { + "issues": "https://github.com/phplrt/phplrt/issues", + "source": "https://github.com/phplrt/phplrt" + }, + "time": "2026-08-25T09:57:58+00:00" + }, + { + "name": "psr/container", + "version": "2.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/container.git", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "shasum": "" + }, + "require": { + "php": ">=7.4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Container\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common Container Interface (PHP FIG PSR-11)", + "homepage": "https://github.com/php-fig/container", + "keywords": [ + "PSR-11", + "container", + "container-interface", + "container-interop", + "psr" + ], + "support": { + "issues": "https://github.com/php-fig/container/issues", + "source": "https://github.com/php-fig/container/tree/2.0.2" + }, + "time": "2021-11-05T16:47:00+00:00" + }, + { + "name": "symfony/console", + "version": "v8.0.15", + "source": { + "type": "git", + "url": "https://github.com/symfony/console.git", + "reference": "7327288efcaf02a3838d2de0ae23915d64713e14" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/console/zipball/7327288efcaf02a3838d2de0ae23915d64713e14", + "reference": "7327288efcaf02a3838d2de0ae23915d64713e14", + "shasum": "" + }, + "require": { + "php": ">=8.4", + "symfony/polyfill-mbstring": "^1.0", + "symfony/service-contracts": "^2.5|^3", + "symfony/string": "^7.4|^8.0" + }, + "provide": { + "psr/log-implementation": "1.0|2.0|3.0" + }, + "require-dev": { + "psr/log": "^1|^2|^3", + "symfony/config": "^7.4|^8.0", + "symfony/dependency-injection": "^7.4|^8.0", + "symfony/event-dispatcher": "^7.4|^8.0", + "symfony/http-foundation": "^7.4|^8.0", + "symfony/http-kernel": "^7.4|^8.0", + "symfony/lock": "^7.4|^8.0", + "symfony/messenger": "^7.4|^8.0", + "symfony/process": "^7.4|^8.0", + "symfony/stopwatch": "^7.4|^8.0", + "symfony/var-dumper": "^7.4|^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Console\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Eases the creation of beautiful and testable command line interfaces", + "homepage": "https://symfony.com", + "keywords": [ + "cli", + "command-line", + "console", + "terminal" + ], + "support": { + "source": "https://github.com/symfony/console/tree/v8.0.15" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-07-27T13:52:45+00:00" + }, + { + "name": "symfony/deprecation-contracts", + "version": "v3.7.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/deprecation-contracts.git", + "reference": "f3202fa1b5097b0af062dc978b32ecf63404e31d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/f3202fa1b5097b0af062dc978b32ecf63404e31d", + "reference": "f3202fa1b5097b0af062dc978b32ecf63404e31d", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.7-dev" + } + }, + "autoload": { + "files": [ + "function.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "A generic function and convention to trigger deprecation notices", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.7.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-06-05T06:23:12+00:00" + }, + { + "name": "symfony/polyfill-ctype", + "version": "v1.37.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-ctype.git", + "reference": "141046a8f9477948ff284fa65be2095baafb94f2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/141046a8f9477948ff284fa65be2095baafb94f2", + "reference": "141046a8f9477948ff284fa65be2095baafb94f2", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "provide": { + "ext-ctype": "*" + }, + "suggest": { + "ext-ctype": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Gert de Pagter", + "email": "BackEndTea@gmail.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for ctype functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "ctype", + "polyfill", + "portable" + ], + "support": { + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.37.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-04-10T16:19:22+00:00" + }, + { + "name": "symfony/polyfill-intl-grapheme", + "version": "v1.41.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-intl-grapheme.git", + "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", + "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "suggest": { + "ext-intl": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Grapheme\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for intl's grapheme_* functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "grapheme", + "intl", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.41.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-07-28T08:25:59+00:00" + }, + { + "name": "symfony/polyfill-intl-normalizer", + "version": "v1.42.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-intl-normalizer.git", + "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/aa20edea75bd9c48cfecc8360922e5a6e5c44502", + "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "suggest": { + "ext-intl": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Normalizer\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for intl's Normalizer class and related functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "intl", + "normalizer", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.42.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-08-07T06:33:24+00:00" + }, + { + "name": "symfony/polyfill-mbstring", + "version": "v1.38.2", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-mbstring.git", + "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "shasum": "" + }, + "require": { + "ext-iconv": "*", + "php": ">=7.2" + }, + "provide": { + "ext-mbstring": "*" + }, + "suggest": { + "ext-mbstring": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for the Mbstring extension", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "mbstring", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-05-27T06:59:30+00:00" + }, + { + "name": "symfony/service-contracts", + "version": "v3.7.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/service-contracts.git", + "reference": "c0a284bab1ed8aa0417e3d69250ab437739563a0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/c0a284bab1ed8aa0417e3d69250ab437739563a0", + "reference": "c0a284bab1ed8aa0417e3d69250ab437739563a0", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "psr/container": "^1.1|^2.0", + "symfony/deprecation-contracts": "^2.5|^3" + }, + "conflict": { + "ext-psr": "<1.1|>=2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.7-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\Service\\": "" + }, + "exclude-from-classmap": [ + "/Test/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to writing services", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "support": { + "source": "https://github.com/symfony/service-contracts/tree/v3.7.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-06-16T09:55:08+00:00" + }, + { + "name": "symfony/string", + "version": "v8.0.15", + "source": { + "type": "git", + "url": "https://github.com/symfony/string.git", + "reference": "1a6a4245943af4dabe57d269bd0903f9d140a15e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/string/zipball/1a6a4245943af4dabe57d269bd0903f9d140a15e", + "reference": "1a6a4245943af4dabe57d269bd0903f9d140a15e", + "shasum": "" + }, + "require": { + "php": ">=8.4", + "symfony/polyfill-ctype": "^1.8", + "symfony/polyfill-intl-grapheme": "^1.33", + "symfony/polyfill-intl-normalizer": "^1.0", + "symfony/polyfill-mbstring": "^1.0" + }, + "conflict": { + "symfony/translation-contracts": "<2.5" + }, + "require-dev": { + "symfony/emoji": "^7.4|^8.0", + "symfony/http-client": "^7.4|^8.0", + "symfony/intl": "^7.4|^8.0", + "symfony/translation-contracts": "^2.5|^3.0", + "symfony/var-exporter": "^7.4|^8.0" + }, + "type": "library", + "autoload": { + "files": [ + "Resources/functions.php" + ], + "psr-4": { + "Symfony\\Component\\String\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides an object-oriented API to strings and deals with bytes, UTF-8 code points and grapheme clusters in a unified way", + "homepage": "https://symfony.com", + "keywords": [ + "grapheme", + "i18n", + "string", + "unicode", + "utf-8", + "utf8" + ], + "support": { + "source": "https://github.com/symfony/string/tree/v8.0.15" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-07-28T07:34:23+00:00" + }, + { + "name": "twig/twig", + "version": "v3.28.0", + "source": { + "type": "git", + "url": "https://github.com/twigphp/Twig.git", + "reference": "597c12ed286fb9d1701a36684ce6e0cbe28ebc8b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/597c12ed286fb9d1701a36684ce6e0cbe28ebc8b", + "reference": "597c12ed286fb9d1701a36684ce6e0cbe28ebc8b", + "shasum": "" + }, + "require": { + "php": ">=8.1.0", + "symfony/deprecation-contracts": "^2.5|^3", + "symfony/polyfill-ctype": "^1.8", + "symfony/polyfill-mbstring": "^1.3" + }, + "require-dev": { + "php-cs-fixer/shim": "^3.0@stable", + "phpstan/phpstan": "^2.0@stable", + "psr/container": "^1.0|^2.0", + "symfony/phpunit-bridge": "^5.4.9|^6.4|^7.0" + }, + "type": "library", + "autoload": { + "files": [ + "src/Resources/core.php", + "src/Resources/debug.php", + "src/Resources/escaper.php", + "src/Resources/string_loader.php" + ], + "psr-4": { + "Twig\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com", + "homepage": "http://fabien.potencier.org", + "role": "Lead Developer" + }, + { + "name": "Twig Team", + "role": "Contributors" + }, + { + "name": "Armin Ronacher", + "email": "armin.ronacher@active-4.com", + "role": "Project Founder" + } + ], + "description": "Twig, the flexible, fast, and secure template language for PHP", + "homepage": "https://twig.symfony.com", + "keywords": [ + "templating" + ], + "support": { + "issues": "https://github.com/twigphp/Twig/issues", + "source": "https://github.com/twigphp/Twig/tree/v3.28.0" + }, + "funding": [ + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/twig/twig", + "type": "tidelift" + } + ], + "time": "2026-07-03T20:44:34+00:00" + } + ], + "packages-dev": [], + "aliases": [], + "minimum-stability": "dev", + "stability-flags": {}, + "prefer-stable": true, + "prefer-lowest": false, + "platform": {}, + "platform-dev": {}, + "platform-overrides": { + "php": "8.4.0" + }, + "plugin-api-version": "2.9.0" +} diff --git a/tools/phplrt/fuzz.php b/tools/phplrt/fuzz.php new file mode 100644 index 00000000..4a209846 --- /dev/null +++ b/tools/phplrt/fuzz.php @@ -0,0 +1,147 @@ + [count] [seed] + * + * @internal this is a development tool, not part of the library + */ + +declare(strict_types=1); + +use PHPStan\PhpDocParser\Lexer\Lexer; +use PHPStan\PhpDocParser\ParserConfig; +use PHPStan\PhpDocParser\Tools\Fuzzer\Generator; +use PHPStan\PhpDocParser\Tools\Fuzzer\TokenSource; +use PHPStan\PhpDocParser\Tools\Fuzzer\TokenStream; +use PHPStan\PhpDocParser\Tools\Fuzzer\TokenStreamLexer; +use Phplrt\Compiler\Compiler; +use Phplrt\Contracts\Lexer\Channel; +use Phplrt\Parser\Analysis\Mode; +use Phplrt\Parser\Analysis\Result\PartialResult; +use Phplrt\Parser\Analysis\Result\SuccessfulResult; +use Phplrt\Source\FileSource; + +require __DIR__ . '/../../vendor/autoload.php'; +require __DIR__ . '/vendor/autoload.php'; +require __DIR__ . '/Fuzzer/TokenStream.php'; +require __DIR__ . '/Fuzzer/Generator.php'; + +/** + * How many times an input is written before the tool gives up on a grammar it + * cannot write anything readable from. + */ +const ATTEMPTS_PER_INPUT = 40; + +$grammar = $argv[1] ?? null; +$directory = $argv[2] ?? null; +$count = (int) ($argv[3] ?? 1000); +$seed = isset($argv[4]) ? (int) $argv[4] : null; + +if ($grammar === null || $directory === null) { + \fwrite(\STDERR, "Usage: php tools/phplrt/fuzz.php [count] [seed]\n"); + + exit(1); +} + +\mt_srand($seed ?? \random_int(0, \PHP_INT_MAX)); + +$compiled = new Compiler() + ->load(FileSource::createFromPathname($grammar)) + ->build(); + +/** @var array $ids */ +$ids = \array_flip($compiled->lexer->names); + +$stream = new TokenStream($ids); +$parser = $compiled->parser->toParser(new TokenStreamLexer()); +$lexer = new Lexer(new ParserConfig([])); + +$generator = new Generator( + $compiled->parser->grammar, + $compiled->lexer->names, + $compiled->parser->initial, +); + +if (!\is_dir($directory) && !\mkdir($directory, 0777, true)) { + \fwrite(\STDERR, \sprintf("Cannot create %s\n", $directory)); + + exit(1); +} + +foreach ((array) \glob($directory . '/*.tst') as $stale) { + \unlink((string) $stale); +} + +$written = 0; +$attempts = 0; + +while ($written < $count && $attempts < $count * ATTEMPTS_PER_INPUT) { + $attempts++; + + $tokens = $generator->generate(); + + if ($tokens === null) { + continue; + } + + $text = $generator->render($tokens); + $read = $stream->create($lexer->tokenize($text)); + + // The text has to read back as the very tokens it was written of + $actual = []; + + foreach ($read as $token) { + if ($token->channel === Channel::EndOfInput) { + continue; + } + + $actual[] = $token->id; + } + + if ($generator->written($tokens) !== $actual) { + continue; + } + + // And the grammar has to recognize it, all of it + $result = $parser->analyze(new TokenSource($read), Mode::SyntaxCheck); + + if (!$result instanceof SuccessfulResult || $result instanceof PartialResult) { + continue; + } + + \file_put_contents(\sprintf('%s/%04d.tst', $directory, $written), $text); + $written++; +} + +if ($written === 0) { + \fwrite(\STDERR, \sprintf("Nothing could be written from %s\n", $grammar)); + + exit(1); +} + +echo $written, "\n";