diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 569d88555..e1f08d5e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,28 @@ jobs: - name: Check for grammar ambiguities run: ./gradlew checkGrammarAmbiguity --no-daemon + - name: Restore and save lookahead facts + uses: actions/cache@v5 + with: + path: build/lookahead-cache/ + key: lookahead-facts-${{ runner.os }}-${{ hashFiles('src/test/java/net/sf/jsqlparser/parser/audit/**', 'build.gradle', 'gradle/wrapper/gradle-wrapper.properties') }}-${{ github.sha }} + restore-keys: | + lookahead-facts-${{ runner.os }}-${{ hashFiles('src/test/java/net/sf/jsqlparser/parser/audit/**', 'build.gradle', 'gradle/wrapper/gradle-wrapper.properties') }}- + + - name: Report changed lookahead contexts + env: + LOOKAHEAD_BASE: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} + run: ./gradlew lookaheadAudit --no-daemon -PlookaheadBase="$LOOKAHEAD_BASE" + + - name: Upload lookahead review report + if: always() + uses: actions/upload-artifact@v5 + with: + name: lookahead-review + path: build/reports/lookahead/ + retention-days: 7 + if-no-files-found: ignore + - name: Upload grammar warning report if: failure() uses: actions/upload-artifact@v5 @@ -216,4 +238,4 @@ jobs: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v5 \ No newline at end of file + uses: actions/deploy-pages@v5 diff --git a/build.gradle b/build.gradle index a853a8c71..d0102230b 100644 --- a/build.gradle +++ b/build.gradle @@ -265,7 +265,7 @@ tasks.register('xmldoc', Javadoc) { } test { - environment = [ 'EXPORT_TEST_TO_FILE': 'False' ] + environment 'EXPORT_TEST_TO_FILE', 'False' useJUnitPlatform() // set heap size for the test JVM(s) @@ -846,3 +846,24 @@ tasks.register('checkGrammarAmbiguity') { tasks.named('check') { dependsOn('checkGrammarAmbiguity') } + +// Report base/working-tree context changes without making historical lookahead debt a new gate. +// The analyzer lives in test sources: no audit classes or dependencies enter the library JAR. +tasks.register('lookaheadAudit', JavaExec) { + group = 'verification' + description = 'Reports changed lookahead decisions and stale evidence against a Git base.' + dependsOn(testClasses) + classpath = sourceSets.test.runtimeClasspath + mainClass = 'net.sf.jsqlparser.parser.audit.LookaheadAudit' + javaLauncher = javaToolchains.launcherFor(java.toolchain) + jvmArgs '-Xss4m', '-Xmx1g', '-Dfile.encoding=UTF-8' + workingDir = projectDir + doFirst { + args = [projectDir.absolutePath, + providers.gradleProperty('lookaheadBase').getOrElse('HEAD'), + layout.buildDirectory.dir('reports/lookahead').get().asFile.absolutePath, + file('config/lookahead-evidence.tsv').absolutePath, + providers.gradleProperty('lookaheadFailOnFindings').getOrElse('false'), + providers.gradleProperty('lookaheadFull').getOrElse('false')] + } +} diff --git a/config/lookahead-evidence.tsv b/config/lookahead-evidence.tsv new file mode 100644 index 000000000..5f478cc11 --- /dev/null +++ b/config/lookahead-evidence.tsv @@ -0,0 +1,3 @@ +# keyfingerprintreasonregression test[context|token-probe] +ConnectionOrCloudConnectionDefinition eb5720afb73b771120122d76c643490a1d23e8751f3d516026d96a2e807e0f2a AT conn and AT CLOUD share the first token; retain the second-token decision. LookaheadCleanupTest.preservesCloudAndOrdinaryConnections +LockStatement e95699a7c5c185663cd0606f88a5613ace1e51f6e2e0d61d2a18ac148b16426d Bound the complete TABLE token probe to one token while preserving its restriction; this certificate does not cover the complete lock AST. LookaheadTokenProbeTest and LookaheadCleanupTest.boundedTableKeywordKeepsLockTargetAndWait token-probe diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index f49b3ddba..11125fe27 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -4159,7 +4159,7 @@ Execute OracleImplicitCall(): name=ColumnIdentifier() { call.setName(name.getNames()); call.setExecType(Execute.ExecType.IMPLICIT); requireDdlSyntax(!Arrays.asList("RAISE", "RETURN", "GOTO", "EXIT", "CONTINUE", "LOOP", "WHILE", "FOR") .contains(call.getName().toUpperCase(java.util.Locale.ROOT)), "Unsupported PL/SQL control statement"); } - [ LOOKAHEAD("(") arguments=ExpressionList() { call.setExprList(arguments); } ] + [ LOOKAHEAD(1, "(") arguments=ExpressionList() { call.setExprList(arguments); } ] { return call; } } @@ -4398,7 +4398,7 @@ LockStatement LockStatement(): { } { { statement.setUseTableKeyword(false); } - [ LOOKAHEAD() { statement.setUseTableKeyword(true); } ] + [ LOOKAHEAD(1, ) { statement.setUseTableKeyword(true); } ] target=LockTarget() { statement.getTargets().add(target); } ( "," target=LockTarget() { statement.getTargets().add(target); } )* [ mode=TableLockMode() { statement.setLockMode(mode); } ] @@ -4624,7 +4624,6 @@ DBMSDestination DBMSDestination() #DBMSDestination: { connectionDefinition = ConnectionDefinition() { dbmsDestination.setConnectionDefinition(connectionDefinition); } ( - LOOKAHEAD(3) table = Table() { dbmsDestination.setTable(table); } [ LOOKAHEAD(2) columns = ParenthesedColumnList() { dbmsDestination.setColumns(columns); } ] [ LOOKAHEAD(2) dbmsTableDestinationOptions = DBMSTableDestinationOptionList() { dbmsDestination.setDBMSTableDestinationOptions(dbmsTableDestinationOptions); } ] @@ -7701,9 +7700,9 @@ Select Select() #Select: [ LOOKAHEAD(2) interpolateElements = InterpolateClause() { select.setInterpolate(interpolateElements); } ] [ LOOKAHEAD(2, { !(select instanceof Values) }) optionClause = OptionClause() { select.setOption(optionClause); } ] - [ LOOKAHEAD() limit=LimitWithOffset() {select.setLimit(limit);} ] - [ LOOKAHEAD() offset = Offset() { select.setOffset(offset);} ] - [ LOOKAHEAD() fetch = Fetch() { select.setFetch(fetch);} ] + [ LOOKAHEAD(1, ) limit=LimitWithOffset() {select.setLimit(limit);} ] + [ LOOKAHEAD(1, ) offset = Offset() { select.setOffset(offset);} ] + [ LOOKAHEAD(1, ) fetch = Fetch() { select.setFetch(fetch);} ] [ LOOKAHEAD( ) withIsolation = WithIsolation() { select.setIsolation(withIsolation);} ] ) ) @@ -7752,8 +7751,8 @@ PlainSelect FromFirstSelect() #FromFirstSelect: [ LOOKAHEAD(2) qualify=Qualify() { plainSelect.setQualify(qualify); } ] [ LOOKAHEAD( ) orderByElements=OrderByElements() { plainSelect.setOrderByElements(orderByElements); } ] - [ LOOKAHEAD() limit=LimitWithOffset() { plainSelect.setLimit(limit); } ] - [ LOOKAHEAD() offset=Offset() { plainSelect.setOffset(offset); } ] + [ LOOKAHEAD(1, ) limit=LimitWithOffset() { plainSelect.setLimit(limit); } ] + [ LOOKAHEAD(1, ) offset=Offset() { plainSelect.setOffset(offset); } ] { linkAST(plainSelect, jjtThis); return plainSelect; @@ -8157,8 +8156,8 @@ TableStatement TableStatement(): table = Table() { tableStatement.setTable(table); } [ LOOKAHEAD( ) orderByElements = OrderByElements() { tableStatement.setOrderByElements(orderByElements); } ] - [ LOOKAHEAD() limit = LimitWithOffset() { tableStatement.setLimit(limit);} ] - [ LOOKAHEAD() offset = Offset() { tableStatement.setOffset(offset);} ] + [ LOOKAHEAD(1, ) limit = LimitWithOffset() { tableStatement.setLimit(limit);} ] + [ LOOKAHEAD(1, ) offset = Offset() { tableStatement.setOffset(offset);} ] { return tableStatement; } /* Support operationList */ } @@ -8484,7 +8483,7 @@ PlainSelect PlainSelect() #PlainSelect: selectItems=SelectItemsList() - [ LOOKAHEAD() + [ LOOKAHEAD(1, ) ( LOOKAHEAD({ isMySqlSelectIntoAhead() }) mySqlSelectIntoClause = MySqlSelectIntoClause(MySqlSelectIntoClause.Position.BEFORE_FROM) @@ -8546,8 +8545,8 @@ PlainSelect PlainSelect() #PlainSelect: // optionally attach ClickHouse's `LIMIT ... BY ...`. Checking BY right after the limit // expression avoids a numeric LOOKAHEAD, which cannot see past a long parenthesized // subquery and would wrongly commit to LIMIT BY (issue #2359). - [ LOOKAHEAD() limit = LimitWithOffset() - [ LOOKAHEAD() expressionList = ExpressionList() { limit.setByExpressions(expressionList); } ] + [ LOOKAHEAD(1, ) limit = LimitWithOffset() + [ LOOKAHEAD(1, ) expressionList = ExpressionList() { limit.setByExpressions(expressionList); } ] { if (limit.getByExpressions() != null) { plainSelect.setLimitBy(limit); @@ -8555,13 +8554,13 @@ PlainSelect PlainSelect() #PlainSelect: plainSelect.setLimit(limit); } } ] - [ LOOKAHEAD() offset = Offset() { plainSelect.setOffset(offset); } ] + [ LOOKAHEAD(1, ) offset = Offset() { plainSelect.setOffset(offset); } ] [ LOOKAHEAD(, { limit==null }) limit = LimitWithOffset() { plainSelect.setLimit(limit); } ] [ LOOKAHEAD( ) mySqlProcedureAnalyse = MySqlProcedureAnalyse() { plainSelect.setMySqlProcedureAnalyse(mySqlProcedureAnalyse); } ] - [ LOOKAHEAD() fetch = Fetch() { plainSelect.setFetch(fetch); } ] + [ LOOKAHEAD(1, ) fetch = Fetch() { plainSelect.setFetch(fetch); } ] [ LOOKAHEAD( ) withIsolation = WithIsolation() { plainSelect.setIsolation(withIsolation); } ] [ LOOKAHEAD(2) @@ -8578,7 +8577,7 @@ PlainSelect PlainSelect() #PlainSelect: ( LOOKAHEAD(2) "," updateTable = Table() { updateTables.add(updateTable); } )* { plainSelect.setForUpdateTables(updateTables); } ] - [ LOOKAHEAD() wait = Wait() { plainSelect.setWait(wait); } ] + [ LOOKAHEAD(1, ) wait = Wait() { plainSelect.setWait(wait); } ] [ LOOKAHEAD(2) ( { plainSelect.setNoWait(true); } | { plainSelect.setSkipLocked(true); }) ] [ LOOKAHEAD( ) orderByElements = OrderByElements() @@ -8596,7 +8595,7 @@ PlainSelect PlainSelect() #PlainSelect: { plainSelect.setMySqlSelectIntoClause(mySqlSelectIntoClause); } ] [ LOOKAHEAD(2) settings = UpdateSets() { plainSelect.setSettings(settings); } ] - [ LOOKAHEAD() optimize = OptimizeFor() { plainSelect.setOptimizeFor(optimize); } ] + [ LOOKAHEAD(1, ) optimize = OptimizeFor() { plainSelect.setOptimizeFor(optimize); } ] [ LOOKAHEAD(3) intoTempTable = Table() { plainSelect.setIntoTempTable(intoTempTable);} ] [ LOOKAHEAD(3) { plainSelect.setUseWithNoLog(true); } ] { @@ -8669,10 +8668,10 @@ Select SetOperationList(Select select) #SetOperationList: { [ LOOKAHEAD(2) orderByElements=OrderByElements() {list.setOrderByElements(orderByElements);} ] [ LOOKAHEAD(2) interpolateElements = InterpolateClause() { list.setInterpolate(interpolateElements); } ] - [ LOOKAHEAD() limit = LimitWithOffset() { list.setLimit(limit); } ] - [ LOOKAHEAD() offset = Offset() { list.setOffset(offset); } ] + [ LOOKAHEAD(1, ) limit = LimitWithOffset() { list.setLimit(limit); } ] + [ LOOKAHEAD(1, ) offset = Offset() { list.setOffset(offset); } ] [ LOOKAHEAD(, { limit==null }) limit = LimitWithOffset() { list.setLimit(limit); } ] - [ LOOKAHEAD() fetch = Fetch() { list.setFetch(fetch); } ] + [ LOOKAHEAD(1, ) fetch = Fetch() { list.setFetch(fetch); } ] [ LOOKAHEAD( ) withIsolation = WithIsolation() { list.setIsolation(withIsolation); } ] { @@ -9152,7 +9151,7 @@ ColumnsTransformer ColumnsTransformer(): | { transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.RENAME); } ( - LOOKAHEAD("(") "(" replaceItems = SelectItemsList() ")" + LOOKAHEAD(1, "(") "(" replaceItems = SelectItemsList() ")" | item = SelectItem() { @@ -9625,21 +9624,21 @@ void MySqlSelectIntoOutfileTail(MySqlSelectIntoClause intoClause): (token= | token=) { intoClause.setCharacterSet(token.image); } ( - LOOKAHEAD(( | )) + LOOKAHEAD(1, ( | )) MySqlSelectIntoFieldsClause(intoClause) - [ LOOKAHEAD() MySqlSelectIntoLinesClause(intoClause) ] + [ LOOKAHEAD(1, ) MySqlSelectIntoLinesClause(intoClause) ] | - LOOKAHEAD() + LOOKAHEAD(1, ) MySqlSelectIntoLinesClause(intoClause) | { } ) | - LOOKAHEAD(( | )) + LOOKAHEAD(1, ( | )) MySqlSelectIntoFieldsClause(intoClause) - [ LOOKAHEAD() MySqlSelectIntoLinesClause(intoClause) ] + [ LOOKAHEAD(1, ) MySqlSelectIntoLinesClause(intoClause) ] | - LOOKAHEAD() + LOOKAHEAD(1, ) MySqlSelectIntoLinesClause(intoClause) | { } @@ -9660,7 +9659,7 @@ void MySqlSelectIntoFieldsClause(MySqlSelectIntoClause intoClause): token= { intoClause.setFieldsTerminatedBy(stringValue(token)); } ( - LOOKAHEAD(( | )) + LOOKAHEAD(1, ( | )) [ { intoClause.setFieldsOptionallyEnclosed(true); } ] token= { intoClause.setFieldsEnclosedBy(stringValue(token)); } @@ -9676,7 +9675,7 @@ void MySqlSelectIntoFieldsClause(MySqlSelectIntoClause intoClause): { } ) | - LOOKAHEAD(( | )) + LOOKAHEAD(1, ( | )) [ { intoClause.setFieldsOptionallyEnclosed(true); } ] token= { intoClause.setFieldsEnclosedBy(stringValue(token)); } @@ -10641,7 +10640,7 @@ OrderByElement OrderByElement(): } { columnReference = Expression() - [ LOOKAHEAD() (collateToken= | collateToken=) { columnReference = new CollateExpression(columnReference, collateToken.image); } ] + [ LOOKAHEAD(1, ) (collateToken= | collateToken=) { columnReference = new CollateExpression(columnReference, collateToken.image); } ] [ LOOKAHEAD(2) ( | ( { orderByElement.setAsc(false); } )) { orderByElement.setAscDescPresent(true); } ] [ LOOKAHEAD(2) [ LOOKAHEAD(2) ( @@ -10740,9 +10739,9 @@ OptionHint OptionHint(): name = OptionHintName() { optionHint.setName(name); } [ ( - LOOKAHEAD("=") "=" value = Expression() { optionHint.setValue(value); optionHint.setUseEquals(true); } + LOOKAHEAD(1, "=") "=" value = Expression() { optionHint.setValue(value); optionHint.setUseEquals(true); } | - LOOKAHEAD("(") "(" parameter = OptionHintParameter() { optionHint.addParameter(parameter); } + LOOKAHEAD(1, "(") "(" parameter = OptionHintParameter() { optionHint.addParameter(parameter); } ( "," parameter = OptionHintParameter() { optionHint.addParameter(parameter); } )* ")" | @@ -11581,7 +11580,7 @@ Expression IsJsonExpression(Expression leftExpression): [ LOOKAHEAD(2) ( { result.setUniqueKeys(IsJsonExpression.UniqueKeys.WITH); } | { result.setUniqueKeys(IsJsonExpression.UniqueKeys.WITHOUT); } ) { result.setUseKeysKeyword(false); } - [ LOOKAHEAD() { result.setUseKeysKeyword(true); } ] ] + [ LOOKAHEAD(1, ) { result.setUseKeysKeyword(true); } ] ] { result.setLeftExpression(leftExpression); return result; } } @@ -12263,7 +12262,7 @@ Expression PrimaryExpression() #PrimaryExpression: { retval = new FunctionAllColumns(unwrapParenthesedFunction(retval)); } ] // Share field and method navigation with the other primary expressions. - ( LOOKAHEAD(".") nxt=NavigationStep(retval) { retval = nxt; } )* + ( LOOKAHEAD(1, ".") nxt=NavigationStep(retval) { retval = nxt; } )* ) ) @@ -14014,7 +14013,7 @@ ExpressionList ExecuteArguments(): { } { ( - LOOKAHEAD("(") arguments=ExpressionList() + LOOKAHEAD(1, "(") arguments=ExpressionList() | argument=ExecuteArgument() { arguments.add(argument); } ( LOOKAHEAD(2) "," argument=ExecuteArgument() { arguments.add(argument); } )* @@ -15166,7 +15165,7 @@ String PostgreSqlStorageParameterName(): { String name; String part; } { name=RelObjectName() - ( LOOKAHEAD(".") "." part=RelObjectNameExt() { name += "." + part; } )* + ( LOOKAHEAD(1, ".") "." part=RelObjectNameExt() { name += "." + part; } )* { return name; } } @@ -15429,15 +15428,15 @@ ColumnOption ColumnDefinitionOption(): { | LOOKAHEAD( ) { option = ColumnOption.nullability(false); } | - LOOKAHEAD() { option = ColumnOption.nullability(true); } + LOOKAHEAD(1, ) { option = ColumnOption.nullability(true); } | - LOOKAHEAD() { option = ColumnOption.visibility(true); } + LOOKAHEAD(1, ) { option = ColumnOption.visibility(true); } | - LOOKAHEAD() { option = ColumnOption.visibility(false); } + LOOKAHEAD(1, ) { option = ColumnOption.visibility(false); } | - LOOKAHEAD() { option = ColumnOption.autoIncrement(); } + LOOKAHEAD(1, ) { option = ColumnOption.autoIncrement(); } | - LOOKAHEAD() ( tk= | tk= | tk= ) + LOOKAHEAD(1, ) ( tk= | tk= | tk= ) { option = ColumnOption.storage(accessEnum(ColumnOption.Storage.class, tk.image)); } | LOOKAHEAD({ isKeywordAhead("COMPRESSION") }) ContextualKeyword("COMPRESSION") compression=RelObjectName() @@ -15463,12 +15462,12 @@ ColumnOption ColumnDefinitionOption(): { option.setUseEquals(useEquals); } | - LOOKAHEAD() + LOOKAHEAD(1, ) ( tk= { collation = tk.image; } | collationNames=RelObjectNames() { collation = String.join(".", collationNames.getNames()); } ) { option = ColumnOption.collate(collation); } | - LOOKAHEAD() tk= + LOOKAHEAD(1, ) tk= { option = ColumnOption.comment(stringValue(tk)); } | LOOKAHEAD( ) defaultExpression=Expression() @@ -15501,7 +15500,7 @@ ColumnOption ColumnDefinitionOption(): { LOOKAHEAD( ) { option = ColumnOption.raw("DEFAULT", "ON", "NULL"); } | - LOOKAHEAD() { option = ColumnOption.raw("DEFAULT"); } + LOOKAHEAD(1, ) { option = ColumnOption.raw("DEFAULT"); } // Db2 permits an omitted default value. Keep these implicit defaults raw. [ LOOKAHEAD(2, { !(getToken(1).kind == K_NOT && getToken(2).kind == K_NULL) }) defaultExpression=Expression() { option = ColumnOption.defaultValue(defaultExpression); } ] @@ -16748,7 +16747,7 @@ CreateTable CreateTable(boolean isUsingOrReplace): tk.image.toUpperCase(Locale.ROOT))); } ] [ { createTable.setUseAsKeyword(true); } ] ( - LOOKAHEAD() execute=Execute() { + LOOKAHEAD(1, ) execute=Execute() { requireDdlSyntax(createTable.isUseAsKeyword() && createTable.getDuplicateHandling() == null, "CREATE TABLE EXECUTE requires AS and cannot use duplicate handling"); createTable.setExecute(execute); @@ -16826,7 +16825,7 @@ ForeignDataOption ForeignDataOption(): } { ( - LOOKAHEAD() name=RelObjectName() + LOOKAHEAD(1, ) name=RelObjectName() { option = new ForeignDataOption(name, null); option.setAction(ForeignDataOption.Action.DROP); } | [ LOOKAHEAD(1) ( { action = ForeignDataOption.Action.ADD; } @@ -16841,7 +16840,7 @@ TableElement CreateTableElement(boolean typed): { TableElement element; } { ( - LOOKAHEAD() element=LikeClause() + LOOKAHEAD(1, ) element=LikeClause() | LOOKAHEAD(3) element=CreateTableConstraint() | element=CreateTableColumnDefinition(typed) ) @@ -17294,7 +17293,7 @@ String LargeObjectTypeName(): Token object; } { - [ LOOKAHEAD() national= ] + [ LOOKAHEAD(1, ) national= ] ( type= | type= | type= | type= ) large= object= { return (national == null ? "" : national.image + " ") @@ -17470,7 +17469,7 @@ ColDataType ColDataType(): ( LOOKAHEAD(1) typeModifier=MySqlTypeModifier() { colDataType.addTypeModifier(typeModifier); } )* ( - LOOKAHEAD() { tk = null; } + LOOKAHEAD(1, ) { tk = null; } [ LOOKAHEAD(2) "[" tk= "]" ] { array.add(tk == null ? null : Integer.valueOf(tk.image)); colDataType.setArrayData(array); @@ -17975,7 +17974,7 @@ void AlterExpressionUsingIndex(AlterExpression alterExp): { alterExp.addParameters("USING"); } [ LOOKAHEAD(2) { alterExp.addParameters("INDEX"); } ] ( - LOOKAHEAD(2, ) sk4=RelObjectName() + LOOKAHEAD(1, ) sk4=RelObjectName() { alterExp.addParameters("TABLESPACE"); alterExp.addParameters(sk4); } | sk4=RelObjectName() { alterExp.addParameters(sk4); } @@ -19961,7 +19960,7 @@ Statement Alter(): | LOOKAHEAD({ isKeywordAhead("STATISTICS") }) statement = AlterStatistics() | - LOOKAHEAD() statement = AlterPolicy() + LOOKAHEAD(1, ) statement = AlterPolicy() | LOOKAHEAD({ isKeywordAhead("ROLE") || isKeywordAhead("USER") || isKeywordAhead("GROUP") }) statement = AlterRole() @@ -19975,11 +19974,11 @@ Statement Alter(): // Other dialects keep unmodelled ALTER DATABASE forms as unsupported statements. LOOKAHEAD(AlterDatabase()) statement = AlterDatabase() | - LOOKAHEAD() statement = AlterSchema() + LOOKAHEAD(1, ) statement = AlterSchema() | - LOOKAHEAD() statement = AlterType() + LOOKAHEAD(1, ) statement = AlterType() | - LOOKAHEAD() statement = AlterDomain() + LOOKAHEAD(1, ) statement = AlterDomain() | LOOKAHEAD({ isKeywordAhead("EXTENSION") }) statement = AlterExtension() | @@ -20200,7 +20199,7 @@ RelationAlterAction PostgreSqlRelationAction(): action.setStatistics(statistics); } ) | - LOOKAHEAD("(") options=PostgreSqlIndexOptions() { + LOOKAHEAD(1, "(") options=PostgreSqlIndexOptions() { for (IndexOption option : options) { requireDdlSyntax(option.getValue() != null && option.isUseEquals(), "Column attributes require a value and an equals sign"); @@ -21325,9 +21324,9 @@ Statement Create(): | LOOKAHEAD({ isPostgreSqlTriggerAhead() }) statement = PostgreSqlTrigger(isUsingOrReplace) | - LOOKAHEAD() { requireTypeDdlSyntax(!isUsingOrReplace, "OR REPLACE is not supported for CREATE TYPE"); } statement = CreateType() + LOOKAHEAD(1, ) { requireTypeDdlSyntax(!isUsingOrReplace, "OR REPLACE is not supported for CREATE TYPE"); } statement = CreateType() | - LOOKAHEAD() { requireTypeDdlSyntax(!isUsingOrReplace, "OR REPLACE is not supported for CREATE DOMAIN"); } statement = CreateDomain() + LOOKAHEAD(1, ) { requireTypeDdlSyntax(!isUsingOrReplace, "OR REPLACE is not supported for CREATE DOMAIN"); } statement = CreateDomain() | LOOKAHEAD({ isKeywordAhead("EXTENSION") && "REPOSITORY".equalsIgnoreCase(getToken(2).image) }) @@ -22844,7 +22843,7 @@ CreateRule CreateRule(boolean orReplace): ( { statement.setNothing(true); } | LOOKAHEAD(Select()) action=Select() { statement.getActions().add(action); } - | LOOKAHEAD("(") "(" { statement.setUseParentheses(true); } + | LOOKAHEAD(1, "(") "(" { statement.setUseParentheses(true); } [ LOOKAHEAD({ getToken(1).kind != ST_SEMICOLON && !")".equals(getToken(1).image) }) action=PostgreSqlRuleAction() { statement.getActions().add(action); } ] ( diff --git a/src/site/sphinx/contribution.rst b/src/site/sphinx/contribution.rst index 41c327fbb..3f1f40e75 100644 --- a/src/site/sphinx/contribution.rst +++ b/src/site/sphinx/contribution.rst @@ -120,6 +120,37 @@ When adding a new keyword token to the Grammar: 2) If the keyword must be reserved (e. |_| g. core SQL syntax like ``SELECT``, ``FROM``, ``WHERE``), add it to the ``Reserved SQL Keywords`` TOKEN block **after** the ``MAX_NON_RESERVED_WORD`` sentinel. 3) Verify that existing tests pass and that the keyword can be used as a ``Schema``, ``Table``, ``Column``, ``Function`` or ``Alias`` name where expected. +Review Lookahead Changes +------------------------------ + +Run the reusable lookahead report against the commit used as the review baseline: + +.. code-block:: shell + + ./gradlew lookaheadAudit -PlookaheadBase=HEAD + +The default baseline is ``HEAD``, so this command also examines uncommitted grammar changes. CI uses the pull request base commit. Results are written to ``build/reports/lookahead/report.json``. The analyzer runs with the project's existing JDK and JavaCC dependencies; it does not require Python or modify sources. + +The report includes ``inventory.tsv`` and ``inventory.json`` for every explicit decision, plus ``report.json`` for changed contexts, dependency roots and stale evidence. Structural anchors ignore source line numbers and unrelated lookahead insertions. Identical anchors remain ambiguous and cannot reuse an individual evidence record. Triage is a review aid, not permission to remove a guard. + +Facts are cached by the complete grammar source and analyzer/generator identity under ``build/lookahead-cache``. A content hit in a separate JVM runs neither JJTree nor JavaCC parsing nor FIRST/FOLLOW iteration. External Java/resource/build fingerprints are rebound on every invocation, including cache hits. CI restores and saves this directory separately from Gradle's dependency cache. Invalid, truncated or incompatible cache files cause a rebuild. These are local build artifacts, not an authorization or a semantic proof. + +A changed grammar still runs JJTree and a complete JavaCC parse and semantic check. The analyzer then recomputes changed FIRST components, call-site FOLLOW equations and affected decision rows; unchanged inputs reuse their facts. Components restart from bottom when invalidated, so deletions and shrinking sets cannot retain old tokens. Actual calls and calls inside probes have separate roles: runtime FOLLOW uses actual calls, while probe facts and symbolic continuation dependencies include probe calls. The ``analysisBefore`` and ``analysisCurrent`` statistics show front-end runs, recomputation and reuse. Use ``-PlookaheadFull=true`` to bypass this custom cache and incremental reuse for a cold comparison; Gradle's ``--no-build-cache`` alone does not disable it. + +Full context evidence includes actual callers, their complete callee closures (including probes), and symbolic continuations beyond FOLLOW1. Caller prefixes and competing alternatives can restrict which inputs reach a callee, so changes in a caller's other branches can conservatively invalidate evidence too. Reuse independent of the whole grammar is limited to an intentionally narrow, effect-free grammar subset. Unknown Java, assignments, signatures, declarations, predicates, recovery, lexer actions, lexical states/transitions and custom runtime hooks retain whole-grammar fallback. JSqlParser's production grammar contains opaque Java, so an unrelated production edit can invalidate full context evidence even when its FIRST/FOLLOW rows are reused. + +``config/lookahead-evidence.tsv`` has four columns: key, fingerprint, reason and regression reference. A fifth column explicitly selects ``context`` (the default) or ``token-probe``. Existing four-column records keep their full context meaning. A token certificate is available only for one complete, positive, unnested single-token probe or token set with no Java side effects or predicate. It binds the token decision, complete lexer/options, generator/analyzer identity and external inputs. It can survive an unrelated production edit, but certifies only that local token restriction; it cannot stand in for AST, continuation, error handling or complete cleanup validation. Unknown scope names and unsupported token certificates fail analysis. + +Production Java/resources, Maven configuration under ``.mvn/``, Gradle configuration under ``gradle/``, root build files, wrappers, the dedicated cleanup regression and analyzer sources are watched. Full token definitions, including named inline regexes, are bound. Unrecognized watched symlinks fail analysis. Commit-derived version metadata is excluded. Evidence covers tracked static inputs and tested behavior, not every possible runtime configuration. + +The report writes current unambiguous fingerprints to ``binding-candidates.tsv``. Review the scope and rerun the referenced tests before updating a stale record. A production name is accepted only when it identifies one decision; otherwise use a unique structural anchor or disambiguate the case. Never copy a narrower fingerprint into a full-context record to silence invalidation. + +Complete one-token syntactic probes can be bounded as ``LOOKAHEAD(1, token)``. Keep the explicit restriction: deleting the probe can widen a called production's FIRST set, and nested lookaheads have separate generator rules. Non-nullable branches with disjoint FIRST sets can suggest default dispatch, but imperative JavaCC code still needs review. Retain state/dialect/interruption checks, verify complete consumption and AST behavior, and test malformed neighbors. This change bounds 57 complete token probes and removes one investigated redundant depth-three dispatch in ``DBMSDestination``; it does not remove every historical lookahead. + +Error locations can change when a rejected branch is entered directly. In ``DBMSDestination``, ``EXA AT conn TABLE`` remains invalid, but now reports the missing table name after consuming ``TABLE`` instead of falling through to the ``STATEMENT`` alternative. The dedicated regression checks this consumption boundary; exact diagnostic text is not promised unchanged. + +The report does not make historical debt a mandatory gate. ``-PlookaheadFailOnFindings=true`` opts into failure when contexts need review; generation or analysis errors always fail. Ordinary ``check`` runs cache, cold/incremental oracle, dependency, token-probe and parser regressions. Full applicable checks and paired project JMH measurements are still required before publishing a cleanup. + Commit a Pull Request --------------------------------- diff --git a/src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java b/src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java new file mode 100644 index 000000000..079c8c1f7 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java @@ -0,0 +1,163 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser; + +import static org.junit.jupiter.api.Assertions.*; +import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; +import net.sf.jsqlparser.statement.ConnectionDefinition; +import net.sf.jsqlparser.statement.CloudConnectionDefinition; +import net.sf.jsqlparser.statement.export.DBMSDestination; +import org.junit.jupiter.api.Test; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.MySqlSelectIntoClause; +import net.sf.jsqlparser.statement.execute.Execute; +import net.sf.jsqlparser.statement.lock.LockStatement; +import net.sf.jsqlparser.statement.create.rule.CreateRule; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.notify.NotifyStatement; +import net.sf.jsqlparser.expression.LongValue; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class LookaheadCleanupTest { + @ParameterizedTest + @ValueSource(strings = {"EXA AT conn TABLE t", "ORA AT conn TABLE schemaName.tableName", + "JDBC DRIVER = 'driverName' AT conn TABLE \"t\"", + "EXA AT conn TABLE t (c1, c2) REPLACE TRUNCATE"}) + void dispatchesTableDestinationsWithCompleteConsumption(String sql) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + DBMSDestination destination = parser.DBMSDestination(); + assertNotNull(destination.getTable()); + assertNull(destination.getStatement()); + assertNotNull(destination.getConnectionDefinition()); + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true, + p -> p.withDialect(Dialect.EXASOL)); + } + + @ParameterizedTest + @ValueSource(strings = {"EXA AT conn STATEMENT 'insert into t values (?)'", + "ORA AT conn STATEMENT 'insert into t values (?)'"}) + void preservesStatementDestinations(String sql) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + DBMSDestination destination = parser.DBMSDestination(); + assertNull(destination.getTable()); + assertEquals("insert into t values (?)", destination.getStatement().getValue()); + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO " + sql, true, + p -> p.withDialect(Dialect.EXASOL)); + } + + @ParameterizedTest + @ValueSource(strings = {"EXA AT conn TABLE", "ORA AT conn STATEMENT", "EXA AT conn SELECT t"}) + void rejectsIncompleteAndWrongDestinationPrefixes(String sql) { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + assertThrows(ParseException.class, parser::DBMSDestination); + if (sql.endsWith("TABLE")) { + assertEquals(CCJSqlParserConstants.K_TABLE, parser.getToken(0).kind); + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + } + } + + @ParameterizedTest + @ValueSource(strings = {"AT conn", "AT '127.0.0.1'", "AT CLOUD NONE conn", + "AT CLOUD AZURE BLOBSTORAGE conn"}) + void preservesCloudAndOrdinaryConnections(String sql) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + ConnectionDefinition connection = parser.ConnectionOrCloudConnectionDefinition(); + if (sql.startsWith("AT CLOUD")) { + assertInstanceOf(CloudConnectionDefinition.class, connection); + } else { + assertEquals(ConnectionDefinition.class, connection.getClass()); + } + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + assertSqlCanBeParsedAndDeparsed("EXPORT sourceTable INTO CSV " + sql + " FILE 'file.csv'", + true, p -> p.withDialect(Dialect.EXASOL)); + } + + @ParameterizedTest + @ValueSource(strings = {"SELECT id FROM t LIMIT 7 OFFSET 2", + "SELECT id FROM t UNION ALL SELECT id FROM u LIMIT 7 OFFSET 2"}) + void boundedSelectTailsKeepTheirAstAndFollowingStatement(String sql) throws Exception { + Select select = (Select) complete(sql); + assertEquals(7, ((LongValue) select.getLimit().getRowCount()).getValue()); + assertEquals(2, ((LongValue) select.getOffset().getOffset()).getValue()); + assertEquals("SELECT 42", CCJSqlParserUtil.parseStatements(sql + "; SELECT 42").get(1) + .toString()); + } + + @Test + void boundedFetchKeepsItsCount() throws Exception { + Select select = (Select) complete("SELECT id FROM t OFFSET 2 ROWS FETCH NEXT 7 ROWS ONLY"); + assertEquals(2, ((LongValue) select.getOffset().getOffset()).getValue()); + assertEquals(7, ((LongValue) select.getFetch().getExpression()).getValue()); + } + + @ParameterizedTest + @ValueSource(strings = {"SELECT a INTO OUTFILE '/tmp/result.txt' FIELDS TERMINATED BY ',' " + + "OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY 'end' FROM t", + "SELECT a FROM t INTO OUTFILE '/tmp/result.txt' COLUMNS TERMINATED BY ',' " + + "OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY 'end'"}) + void boundedOutfileOptionsKeepTheirOrderingAndValues(String sql) throws Exception { + MySqlSelectIntoClause into = ((Select) complete(sql)).getPlainSelect() + .getMySqlSelectIntoClause(); + assertEquals(MySqlSelectIntoClause.Type.OUTFILE, into.getType()); + assertEquals("/tmp/result.txt", into.getFileName().getValue()); + assertEquals(",", into.getFieldsTerminatedBy().getValue()); + assertTrue(into.isFieldsOptionallyEnclosed()); + assertEquals("end", into.getLinesTerminatedBy().getValue()); + } + + @Test + void boundedParenthesesKeepExecutionArgumentsAndRuleActions() throws Exception { + Execute execute = (Execute) complete("CALL p (:a, :b)"); + assertEquals("p", execute.getName()); + assertEquals(2, execute.getExprList().size()); + assertEquals(":a", execute.getExprList().get(0).toString()); + assertEquals(":b", execute.getExprList().get(1).toString()); + CreateRule rule = (CreateRule) complete("CREATE RULE r AS ON INSERT TO t DO " + + "(INSERT INTO log VALUES (9); NOTIFY ch, 'changed')"); + assertEquals(2, rule.getActions().size()); + assertEquals("log", ((Insert) rule.getActions().get(0)).getTable().getName()); + assertEquals("changed", + ((NotifyStatement) rule.getActions().get(1)).getPayload().getValue()); + } + + @ParameterizedTest + @ValueSource(strings = {"LOCK TABLE a IN SHARE MODE WAIT 5", "LOCK a IN SHARE MODE WAIT 5"}) + void boundedTableKeywordKeepsLockTargetAndWait(String sql) throws Exception { + LockStatement lock = (LockStatement) complete(sql); + assertEquals(sql.startsWith("LOCK TABLE"), lock.isUseTableKeyword()); + assertEquals("a", lock.getTable().getName()); + assertEquals("SHARE", lock.getLockMode().getValue()); + assertEquals(5L, lock.getWaitSeconds()); + } + + @ParameterizedTest + @ValueSource(strings = {"SELECT id FROM t LIMIT", "SELECT id FROM t OFFSET", + "SELECT id FROM t FETCH NEXT", "CALL p (", "CALL p (:a,)", "LOCK TABLE", + "CREATE RULE r AS ON INSERT TO t DO (NOTIFY ch", "SELECT a FROM t INTO OUTFILE 'x' " + + "LINES TERMINATED BY 'end' FIELDS TERMINATED BY ','"}) + void boundedGuardsRejectIncompleteAndReorderedNeighbors(String sql) { + assertThrows(net.sf.jsqlparser.JSQLParserException.class, + () -> CCJSqlParserUtil.parse(sql)); + } + + private static Statement complete(String sql) throws Exception { + CCJSqlParser parser = CCJSqlParserUtil.newParser(sql); + Statement statement = parser.Statement(); + assertEquals(CCJSqlParserConstants.EOF, parser.getToken(1).kind); + assertEquals(statement.toString(), CCJSqlParserUtil.parse(statement.toString()).toString()); + return statement; + } + +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/GeneratedLookaheadParser.java b/src/test/java/net/sf/jsqlparser/parser/audit/GeneratedLookaheadParser.java new file mode 100644 index 000000000..2a3a69c0e --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/GeneratedLookaheadParser.java @@ -0,0 +1,100 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.io.File; +import java.io.StringReader; +import java.lang.reflect.InvocationTargetException; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.javacc.parser.Context; + +/** Exercises the actual generator and generated parser instead of a model of lookahead. */ +final class GeneratedLookaheadParser implements AutoCloseable { + private final URLClassLoader loader; + + GeneratedLookaheadParser(Path directory, String grammar) throws Exception { + Files.createDirectories(directory); + Path input = directory.resolve("Fixture.jj"); + Files.writeString(input, grammar); + run(directory, "UTF-8", "org.javacc.parser.Main", "-GRAMMAR_ENCODING=UTF-8", + "-OUTPUT_DIRECTORY=" + directory, input.toString()); + List arguments = + new ArrayList<>(List.of(tool("javac"), "--release", "11", "-encoding", "UTF-8", + "-d", directory.toString())); + try (java.util.stream.Stream files = Files.list(directory)) { + arguments.addAll(files.filter(p -> p.toString().endsWith(".java")).map(Path::toString) + .collect(Collectors.toList())); + } + execute(directory, "javac", arguments); + loader = new URLClassLoader(new java.net.URL[] {directory.toUri().toURL()}, + GeneratedLookaheadParser.class.getClassLoader()); + } + + boolean accepts(String input) throws Exception { + Class parser = loader.loadClass("Fixture"); + Object instance = parser.getConstructor(java.io.Reader.class) + .newInstance(new StringReader(input)); + try { + parser.getMethod("Entry").invoke(instance); + return true; + } catch (InvocationTargetException error) { + String type = error.getCause().getClass().getSimpleName(); + if (type.equals("ParseException") || type.equals("TokenMgrError")) { + return false; + } + throw error; + } + } + + static void run(Path directory, String encoding, String main, String... arguments) + throws Exception { + List command = + new ArrayList<>(List.of(tool("java"), "-Dfile.encoding=" + encoding, "-cp", + classpath(), main)); + command.addAll(List.of(arguments)); + execute(directory, main, command); + } + + private static String tool(String name) { + String executable = + System.getProperty("os.name").startsWith("Windows") ? name + ".exe" : name; + return Path.of(System.getProperty("java.home"), "bin", executable).toString(); + } + + private static void execute(Path directory, String name, List command) + throws Exception { + Path log = directory.resolve(name + ".log"); + Process process = new ProcessBuilder(command).redirectErrorStream(true) + .redirectOutput(log.toFile()).start(); + if (process.waitFor() != 0) { + throw new IllegalStateException("Generator failed: " + Files.readString(log)); + } + } + + private static String classpath() throws Exception { + List entries = new ArrayList<>(); + for (Class type : List.of(LookaheadAudit.class, Context.class, + Class.forName("org.javacc.java.JavaCodeGenerator"))) { + entries.add(Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()) + .toString()); + } + return String.join(File.pathSeparator, entries); + } + + @Override + public void close() throws Exception { + loader.close(); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAnalysis.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAnalysis.java new file mode 100644 index 000000000..9a706cbd1 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAnalysis.java @@ -0,0 +1,415 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.util.*; +import org.javacc.parser.*; + +/** Conservative lexical facts for review; never a parser equivalence proof. */ +final class LookaheadAnalysis { + final Context context; + final Map first = new LinkedHashMap<>(); + final Map> follows = new LinkedHashMap<>(); + final IdentityHashMap> localFollows = new IdentityHashMap<>(); + final IdentityHashMap parents = new IdentityHashMap<>(); + final List> rows = new ArrayList<>(); + private final int defaultLookahead; + + static final class First { + final Set tokens = new TreeSet<>(); + boolean nullable; + final Set caveats = new TreeSet<>(); + + void add(First other) { + tokens.addAll(other.tokens); + caveats.addAll(other.caveats); + } + + String signature() { + return tokens + ":" + nullable + ":" + caveats; + } + } + + LookaheadAnalysis(Context context) { + this.context = context; + defaultLookahead = Options.getLookahead(); + } + + static List children(Expansion e) { + if (e instanceof Sequence) { + return ((Sequence) e).units; + } + if (e instanceof Choice) { + return ((Choice) e).getChoices(); + } + if (e instanceof ZeroOrOne) { + return List.of(((ZeroOrOne) e).getExpansion()); + } + if (e instanceof ZeroOrMore) { + return List.of(((ZeroOrMore) e).getExpansion()); + } + if (e instanceof OneOrMore) { + return List.of(((OneOrMore) e).getExpansion()); + } + if (e instanceof TryBlock) { + return List.of(((TryBlock) e).exp); + } + return List.of(); // A NonTerminal is a call; do not traverse its target here. + } + + First inspect(Expansion e) { + First f = new First(); + if (e == null) { + f.caveats.add("opaque-production"); + return f; + } + if (e instanceof RegularExpression) { + return inspectToken((RegularExpression) e, f); + } + if (e instanceof Lookahead) { + return inspectLookahead((Lookahead) e, f); + } + if (e instanceof Action) { + f.nullable = true; + f.caveats.add("java-action"); + return f; + } + if (e instanceof NonTerminal) { + return inspectCall((NonTerminal) e, f); + } + if (e instanceof Sequence) { + return inspectSequence((Sequence) e, f); + } + if (e instanceof Choice) { + return inspectChoice((Choice) e, f); + } + return inspectGroup(e, f); + } + + private First inspectToken(RegularExpression token, First result) { + result.tokens.add(token.ordinal); + if (!token.lhsTokens.isEmpty() || token.rhsToken != null) { + result.caveats.add("java-token-assignment"); + } + return result; + } + + private First inspectLookahead(Lookahead lookahead, First result) { + result.nullable = true; + if (!lookahead.getActionTokens().isEmpty()) { + result.caveats.add("semantic-predicate"); + } + return result; + } + + private First inspectCall(NonTerminal call, First result) { + if (!call.getLhsTokens().isEmpty() || !call.getArgumentTokens().isEmpty() + || !call.getParametrizedTypeTokens().isEmpty()) { + result.caveats.add("java-call"); + } + First target = first.get(call.getName()); + if (target == null) { + result.caveats.add("unresolved-call"); + } else { + result.add(target); + result.nullable = target.nullable; + } + return result; + } + + private First inspectSequence(Sequence sequence, First result) { + result.nullable = true; + for (Expansion child : children(sequence)) { + First current = inspect(child); + result.caveats.addAll(current.caveats); + if (result.nullable) { + result.tokens.addAll(current.tokens); + result.nullable = current.nullable; + } + } + return result; + } + + private First inspectChoice(Choice choice, First result) { + for (Expansion child : children(choice)) { + First current = inspect(child); + result.add(current); + result.nullable |= current.nullable; + } + return result; + } + + private First inspectGroup(Expansion e, First f) { + List cs = children(e); + if (!cs.isEmpty()) { + First c = inspect(cs.get(0)); + f.add(c); + f.nullable = c.nullable; + if (e instanceof ZeroOrOne || e instanceof ZeroOrMore) { + f.nullable = true; + } + if (e instanceof TryBlock) { + f.caveats.add("exception-recovery"); + } + } else { + f.caveats.add("opaque-" + e.getClass().getSimpleName()); + } + return f; + } + + void recordParents(Expansion e, Expansion parent) { + if (e == null) { + return; + } + parents.put(e, parent); + for (Expansion child : children(e)) { + recordParents(child, e); + } + } + + boolean propagate(Expansion e, Set after) { + if (e == null) { + return false; + } + boolean changed = localFollows.computeIfAbsent(e, k -> new TreeSet<>()).addAll(after); + if (!(e instanceof NonTerminal)) { + if (e instanceof Sequence) { + Set next = new TreeSet<>(after); + List cs = children(e); + for (int i = cs.size() - 1; i >= 0; i--) { + Expansion c = cs.get(i); + changed |= propagate(c, next); + First cf = inspect(c); + Set n = new TreeSet<>(cf.tokens); + if (cf.nullable) { + n.addAll(next); + } + next = n; + } + } else { + Set next = new TreeSet<>(after); + if (e instanceof ZeroOrMore || e instanceof OneOrMore) { + next.addAll(inspect(e).tokens); + } + for (Expansion c : children(e)) { + changed |= propagate(c, next); + } + } + } + return changed; + } + + boolean oneTokenOnly(Expansion e) { + if (e instanceof RegularExpression) { + RegularExpression token = (RegularExpression) e; + return token.lhsTokens.isEmpty() && token.rhsToken == null; + } + if (e instanceof Sequence) { + List real = new ArrayList<>(); + for (Expansion c : children(e)) { + if (c instanceof Lookahead && !((Lookahead) c).isExplicit() + && ((Lookahead) c).getActionTokens().isEmpty()) { + continue; + } + real.add(c); + } + return real.size() == 1 && oneTokenOnly(real.get(0)); + } + if (e instanceof Choice) { + return !children(e).isEmpty() && children(e).stream().allMatch(this::oneTokenOnly); + } + return false; + } + + List tokenNames(Set tokens) { + List names = new ArrayList<>(); + for (int n : tokens) { + if (n == -1) { + names.add(""); + continue; + } + if (n == 0) { + names.add(""); + continue; + } + RegularExpression re = context.globals().rexps_of_tokens.get(n); + if (re != null && !re.label.isEmpty()) { + names.add("<" + re.label + ">"); + } else { + if (re instanceof RStringLiteral) { + names.add("\"" + ((RStringLiteral) re).image + "\""); + } else { + names.add(""); + } + } + } + return names; + } + + void visit(Expansion e, String production, Set seen) { + visit(e, production, seen, false); + } + + private void visit(Expansion e, String production, Set seen, boolean inProbe) { + if (e == null || !seen.add(e)) { + return; + } + if (e instanceof Lookahead && ((Lookahead) e).isExplicit()) { + recordLookahead((Lookahead) e, production, inProbe); + visit(((Lookahead) e).getLaExpansion(), production, seen, true); + } + for (Expansion child : children(e)) { + visit(child, production, seen, inProbe); + } + } + + private First competitors(Expansion decision, Expansion branch) { + First result = new First(); + if (decision instanceof Choice) { + Set selected = Collections.newSetFromMap(new IdentityHashMap<>()); + selected.add(branch); + for (Expansion other : children(decision)) { + if (!selected.contains(other)) { + addCompetitor(result, other, decision); + } + } + return result; + } + if (decision instanceof ZeroOrOne || decision instanceof ZeroOrMore + || decision instanceof OneOrMore) { + result.tokens.addAll(localFollows.getOrDefault(decision, Set.of(-1))); + } else { + result.tokens.add(-1); + } + return result; + } + + private void addCompetitor(First result, Expansion other, Expansion decision) { + First competing = inspect(other); + result.add(competing); + if (competing.nullable) { + result.tokens.addAll(localFollows.getOrDefault(decision, Set.of(-1))); + } + } + + private String triage(Lookahead la, Expansion decision, First branch, First competitors, + Set overlap, String semantic) { + if (!semantic.isEmpty()) { + return la.getAmount() == 0 ? "retain-state-or-policy" : "retain-combined-guard"; + } + if (la.getAmount() == 0) { + return "review-shared-prefix-or-unknown"; + } + if (oneTokenOnly(la.getLaExpansion())) { + return "review-exact-token-probe"; + } + return branchTriage(la, decision, branch, competitors, overlap); + } + + private String branchTriage(Lookahead la, Expansion decision, First branch, First competitors, + Set overlap) { + if (!scansActualBranch(la) || !branch.caveats.isEmpty() + || !competitors.caveats.isEmpty()) { + return "review-shared-prefix-or-unknown"; + } + if (la.getAmount() == defaultLookahead) { + return "review-default-depth"; + } + if (decision instanceof Choice && !branch.nullable && overlap.isEmpty() + && !competitors.tokens.contains(-1)) { + return "review-disjoint-FIRST1"; + } + return "review-shared-prefix-or-unknown"; + } + + private boolean scansActualBranch(Lookahead lookahead) { + Set branch = Collections.newSetFromMap(new IdentityHashMap<>()); + branch.add(parents.get(lookahead)); + return branch.contains(lookahead.getLaExpansion()); + } + + private void recordLookahead(Lookahead la, String production, boolean inProbe) { + Expansion sequence = parents.get(la); + Expansion decision = parents.get(sequence); + First branch = inspect(sequence); + First competing = competitors(decision, sequence); + Set overlap = new TreeSet<>(branch.tokens); + overlap.retainAll(competing.tokens); + String semantic = la.getActionTokens().stream().map(t -> t.image) + .collect(java.util.stream.Collectors.joining(" ")); + Map row = new TreeMap<>(); + row.put("production", production); + row.put("line", la.getLine()); + row.put("column", la.getColumn()); + row.put("amount", la.getAmount()); + row.put("semantic", semantic); + row.put("inProbe", inProbe); + row.put("tokenProbe", !inProbe && la.getAmount() > 0 && semantic.isEmpty() + && oneTokenOnly(la.getLaExpansion())); + row.put("triage", triage(la, decision, branch, competing, overlap, semantic)); + row.put("probeFirst", tokenNames(inspect(la.getLaExpansion()).tokens)); + row.put("decision", decision == null ? "unknown" : decision.getClass().getSimpleName()); + row.put("branchFirst", tokenNames(branch.tokens)); + row.put("nullable", branch.nullable); + row.put("caveats", branch.caveats); + row.put("competitorFirst", tokenNames(competing.tokens)); + row.put("competitorCaveats", competing.caveats); + row.put("overlap", tokenNames(overlap)); + rows.add(row); + } + + static String sha(String value) { + return sha(value.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + + static String sha(byte[] value) { + try { + byte[] bytes = java.security.MessageDigest.getInstance("SHA-256").digest(value); + StringBuilder result = new StringBuilder(); + for (byte b : bytes) { + result.append(String.format(Locale.ROOT, "%02x", b)); + } + return result.toString(); + } catch (java.security.NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + static String json(Object value) { + if (value == null || value instanceof Boolean || value instanceof Number) { + return String.valueOf(value); + } + if (value instanceof Map) { + List parts = new ArrayList<>(); + ((Map) value).forEach((k, v) -> parts.add(json(k.toString()) + ":" + json(v))); + return "{" + String.join(",", parts) + "}"; + } + if (value instanceof Collection) { + List parts = new ArrayList<>(); + for (Object item : (Collection) value) { + parts.add(json(item)); + } + return "[" + String.join(",", parts) + "]"; + } + StringBuilder result = new StringBuilder("\""); + for (char ch : value.toString().toCharArray()) { + if (ch == '"' || ch == '\\') { + result.append('\\').append(ch); + } else { + if (ch < 32) { + result.append(String.format(Locale.ROOT, "\\u%04x", (int) ch)); + } else { + result.append(ch); + } + } + } + return result.append('"').toString(); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAudit.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAudit.java new file mode 100644 index 000000000..3cdffd2f9 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAudit.java @@ -0,0 +1,334 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; +import java.util.stream.Collectors; + +/** Repository entry point: read git and files, calculate both snapshots, write a review report. */ +public final class LookaheadAudit { + private static final String GRAMMAR = + "src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt"; + private static final String TOOL = "src/test/java/net/sf/jsqlparser/parser/audit/"; + private static final Set BUILD_INPUTS = Set.of("build.gradle", "settings.gradle", + "build.gradle.kts", "settings.gradle.kts", "gradle.properties", "pom.xml", "gradlew", + "gradlew.bat", "mvnw", "mvnw.cmd", + "src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java"); + private static final List INPUT_DIRECTORIES = + List.of("src/main/java", "src/main/resources", ".mvn", "gradle"); + + private LookaheadAudit() {} + + public static void main(String[] args) throws Exception { + if (args.length < 4 || args.length > 6) { + throw new IllegalArgumentException( + "root base-ref report-directory evidence.tsv [fail-on-findings] [full-analysis]"); + } + Path root = Path.of(args[0]).toAbsolutePath(); + Path output = Path.of(args[2]).toAbsolutePath(); + + Files.createDirectories(output); + String base = baseCommit(root, args[1]); + String engine = engine(root); + Map beforeInputs = baseInputs(root, base); + Map afterInputs = workingInputs(root); + boolean full = args.length == 6 && Boolean.parseBoolean(args[5]); + Path cache = root.resolve("build/lookahead-cache"); + + System.out.println("Calculating base lookahead facts at " + base); + LookaheadSnapshot before = + load(git(root, "show", base + ":" + GRAMMAR), output.resolve("base"), + LookaheadAnalysis.sha(LookaheadAnalysis.json(beforeInputs)), engine, + cache, null, full); + System.out.println("Calculating working-tree lookahead facts"); + LookaheadSnapshot after = + load(Files.readString(root.resolve(GRAMMAR)), output.resolve("current"), + LookaheadAnalysis.sha(LookaheadAnalysis.json(afterInputs)), engine, + cache, before, full); + Evidence evidence = readEvidence(Path.of(args[3]), after.contexts()); + Map report = + LookaheadSnapshot.compare(before, after, evidence.bindings, evidence.scopes); + report.put("baseCommit", base); + report.put("watchedBefore", beforeInputs); + report.put("watchedCurrent", afterInputs); + report.put("analysisBefore", before.statistics); + report.put("analysisCurrent", after.statistics); + report.put("guarantee", + "Tracked-context change detection only; no automatic rewrite or semantic equivalence claim."); + Files.writeString(output.resolve("report.json"), LookaheadAnalysis.json(report) + "\n"); + writeInventory(output, after); + Files.write(output.resolve("binding-candidates.tsv"), evidence.proposed); + System.out.println("Lookahead report: " + output.resolve("report.json")); + if (args.length >= 5 && Boolean.parseBoolean(args[4]) + && Boolean.TRUE.equals(report.get("needsReview"))) { + throw new IllegalStateException("Lookahead contexts require review; see report.json"); + } + } + + private static void writeInventory(Path output, LookaheadSnapshot snapshot) throws IOException { + List lines = new ArrayList<>(); + lines.add("anchor\tproduction\tline\tcolumn\tamount\ttriage\ttokenProbe"); + for (Map site : snapshot.sites) { + lines.add(List.of("anchor", "production", "line", "column", "amount", "triage", + "tokenProbe").stream().map(key -> String.valueOf(site.get(key))) + .collect(Collectors.joining("\t"))); + } + Files.write(output.resolve("inventory.tsv"), lines); + Files.writeString(output.resolve("inventory.json"), + LookaheadAnalysis.json(snapshot.sites) + "\n"); + } + + private static String baseCommit(Path root, String argument) throws Exception { + if (!git(root, "rev-parse", "--show-object-format").trim().equals("sha1")) { + throw new IllegalArgumentException( + "This repository entry point requires Git SHA-1 objects"); + } + String ref = argument.matches("0+") ? "HEAD" : argument; + return git(root, "rev-parse", "--verify", "--end-of-options", ref + "^{commit}").trim(); + } + + private static Evidence readEvidence(Path path, + Map>> contexts) throws IOException { + Evidence evidence = new Evidence(); + for (String line : Files.readAllLines(path)) { + if (!line.isBlank() && !line.startsWith("#")) { + evidence.bind(line, contexts); + } + } + return evidence; + } + + private static final class Evidence { + private final Map bindings = new TreeMap<>(); + private final Map scopes = new TreeMap<>(); + private final List proposed = new ArrayList<>(); + + private void bind(String line, Map>> contexts) { + String[] fields = line.split("\t", 5); + if (fields.length < 4 || fields[2].isBlank() || fields[3].isBlank()) { + throw new IllegalArgumentException( + "Evidence requires key, fingerprint, reason and test reference"); + } + String scope = scope(fields); + List matching = contexts.keySet().stream() + .filter(k -> k.equals(fields[0]) || k.startsWith(fields[0] + ":")) + .collect(Collectors.toList()); + boolean unique = matching.size() == 1; + String key = unique ? matching.get(0) : "unbound:" + fields[0]; + String fingerprint = scope.equals("token-probe") ? "tokenFingerprint" : "fingerprint"; + validateTokenScope(scope, unique ? contexts.get(key) : List.of()); + bindings.put(key, fields[1]); + scopes.put(key, scope); + if (unique) { + propose(fields, contexts.get(key), fingerprint, scope); + } + } + + private static String scope(String[] fields) { + String scope = fields.length == 5 ? fields[4] : "context"; + if (!Set.of("context", "token-probe").contains(scope)) { + throw new IllegalArgumentException("Unknown evidence scope: " + scope); + } + return scope; + } + + private static void validateTokenScope(String scope, List> bindings) { + if (scope.equals("token-probe") && (bindings.size() != 1 + || !bindings.get(0).containsKey("tokenFingerprint"))) { + throw new IllegalArgumentException( + "token-probe scope requires one complete, positive, unnested token probe"); + } + } + + private void propose(String[] fields, List> bindings, + String fingerprint, String scope) { + if (bindings.size() == 1) { + proposed.add(fields[0] + "\t" + bindings.get(0).get(fingerprint) + "\t" + + fields[2] + "\t" + fields[3] + + (fields.length == 5 ? "\t" + scope : "")); + } + } + + } + + static boolean watched(String path) { + return BUILD_INPUTS.contains(path) || INPUT_DIRECTORIES.stream() + .anyMatch(directory -> path.equals(directory) || path.startsWith(directory + "/")); + } + + static LookaheadSnapshot generate(String source, Path directory, String external, String engine) + throws Exception { + return generate(source, directory, external, engine, null); + } + + static LookaheadSnapshot load(String source, Path directory, String external, String engine, + Path cache, LookaheadSnapshot previous, boolean full) throws Exception { + long started = System.nanoTime(); + String key = LookaheadCache.key(source, engine); + LookaheadSnapshot snapshot = full ? null : LookaheadCache.read(cache, key); + if (snapshot == null) { + snapshot = generate(source, directory, external, engine, full ? null : previous); + snapshot.statistics.put("frontEndCacheHit", false); + snapshot.statistics.put("jjtreeRuns", 1); + if (!full) { + LookaheadCache.write(cache, key, snapshot); + } + } else { + // Each read returns a separate value, including when base and current keys match. + snapshot.externalHash = external; + snapshot.statistics.put("mode", "content-hit"); + snapshot.statistics.put("frontEndCacheHit", true); + for (String name : List.of("jjtreeRuns", "javaCCParses", "parsedProductions", + "firstProductionsRecomputed", "firstComponentsRecomputed", + "ruleProductionsRecomputed", "followProductionsRecomputed", "sitesRecomputed", + "dependencyContextsRecomputed")) { + snapshot.statistics.put(name, 0); + } + snapshot.statistics.put("firstProductionsReused", snapshot.productions.size()); + snapshot.statistics.put("followProductionsReused", snapshot.productions.size()); + snapshot.statistics.put("sitesReused", snapshot.sites.size()); + snapshot.statistics.put("dependencyContextsReused", snapshot.dependencyCache.size()); + } + snapshot.statistics.put("elapsedMillis", (int) ((System.nanoTime() - started) / 1_000_000)); + return snapshot; + } + + private static LookaheadSnapshot generate(String source, Path directory, String external, + String engine, LookaheadSnapshot previous) throws Exception { + Files.createDirectories(directory); + Path input = directory.resolve("JSqlParserCC.jjt"); + Path generated = directory.resolve("generated"); + + Files.writeString(input, source); + Path java = Path.of(System.getProperty("java.home"), "bin", + System.getProperty("os.name").startsWith("Windows") ? "java.exe" : "java"); + Process process = new ProcessBuilder(java.toString(), "-Dfile.encoding=UTF-8", "-cp", + System.getProperty("java.class.path"), + "org.javacc.jjtree.Main", "-GRAMMAR_ENCODING=UTF-8", + "-OUTPUT_DIRECTORY=" + generated, input.toString()) + .redirectErrorStream(true).redirectOutput(directory.resolve("jjtree.log").toFile()) + .start(); + if (process.waitFor() != 0) { + throw new IllegalStateException( + "JJTree failed; see " + directory.resolve("jjtree.log")); + } + return LookaheadSnapshot.parse(Files.readString(generated.resolve("JSqlParserCC.jj")), + external, engine, previous); + } + + private static Map baseInputs(Path root, String base) throws Exception { + Map result = new TreeMap<>(); + for (String entry : git(root, "ls-tree", "-r", "-z", "--full-tree", base).split("\u0000")) { + if (entry.isEmpty()) { + continue; + } + String[] fields = entry.split("\t", 2); + String[] header = fields[0].split(" "); + + if (watched(fields[1])) { + if (!header[1].equals("blob") || !header[0].startsWith("100")) { + throw new IllegalStateException("Unsupported watched input " + fields[1]); + } + result.put(fields[1], header[2]); + } + } + return result; + } + + static Map workingInputs(Path root) throws Exception { + Map result = new TreeMap<>(); + for (String folder : INPUT_DIRECTORIES) { + Path start = root.resolve(folder); + if (!Files.exists(start, LinkOption.NOFOLLOW_LINKS)) { + continue; + } + try (java.util.stream.Stream paths = Files.walk(start)) { + for (Path path : paths.collect(Collectors.toList())) { + if (Files.isSymbolicLink(path)) { + throw new IllegalStateException("Symlink in watched inputs: " + path); + } + if (Files.isRegularFile(path)) { + addInput(root, path, result); + } + } + } + } + for (String file : BUILD_INPUTS) { + if (Files.exists(root.resolve(file), LinkOption.NOFOLLOW_LINKS)) { + addInput(root, root.resolve(file), result); + } + } + return result; + } + + private static void addInput(Path root, Path path, Map result) + throws Exception { + if (Files.isSymbolicLink(path)) { + throw new IllegalStateException("Symlink in watched inputs: " + path); + } + byte[] bytes = Files.readAllBytes(path); + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + digest.update(("blob " + bytes.length + "\0").getBytes(StandardCharsets.US_ASCII)); + StringBuilder hash = new StringBuilder(); + for (byte b : digest.digest(bytes)) { + hash.append(String.format(Locale.ROOT, "%02x", b)); + } + result.put(root.relativize(path).toString().replace('\\', '/'), hash.toString()); + } + + private static String engine(Path root) throws Exception { + Map inputs = new TreeMap<>(); + for (String property : List.of("java.runtime.version", "java.vendor", "os.name")) { + inputs.put(property, System.getProperty(property, "")); + } + inputs.put("jjtree-arguments", "file.encoding=UTF-8;GRAMMAR_ENCODING=UTF-8"); + try (java.util.stream.Stream files = Files.walk(root.resolve(TOOL))) { + for (Path path : files.collect(Collectors.toList())) { + if (Files.isSymbolicLink(path)) { + throw new IllegalStateException("Symlink in analyzer inputs: " + path); + } + if (Files.isRegularFile(path)) { + inputs.put(root.relativize(path).toString().replace('\\', '/'), + LookaheadAnalysis.sha(Files.readAllBytes(path))); + } + } + } + int generators = 0; + for (String entry : System.getProperty("java.class.path") + .split(java.util.regex.Pattern.quote(File.pathSeparator))) { + Path artifact = Path.of(entry); + String name = artifact.getFileName().toString(); + if (name.startsWith("javacc-") && name.endsWith(".jar")) { + inputs.put(name, LookaheadAnalysis.sha(Files.readAllBytes(artifact))); + generators++; + } + } + if (generators < 2) { + throw new IllegalStateException("Expected both JavaCC core and Java generator JARs"); + } + return LookaheadAnalysis.sha(LookaheadAnalysis.json(inputs)); + } + + private static String git(Path root, String... arguments) throws Exception { + List command = new ArrayList<>(List.of("git", "-C", root.toString())); + command.addAll(List.of(arguments)); + Process process = + new ProcessBuilder(command).redirectError(ProcessBuilder.Redirect.INHERIT).start(); + byte[] output = process.getInputStream().readAllBytes(); + if (process.waitFor() != 0) { + throw new IllegalStateException("Git read failed: " + command); + } + return new String(output, StandardCharsets.UTF_8); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAuditTest.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAuditTest.java new file mode 100644 index 000000000..f26f237f5 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadAuditTest.java @@ -0,0 +1,188 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import static org.junit.jupiter.api.Assertions.*; +import java.util.*; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class LookaheadAuditTest { + private static final String GRAMMAR = "options { STATIC=false; }\n" + + "PARSER_BEGIN(Fixture) public class Fixture { boolean enabled=true; } PARSER_END(Fixture)\n" + + "TOKEN: { | | }\n" + + "void Direct():{} { (LOOKAHEAD(2) Leaf() | ) }\n" + + "void Indirect():{} { (LOOKAHEAD(2) Bridge() | ) }\n" + + "void Bridge():{} { Leaf() }\nvoid Leaf():{} { }\n" + + "void Optional():{} { [LOOKAHEAD(2) ] }\n" + + "void Caller():{} { Optional() Tail() }\nvoid Tail():{} { [] }\n" + + "void Gate():{} { (LOOKAHEAD({enabled}) | ) }\n" + + "void Anchor():{} { [LOOKAHEAD() ] [LOOKAHEAD() ] }\n" + + "void Duplicate():{} { [LOOKAHEAD(2) ] [LOOKAHEAD(2) ] }\n" + + "void Recovery():{} { try { } catch (ParseException error) { enabled=true; } finally { enabled=true; } }\n" + + "void UnknownChoice():{} { (LOOKAHEAD(2) | { enabled=true; } ) }\n" + + "void NullableAlternative():{} { (LOOKAHEAD(2) | []) }\n"; + private static LookaheadSnapshot baseline; + private static Map evidence; + + @BeforeAll + static void baseline() throws Exception { + baseline = parse(GRAMMAR); + evidence = binding(baseline, "Direct"); + } + + private static LookaheadSnapshot parse(String grammar) throws Exception { + return LookaheadSnapshot.parse(grammar, "external", "engine"); + } + + private static Map binding(LookaheadSnapshot snapshot, String production) { + Map>> contexts = snapshot.contexts(); + String key = contexts.keySet().stream().filter(k -> k.startsWith(production + ":")) + .findFirst().orElseThrow(); + return Map.of(key, contexts.get(key).get(0).get("fingerprint").toString()); + } + + private static Map compare(String text) throws Exception { + return LookaheadSnapshot.compare(baseline, parse(text), evidence); + } + + @SuppressWarnings("unchecked") + private static List> findings(Map report) { + return (List>) report.get("changes"); + } + + @Test + void commentsAndLineMovesDoNotReopenEvidence() throws Exception { + assertEquals(false, compare("// shifted line\n\n" + GRAMMAR).get("needsReview")); + Map report = compare(GRAMMAR.replace("void Leaf()", "\n\nvoid Leaf()")); + assertEquals(false, report.get("needsReview")); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("reusableEvidence")); + } + + @Test + void insertedProbeDoesNotRenumberExistingAnchors() throws Exception { + LookaheadSnapshot changed = parse( + GRAMMAR.replace("void Anchor():{} {", "void Anchor():{} { [LOOKAHEAD() ]")); + assertTrue(changed.contexts().keySet().containsAll(baseline.contexts().keySet())); + assertEquals(1, findings(LookaheadSnapshot.compare(baseline, changed, evidence)).stream() + .filter(r -> r.get("kind").equals("added")).count()); + } + + @Test + void indirectCalleeChangesInvalidateEvidenceEvenWhenFirstIsUnchanged() throws Exception { + Map report = compare( + GRAMMAR.replace("void Leaf():{} { }", "void Leaf():{} { }")); + Map indirect = findings(report).stream() + .filter(r -> r.get("anchor").toString().startsWith("Indirect:")).findFirst() + .orElseThrow(); + assertEquals(false, indirect.get("localFactsChanged")); + assertEquals(List.of("Leaf"), indirect.get("dependencyChanges")); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("invalidatedEvidence")); + } + + @Test + void nullableAndFollowChangesReachAnUnchangedCallee() throws Exception { + for (String replacement : List.of("", "[]")) { + Map report = compare(GRAMMAR.replace("void Tail():{} { [] }", + "void Tail():{} { " + replacement + " }")); + Map finding = findings(report).stream() + .filter(r -> r.get("anchor").toString().startsWith("Optional:")).findFirst() + .orElseThrow(); + assertEquals(true, finding.get("localFactsChanged")); + assertEquals(List.of("Tail"), finding.get("dependencyChanges")); + } + } + + @Test + void changedDepthAndDeletionAreVisible() throws Exception { + assertTrue(findings(compare(GRAMMAR.replace("LOOKAHEAD(2) Leaf()", "LOOKAHEAD(3) Leaf()"))) + .stream().anyMatch(r -> r.get("anchor").toString().startsWith("Direct:"))); + assertEquals(1, findings(compare(GRAMMAR.replace("[LOOKAHEAD() ]", "[]"))).stream() + .filter(r -> r.get("kind").equals("removed")).count()); + } + + @Test + void parserPolicyLexerExternalAndEngineChangesCannotReuseEvidence() throws Exception { + for (String text : List.of( + GRAMMAR.replace("boolean enabled=true;", "boolean enabled=false;"), + GRAMMAR.replace("", ""))) { + Map report = compare(text); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("invalidatedEvidence")); + assertEquals(List.of(), report.get("changes")); + } + for (LookaheadSnapshot changed : List.of(LookaheadSnapshot.parse(GRAMMAR, "new", "engine"), + LookaheadSnapshot.parse(GRAMMAR, "external", "new"))) { + assertEquals(new ArrayList<>(evidence.keySet()), LookaheadSnapshot + .compare(baseline, changed, evidence).get("invalidatedEvidence")); + } + } + + @Test + void recoveryJavaChangesInvalidateUnrelatedEvidenceConservatively() throws Exception { + Map report = compare(GRAMMAR.replace( + "catch (ParseException error) { enabled=true; } finally { enabled=true; }", + "catch (ParseException error) { enabled=false; } finally { enabled=false; }")); + assertEquals(new ArrayList<>(evidence.keySet()), report.get("invalidatedEvidence")); + assertEquals(List.of("opaque-Java-or-predicate", "grammar-visible-to-opaque-Java"), + report.get("globalSignals")); + assertEquals(List.of(), report.get("changes")); + } + + @Test + void ambiguousAndMissingBindingsAreConservative() { + Map duplicate = binding(baseline, "Duplicate"); + assertEquals(new ArrayList<>(duplicate.keySet()), LookaheadSnapshot + .compare(baseline, baseline, duplicate).get("invalidatedEvidence")); + assertEquals(List.of("unbound:missing"), + LookaheadSnapshot.compare(baseline, baseline, Map.of("unbound:missing", "hash")) + .get("invalidatedEvidence")); + } + + @Test + void externalWatchScopeIncludesAllProductionModelsResourcesAndBuildInputs() { + assertTrue(LookaheadAudit.watched("src/main/java/example/AnUnrelatedModel.java")); + assertTrue(LookaheadAudit.watched("src/main/resources/dialect.properties")); + assertTrue(LookaheadAudit.watched("pom.xml")); + assertTrue(LookaheadAudit.watched("build.gradle")); + for (String path : List.of(".mvn/jvm.config", ".mvn/maven.config", ".mvn/extensions.xml", + "gradle/wrapper/gradle-wrapper.jar", "gradlew", "mvnw", ".mvn", "gradle")) { + assertTrue(LookaheadAudit.watched(path), path); + } + assertTrue(LookaheadAudit + .watched("src/test/java/net/sf/jsqlparser/parser/LookaheadCleanupTest.java")); + assertFalse(LookaheadAudit.watched("build/generated/Parser.java")); + } + + @Test + void predicatesAreRetainedAndNothingIsAutomaticallyRewritten() { + Map gate = baseline.sites.stream() + .filter(r -> r.get("production").equals("Gate")).findFirst().orElseThrow(); + assertEquals("retain-state-or-policy", gate.get("triage")); + assertEquals(0, LookaheadSnapshot.compare(baseline, baseline, evidence) + .get("automaticGrammarChanges")); + } + + @Test + void competingActionsPreventAClaimOfDisjointDispatch() { + Map decision = baseline.sites.stream() + .filter(r -> r.get("production").equals("UnknownChoice")).findFirst().orElseThrow(); + assertEquals("review-shared-prefix-or-unknown", decision.get("triage")); + assertEquals(Set.of("java-action"), decision.get("competitorCaveats")); + } + + @Test + void nullableCompetitorContributesFollowToTheDecision() { + Map decision = baseline.sites.stream() + .filter(r -> r.get("production").equals("NullableAlternative")).findFirst() + .orElseThrow(); + assertEquals("review-shared-prefix-or-unknown", decision.get("triage")); + assertEquals(List.of(""), decision.get("overlap")); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadCache.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadCache.java new file mode 100644 index 000000000..5f1ef1f8d --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadCache.java @@ -0,0 +1,186 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.io.*; +import java.nio.file.*; +import java.util.*; + +/** Local, content-addressed data cache. Never deserializes JavaCC objects or executable objects. */ +final class LookaheadCache { + private static final String FORMAT = "lookahead-facts-2"; + private static final int LIMIT = 64 * 1024 * 1024; + + private LookaheadCache() {} + + static String key(String source, String engine) { + return LookaheadAnalysis.sha(FORMAT + "\n" + engine + "\n" + source); + } + + static LookaheadSnapshot read(Path directory, String key) { + Path path = directory.resolve(key + ".facts"); + try { + if (!Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS) + || Files.size(path) > LIMIT) { + return null; + } + try (DataInputStream input = new DataInputStream(Files.newInputStream(path))) { + if (!FORMAT.equals(input.readUTF()) || !key.equals(input.readUTF())) { + return null; + } + String checksum = input.readUTF(); + byte[] bytes = readBytes(input); + if (!checksum.equals(LookaheadAnalysis.sha(bytes)) || input.read() != -1) { + return null; + } + try (DataInputStream data = new DataInputStream(new ByteArrayInputStream(bytes))) { + LookaheadSnapshot snapshot = LookaheadSnapshot.fromCache(readValue(data, 0)); + return data.read() == -1 ? snapshot : null; + } + } + } catch (IOException | RuntimeException error) { + // A truncated, incompatible or corrupted cache is a miss, never reusable evidence. + return null; + } + } + + static void write(Path directory, String key, LookaheadSnapshot snapshot) throws IOException { + Files.createDirectories(directory); + if (Files.isSymbolicLink(directory)) { + throw new IOException("Symlink cache directory: " + directory); + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream data = new DataOutputStream(bytes)) { + writeValue(data, snapshot.cacheValue()); + } + byte[] payload = bytes.toByteArray(); + if (payload.length > LIMIT) { + throw new IOException("Lookahead cache exceeds size limit"); + } + Path temporary = Files.createTempFile(directory, "facts-", ".tmp"); + try { + try (DataOutputStream data = + new DataOutputStream(Files.newOutputStream(temporary))) { + data.writeUTF(FORMAT); + data.writeUTF(key); + data.writeUTF(LookaheadAnalysis.sha(payload)); + data.writeInt(payload.length); + data.write(payload); + } + try { + Files.move(temporary, directory.resolve(key + ".facts"), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException unsupported) { + Files.move(temporary, directory.resolve(key + ".facts"), + StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporary); + } + } + + private static int size(DataInputStream input) throws IOException { + int size = input.readInt(); + if (size < 0 || size > LIMIT) { + throw new IOException("Invalid cache size"); + } + return size; + } + + private static byte[] readBytes(DataInputStream input) throws IOException { + int expected = size(input); + byte[] bytes = input.readNBytes(expected); + if (bytes.length != expected) { + throw new EOFException("Truncated cache value"); + } + return bytes; + } + + private static void writeValue(DataOutputStream output, Object value) throws IOException { + if (value == null) { + output.writeByte('n'); + } else if (value instanceof String) { + output.writeByte('s'); + byte[] bytes = value.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); + output.writeInt(bytes.length); + output.write(bytes); + } else if (value instanceof Boolean) { + output.writeByte('b'); + output.writeBoolean((Boolean) value); + } else if (value instanceof Integer) { + output.writeByte('i'); + output.writeInt((Integer) value); + } else if (value instanceof Map) { + output.writeByte('m'); + output.writeInt(((Map) value).size()); + for (Map.Entry entry : ((Map) value).entrySet()) { + writeValue(output, entry.getKey().toString()); + writeValue(output, entry.getValue()); + } + } else if (value instanceof Collection) { + output.writeByte(value instanceof Set ? 't' : 'l'); + output.writeInt(((Collection) value).size()); + for (Object item : (Collection) value) { + writeValue(output, item); + } + } else { + throw new IOException("Unsupported cache value: " + value.getClass()); + } + } + + private static Object readValue(DataInputStream input, int depth) throws IOException { + if (depth > 64) { + throw new IOException("Invalid cache nesting"); + } + int tag = input.readUnsignedByte(); + switch (tag) { + case 'n': + return null; + case 's': + return new String(readBytes(input), + java.nio.charset.StandardCharsets.UTF_8); + case 'b': + return input.readBoolean(); + case 'i': + return input.readInt(); + case 'm': + return readMap(input, depth); + case 't': + case 'l': + return readCollection(input, depth, tag == 't'); + default: + throw new IOException("Unknown cache value"); + } + } + + private static Map readMap(DataInputStream input, int depth) + throws IOException { + Map result = new TreeMap<>(); + int size = size(input); + for (int i = 0; i < size; i++) { + Object key = readValue(input, depth + 1); + if (!(key instanceof String) || result.containsKey(key)) { + throw new IOException("Invalid cache key"); + } + result.put((String) key, readValue(input, depth + 1)); + } + return result; + } + + private static Collection readCollection(DataInputStream input, int depth, boolean set) + throws IOException { + Collection result = set ? new LinkedHashSet<>() : new ArrayList<>(); + int size = size(input); + for (int i = 0; i < size; i++) { + result.add(readValue(input, depth + 1)); + } + return result; + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadCacheTest.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadCacheTest.java new file mode 100644 index 000000000..74bf02f30 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadCacheTest.java @@ -0,0 +1,137 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import static org.junit.jupiter.api.Assertions.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LookaheadCacheTest { + private static final String GRAMMAR = "options { STATIC=false; }\n" + + "PARSER_BEGIN(Fixture) public class Fixture {} PARSER_END(Fixture)\n" + + "TOKEN: { | }\n" + + "void Entry():{} { (LOOKAHEAD() | ) }"; + @TempDir + Path root; + + @Test + void independentJvmsReuseFactsAndKeepExternalBindingsSeparate() throws Exception { + Path grammar = root.resolve("src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt"); + Files.createDirectories(grammar.getParent()); + Files.writeString(grammar, GRAMMAR); + Path tool = root.resolve("src/test/java/net/sf/jsqlparser/parser/audit/Engine.java"); + Files.createDirectories(tool.getParent()); + Files.writeString(tool, "// fixed engine input\n"); + git("init", "-q"); + git("add", "."); + git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", + "baseline"); + Path evidence = root.resolve("evidence.tsv"); + Files.writeString(evidence, "Entry\tstale\treviewed context\tregression\n"); + run("cold", evidence, false); + Files.writeString(evidence, + Files.readString(root.resolve("cold/binding-candidates.tsv"))); + String warm = run("warm", evidence, false); + assertEquals(2, occurrences(warm, "\"frontEndCacheHit\":true")); + assertEquals(2, occurrences(warm, "\"javaCCParses\":0")); + assertEquals(2, occurrences(warm, "\"firstProductionsRecomputed\":0")); + assertTrue(warm.contains("\"needsReview\":false")); + Path model = root.resolve("src/main/java/Model.java"); + Files.createDirectories(model.getParent()); + Files.writeString(model, "class Model {}\n"); + String changed = run("external", evidence, false); + assertEquals(2, occurrences(changed, "\"javaCCParses\":0")); + assertTrue(changed.contains("\"invalidatedEvidence\":[\"Entry:")); + assertTrue(changed.contains("\"contextEvidenceReused\":[]")); + String full = run("full", evidence, true); + assertEquals(2, occurrences(full, "\"frontEndCacheHit\":false")); + assertEquals(2, occurrences(full, "\"javaCCParses\":1")); + assertTrue(full.contains("\"invalidatedEvidence\":[\"Entry:")); + assertEquals(Files.readString(root.resolve("warm/inventory.json")), + Files.readString(root.resolve("full/inventory.json"))); + assertEquals(2, Files.readAllLines(root.resolve("warm/inventory.tsv")).size()); + } + + @Test + void corruptedCacheIsRebuiltAndDoesNotSupplyFacts() throws Exception { + Path cache = root.resolve("cache"); + LookaheadSnapshot original = LookaheadAudit.load(GRAMMAR, root.resolve("first"), + "external", "engine", cache, null, false); + String key = LookaheadCache.key(GRAMMAR, "engine"); + Files.writeString(cache.resolve(key + ".facts"), "truncated cache"); + LookaheadSnapshot rebuilt = LookaheadAudit.load(GRAMMAR, root.resolve("rebuilt"), + "external", "engine", cache, null, false); + assertEquals(false, rebuilt.statistics.get("frontEndCacheHit")); + assertEquals(1, rebuilt.statistics.get("javaCCParses")); + assertEquals(original.firstFacts, rebuilt.firstFacts); + assertEquals(original.followFacts, rebuilt.followFacts); + assertEquals(original.contexts(), rebuilt.contexts()); + } + + @Test + void aValidChecksumDoesNotHideATruncatedDeclaredPayload() throws Exception { + Path cache = root.resolve("cache"); + LookaheadSnapshot original = LookaheadAudit.load(GRAMMAR, root.resolve("first"), + "external", "engine", cache, null, false); + Path path = cache.resolve(LookaheadCache.key(GRAMMAR, "engine") + ".facts"); + byte[] encoded = Files.readAllBytes(path); + java.io.ByteArrayInputStream bytes = new java.io.ByteArrayInputStream(encoded); + try (java.io.DataInputStream input = new java.io.DataInputStream(bytes)) { + input.readUTF(); + input.readUTF(); + input.readUTF(); + int position = encoded.length - bytes.available(); + int declared = input.readInt(); + java.nio.ByteBuffer.wrap(encoded, position, 4).putInt(declared + 1); + } + Files.write(path, encoded); + LookaheadSnapshot rebuilt = LookaheadAudit.load(GRAMMAR, root.resolve("rebuilt"), + "external", "engine", cache, null, false); + assertEquals(false, rebuilt.statistics.get("frontEndCacheHit")); + assertEquals(original.contexts(), rebuilt.contexts()); + } + + @Test + void changedEngineCannotReuseAPreviousPayload() throws Exception { + Path cache = root.resolve("cache"); + LookaheadSnapshot before = LookaheadAudit.load(GRAMMAR, root.resolve("before"), + "external", "engine", cache, null, false); + LookaheadSnapshot after = LookaheadAudit.load(GRAMMAR, root.resolve("after"), + "external", "new-engine", cache, before, false); + assertEquals(false, after.statistics.get("frontEndCacheHit")); + assertEquals("lexer-options-or-engine-changed", after.statistics.get("fallback")); + assertEquals(before.productions.size(), after.statistics.get("firstProductionsRecomputed")); + } + + private String run(String name, Path evidence, boolean full) throws Exception { + Path output = root.resolve(name); + GeneratedLookaheadParser.run(root, "UTF-8", LookaheadAudit.class.getName(), root.toString(), + "HEAD", output.toString(), evidence.toString(), "false", Boolean.toString(full)); + return Files.readString(output.resolve("report.json")); + } + + private static long occurrences(String text, String value) { + return java.util.regex.Pattern.compile(java.util.regex.Pattern.quote(value)).matcher(text) + .results().count(); + } + + private void git(String... arguments) throws Exception { + List command = new ArrayList<>(List.of("git", "-C", root.toString())); + command.addAll(List.of(arguments)); + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + assertEquals(0, process.waitFor(), output); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadConservatismTest.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadConservatismTest.java new file mode 100644 index 000000000..7bc1ceb7a --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadConservatismTest.java @@ -0,0 +1,366 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import static org.junit.jupiter.api.Assertions.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class LookaheadConservatismTest { + private static final String HEADER = "options { STATIC=false; }\n" + + "PARSER_BEGIN(Fixture) public class Fixture { Token state, other; } PARSER_END(Fixture)\n" + + "TOKEN: { | | }\n"; + @TempDir + Path directory; + + private static LookaheadSnapshot parse(String grammar) throws Exception { + return LookaheadSnapshot.parse(grammar, "external", "engine"); + } + + private static Map binding(LookaheadSnapshot snapshot, String production) { + return snapshot.contexts().entrySet().stream() + .filter(e -> e.getKey().startsWith(production + ":")) + .collect(java.util.stream.Collectors.toMap(Map.Entry::getKey, + e -> e.getValue().get(0).get("fingerprint").toString())); + } + + private static void invalidates(String before, String after, String production) + throws Exception { + LookaheadSnapshot baseline = parse(before); + Map evidence = binding(baseline, production); + assertFalse(evidence.isEmpty()); + Map report = LookaheadSnapshot.compare(baseline, parse(after), evidence); + assertEquals(true, report.get("needsReview")); + assertEquals(List.copyOf(evidence.keySet()), report.get("invalidatedEvidence")); + assertEquals(List.of(), report.get("reusableEvidence")); + assertEquals(false, LookaheadSnapshot.compare(baseline, parse(before), evidence) + .get("needsReview")); + } + + @Test + void callerContinuationBeyondFollowOneInvalidatesEvidence() throws Exception { + String grammar = HEADER + "void Opt():{} { [LOOKAHEAD(2) ] }\n" + + "void Entry():{} { Opt() }\n"; + String changed = grammar.replace("Opt() ", "Opt() "); + try (GeneratedLookaheadParser original = new GeneratedLookaheadParser( + directory.resolve("original"), grammar); + GeneratedLookaheadParser removed = new GeneratedLookaheadParser( + directory.resolve("removed"), grammar.replace("LOOKAHEAD(2)", "")); + GeneratedLookaheadParser current = new GeneratedLookaheadParser( + directory.resolve("current"), changed); + GeneratedLookaheadParser currentRemoved = new GeneratedLookaheadParser( + directory.resolve("current-removed"), + changed.replace("LOOKAHEAD(2)", ""))) { + for (String input : List.of("ab", "ac", "abab", "abac")) { + assertEquals(input.equals("abab"), original.accepts(input)); + assertEquals(original.accepts(input), removed.accepts(input)); + assertEquals(input.equals("ac") || input.equals("abac"), current.accepts(input)); + assertEquals(input.equals("abac"), currentRemoved.accepts(input)); + } + } + invalidates(grammar, changed, "Opt"); + invalidates(changed, grammar, "Opt"); + } + + @Test + void callerContinuationCalleesAndAncestorsAreDependencies() throws Exception { + String grammar = HEADER + "void Opt():{} { [LOOKAHEAD(2) ] }\n" + + "void Middle():{} { Opt() }\n" + + "void Entry():{} { Middle() Tail() }\n" + + "void Tail():{} { }\n"; + invalidates(grammar, grammar.replace("void Tail():{} { ", + "void Tail():{} { "), "Opt"); + invalidates(grammar, grammar.replace("Middle() Tail()", "Middle() "), "Opt"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void assignmentTargetsInvalidateSharedStatePredicates(boolean production) throws Exception { + String grammar = HEADER + "void Entry():{} { Setter() Gate() }\n" + + "void Setter():{} { state=" + (production ? "Value()" : "") + " }\n" + + "Token Value():{Token value;} { value= {return value;} }\n" + + "void Gate():{} { (LOOKAHEAD({state == null}) | ) }\n"; + String changed = grammar.replace("state=", "other="); + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), changed)) { + assertFalse(before.accepts("ab")); + assertTrue(after.accepts("ab")); + assertTrue(before.accepts("ac")); + assertFalse(after.accepts("ac")); + } + assertNotEquals(parse(grammar).opaqueHash, parse(changed).opaqueHash); + invalidates(grammar, changed, "Gate"); + } + + @Test + void productionSignaturesInvalidateJavaOverloadDecisions() throws Exception { + String grammar = HEADER.replace("Token state, other;", + "boolean flag(Number n) {return true;} boolean flag(Integer n) {return false;} " + + "boolean enabled() {try {return flag(Value());} " + + "catch(ParseException error) {return false;}}") + + "Number Value():{} { {return 1;} }\n" + + "void Entry():{} { (LOOKAHEAD({enabled()}) | ) }\n"; + String changed = grammar.replace("Number Value()", "Integer Value()"); + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), changed)) { + assertTrue(before.accepts("a")); + assertFalse(after.accepts("a")); + } + assertNotEquals(parse(grammar).opaqueHash, parse(changed).opaqueHash); + invalidates(grammar, changed, "Entry"); + } + + @Test + void actionsAfterTheFirstTokenPreventDisjointSuggestions() throws Exception { + String grammar = HEADER.replace("TOKEN: { | | }", + " TOKEN: { | | }\n" + + " TOKEN: { }") + + "void Entry():{} { (LOOKAHEAD(2) {token_source.SwitchTo(S);} | ) }"; + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), grammar.replace("LOOKAHEAD(2)", ""))) { + assertTrue(before.accepts("ab")); + assertFalse(after.accepts("ab")); + assertTrue(before.accepts("d")); + assertTrue(after.accepts("d")); + } + Map site = parse(grammar).sites.get(0); + assertEquals("review-shared-prefix-or-unknown", site.get("triage")); + assertTrue(site.get("caveats").toString().contains("java-action")); + } + + @Test + void independentExplicitProbesAreNotAssumedToMatchTheirBranch() throws Exception { + String grammar = HEADER + "void Entry():{} { " + + "(LOOKAHEAD() [] [] | ) }"; + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), grammar.replace("LOOKAHEAD()", ""))) { + assertFalse(before.accepts("ab")); + assertTrue(after.accepts("ab")); + assertTrue(before.accepts("ac")); + assertTrue(after.accepts("ac")); + } + assertEquals("review-shared-prefix-or-unknown", parse(grammar).sites.get(0).get("triage")); + } + + @Test + void nestedProbeDecisionsRemainVisibleWithoutInventingParents() throws Exception { + String grammar = HEADER + "void Entry():{} { " + + "(LOOKAHEAD(2, (LOOKAHEAD(2) | )) | ) }"; + LookaheadSnapshot snapshot = parse(grammar); + assertEquals(2, snapshot.sites.size()); + assertTrue(snapshot.sites.stream() + .allMatch(s -> s.get("triage").equals("review-shared-prefix-or-unknown"))); + } + + @Test + void mavenEncodingConfigurationIsARealEvidenceInput() throws Exception { + Path root = directory.resolve("repo"); + Files.createDirectories(root.resolve(".mvn")); + Path grammar = root.resolve("src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt"); + Files.createDirectories(grammar.getParent()); + Files.writeString(grammar, + HEADER + "void Entry():{} { (LOOKAHEAD(2) | ) }"); + Path tool = root.resolve("src/test/java/net/sf/jsqlparser/parser/audit/Engine.java"); + Files.createDirectories(tool.getParent()); + Files.writeString(tool, "// fixed test engine input\n"); + Path configuration = root.resolve(".mvn/jvm.config"); + Files.writeString(configuration, "-Dfile.encoding=UTF-8\n"); + git(root, "init"); + git(root, "add", "."); + git(root, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", + "-m", "baseline"); + Path evidence = root.resolve("evidence.tsv"); + Files.writeString(evidence, "Entry\tstale\treviewed\tregression\n"); + Path first = root.resolve("first-report"); + audit(root, first, evidence); + String candidate = Files.readString(first.resolve("binding-candidates.tsv")); + assertTrue(candidate.startsWith("Entry\t")); + Files.writeString(evidence, candidate); + Path stable = root.resolve("stable-report"); + audit(root, stable, evidence); + String unchanged = Files.readString(stable.resolve("report.json")); + assertTrue(unchanged.contains("\"needsReview\":false")); + assertTrue(unchanged.contains("\"reusableEvidence\":[\"Entry:")); + Files.writeString(configuration, "-Dfile.encoding=US-ASCII\n"); + Path second = root.resolve("second-report"); + audit(root, second, evidence); + String changed = Files.readString(second.resolve("report.json")); + assertTrue(changed.contains("\"needsReview\":true")); + assertTrue(changed.contains("production-Java-resources-or-build")); + assertTrue(changed.contains("\"reusableEvidence\":[]")); + assertTrue(changed.contains("\"invalidatedEvidence\":[\"Entry:")); + } + + @Test + void tokenFieldAccessIsPartOfOpaqueJava() throws Exception { + String grammar = HEADER.replace("Token state, other;", "Object state;") + + "void Entry():{} { Setter() Gate() }\n" + + "void Setter():{} { state=.image }\n" + + "void Gate():{} { (LOOKAHEAD({state instanceof String}) | ) }"; + String changed = grammar.replace(".image", ".kind"); + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), changed)) { + assertTrue(before.accepts("ab")); + assertFalse(after.accepts("ab")); + } + assertNotEquals(parse(grammar).opaqueHash, parse(changed).opaqueHash); + invalidates(grammar, changed, "Gate"); + } + + @ParameterizedTest + @ValueSource(strings = {"LOOKAHEAD(0)", "LOOKAHEAD(0,)"}) + void zeroDepthNeverSuggestsAnEquivalentTokenGuard(String probe) throws Exception { + String grammar = HEADER + "void Entry():{} { (" + probe + " | ) }"; + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), grammar.replace(probe, ""))) { + assertFalse(before.accepts("b")); + assertTrue(after.accepts("b")); + } + assertEquals("review-shared-prefix-or-unknown", parse(grammar).sites.get(0).get("triage")); + } + + @Test + void defaultDepthMeansTheConfiguredDefault() throws Exception { + String grammar = HEADER.replace("STATIC=false;", "STATIC=false; LOOKAHEAD=2;") + + "void Entry():{} { [LOOKAHEAD(1) ] }"; + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), grammar.replace("LOOKAHEAD(1)", ""))) { + assertFalse(before.accepts("ac")); + assertTrue(after.accepts("ac")); + } + assertEquals("review-shared-prefix-or-unknown", parse(grammar).sites.get(0).get("triage")); + } + + @Test + void mavenJvmEncodingChangesActualGeneratedTokens() throws Exception { + for (String encoding : List.of("UTF-8", "US-ASCII")) { + Path output = directory.resolve(encoding); + Files.createDirectories(output); + Path input = output.resolve("Fixture.jjt"); + Files.writeString(input, HEADER.replace("", "") + + "void Entry():{} { }"); + GeneratedLookaheadParser.run(output, encoding, "org.javacc.jjtree.Main", + "-GRAMMAR_ENCODING=UTF-8", "-OUTPUT_DIRECTORY=" + output, input.toString()); + String generated = Files.readString(output.resolve("Fixture.jj")); + try (GeneratedLookaheadParser parser = + new GeneratedLookaheadParser(output, generated)) { + assertEquals(encoding.equals("UTF-8"), parser.accepts("Ê°")); + assertEquals(encoding.equals("US-ASCII"), parser.accepts("?")); + } + } + } + + @Test + void effectsAfterTheFirstTokenPropagateThroughCallsAndCompetingBranches() throws Exception { + String grammar = HEADER + + "void Entry():{} { (LOOKAHEAD(2) Effect() | ) }\n"; + for (String effect : List.of("void Effect():{} { {state=null;} }", + "void Effect():{Token token;} { token= }")) { + LookaheadSnapshot snapshot = parse(grammar + effect); + Map site = snapshot.sites.get(0); + assertEquals(List.of(""), site.get("branchFirst")); + assertEquals(false, site.get("nullable")); + assertEquals("review-shared-prefix-or-unknown", site.get("triage")); + assertFalse(site.get("caveats").toString().equals("[]")); + String competing = grammar.replace(" Effect() | ", " | Effect()"); + assertEquals("review-shared-prefix-or-unknown", + parse(competing + effect).sites.get(0).get("triage")); + } + } + + @Test + void javaHelpersCanCallProductionsOutsideTheGrammarCallGraph() throws Exception { + String grammar = HEADER.replace("Token state, other;", + "boolean enabled() {try {Leaf(); return true;} " + + "catch(ParseException error) {return false;}}") + + "void Entry():{} { (LOOKAHEAD({enabled()}) | ) }\n" + + "void Leaf():{} { }"; + String changed = grammar.replace("void Leaf():{} { }", "void Leaf():{} { }"); + try (GeneratedLookaheadParser before = new GeneratedLookaheadParser( + directory.resolve("before"), grammar); + GeneratedLookaheadParser after = new GeneratedLookaheadParser( + directory.resolve("after"), changed)) { + assertTrue(before.accepts("ab")); + assertFalse(after.accepts("ab")); + assertFalse(before.accepts("abb")); + assertTrue(after.accepts("abb")); + assertTrue(before.accepts("c")); + assertTrue(after.accepts("c")); + } + invalidates(grammar, changed, "Entry"); + } + + @Test + void signatureAdditionsCannotBeAssumedUnrelatedToOpaqueJava() throws Exception { + String grammar = HEADER + "void Entry():{} { (LOOKAHEAD(2) | ) }"; + String disconnected = grammar + "\nvoid Unrelated():{} { }"; + invalidates(grammar, disconnected, "Entry"); + String changed = disconnected.replace("void Unrelated():{} { }", + "void Unrelated():{} { }"); + invalidates(disconnected, changed, "Entry"); + LookaheadSnapshot original = parse(disconnected); + Map report = LookaheadSnapshot.compare(original, parse(changed), + binding(original, "Entry")); + assertEquals(List.of("grammar-visible-to-opaque-Java"), report.get("globalSignals")); + assertEquals(List.of(), report.get("changes")); + } + + @Test + void brokenWatchedRootAndNestedSymlinksAreRejected() throws Exception { + org.junit.jupiter.api.Assumptions.assumeFalse( + System.getProperty("os.name").startsWith("Windows")); + int index = 0; + for (String input : List.of(".mvn", ".mvn/jvm.config", "gradle", "gradle/wrapper/jar", + "src/main/java", "src/main/resources", "gradlew")) { + Path root = directory.resolve("symlink-" + index++); + Path link = root.resolve(input); + Files.createDirectories(link.getParent()); + Files.createSymbolicLink(link, root.resolve("missing-target").toAbsolutePath()); + IllegalStateException failure = assertThrows(IllegalStateException.class, + () -> LookaheadAudit.workingInputs(root)); + assertTrue(failure.getMessage().contains("Symlink")); + } + } + + private static void audit(Path root, Path output, Path evidence) throws Exception { + GeneratedLookaheadParser.run(root, "UTF-8", LookaheadAudit.class.getName(), root.toString(), + "HEAD", output.toString(), evidence.toString()); + } + + private static void git(Path root, String... arguments) throws Exception { + java.util.ArrayList command = new java.util.ArrayList<>( + List.of("git", "-C", root.toString())); + command.addAll(List.of(arguments)); + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + assertEquals(0, process.waitFor(), output); + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadGraph.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadGraph.java new file mode 100644 index 000000000..fa61c6ba7 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadGraph.java @@ -0,0 +1,108 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.util.*; +import org.javacc.parser.*; + +/** Dependency components, in callee-before-caller order. */ +final class LookaheadGraph { + final Map> calls = new TreeMap<>(); + final Map> callers = new TreeMap<>(); + final List> components = new ArrayList<>(); + private final Map indexes = new HashMap<>(); + private final Map low = new HashMap<>(); + private final Deque stack = new ArrayDeque<>(); + private final Set active = new HashSet<>(); + + LookaheadGraph(Map> productions) { + this(productions, "calls"); + } + + @SuppressWarnings("unchecked") + LookaheadGraph(Map> productions, String edgeKind) { + productions.forEach((name, value) -> { + calls.put(name, new TreeSet<>((Set) value.get(edgeKind))); + callers.put(name, new TreeSet<>()); + }); + calls.forEach((source, targets) -> targets.forEach(target -> { + if (!callers.containsKey(target)) { + throw new IllegalArgumentException("Unresolved production " + target); + } + callers.get(target).add(source); + })); + for (String name : calls.keySet()) { + if (!indexes.containsKey(name)) { + connect(name); + } + } + } + + static Set calls(Expansion expansion, boolean probes) { + Set result = new TreeSet<>(); + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + Deque pending = new ArrayDeque<>(); + if (expansion != null) { + pending.add(expansion); + } + while (!pending.isEmpty()) { + Expansion current = pending.removeFirst(); + if (seen.add(current)) { + if (current instanceof NonTerminal) { + result.add(((NonTerminal) current).getName()); + } + if (probes && current instanceof Lookahead + && ((Lookahead) current).getLaExpansion() != null) { + pending.add(((Lookahead) current).getLaExpansion()); + } + pending.addAll(LookaheadAnalysis.children(current)); + } + } + return result; + } + + private void connect(String name) { + int index = indexes.size(); + indexes.put(name, index); + low.put(name, index); + stack.push(name); + active.add(name); + for (String target : calls.get(name)) { + if (!indexes.containsKey(target)) { + connect(target); + low.put(name, Math.min(low.get(name), low.get(target))); + } else if (active.contains(target)) { + low.put(name, Math.min(low.get(name), indexes.get(target))); + } + } + if (low.get(name).equals(indexes.get(name))) { + Set component = new TreeSet<>(); + String member; + do { + member = stack.pop(); + active.remove(member); + component.add(member); + } while (!member.equals(name)); + components.add(component); + } + } + + static Set closure(Collection roots, Map> edges) { + Set result = new TreeSet<>(); + Deque pending = new ArrayDeque<>(roots); + while (!pending.isEmpty()) { + String name = pending.removeFirst(); + if (result.add(name)) { + pending.addAll(edges.getOrDefault(name, Set.of())); + } + } + return result; + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadIncremental.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadIncremental.java new file mode 100644 index 000000000..3235bc124 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadIncremental.java @@ -0,0 +1,281 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.util.*; +import org.javacc.parser.*; + +/** Recomputes changed data-flow components from bottom; unchanged facts are immutable inputs. */ +final class LookaheadIncremental { + private final LookaheadSnapshot current; + private final LookaheadSnapshot previous; + private final LookaheadAnalysis facts; + private final LookaheadGraph graph; + private final Map definitions = new TreeMap<>(); + + LookaheadIncremental(LookaheadSnapshot current, LookaheadSnapshot previous, + LookaheadAnalysis facts) { + this.current = current; + this.previous = previous; + this.facts = facts; + graph = new LookaheadGraph(current.productions, "actualCalls"); + for (NormalProduction production : facts.context.globals().bnfproductions) { + definitions.put(production.getLhs(), production); + } + } + + void compute() { + current.statistics.put("mode", previous == null ? "cold" : "incremental"); + computeFirst(); + computeRules(); + computeFollows(); + computeRows(); + } + + private static String componentKey(Set component) { + return LookaheadAnalysis.json(component); + } + + private Map firstDependencies(String production, Set excluded) { + Map result = new TreeMap<>(); + for (String name : graph.calls.get(production)) { + if (!excluded.contains(name)) { + result.put(name, facts.first.get(name).signature()); + } + } + return result; + } + + private void computeFirst() { + definitions.keySet().forEach(name -> facts.first.put(name, + previous == null ? new LookaheadAnalysis.First() + : first(previous.firstFacts.get(name)))); + int recomputed = 0; + int components = 0; + for (Set component : graph.components) { + Map input = new TreeMap<>(); + for (String name : component) { + input.put(name, List.of(current.productions.get(name).get("hash"), + firstDependencies(name, component))); + } + String key = componentKey(component); + String signature = LookaheadAnalysis.sha(LookaheadAnalysis.json(input)); + current.firstInputs.put(key, signature); + if (previous == null || !signature.equals(previous.firstInputs.get(key))) { + component.forEach(name -> facts.first.put(name, new LookaheadAnalysis.First())); + solveFirst(component); + recomputed += component.size(); + components++; + } + } + facts.first.forEach((name, value) -> current.firstFacts.put(name, firstValue(value))); + current.statistics.put("firstProductionsRecomputed", recomputed); + current.statistics.put("firstComponentsRecomputed", components); + current.statistics.put("firstProductionsReused", definitions.size() - recomputed); + } + + private void solveFirst(Set component) { + boolean changed; + do { + changed = false; + for (String name : component) { + NormalProduction production = definitions.get(name); + LookaheadAnalysis.First value = facts.inspect(production.getExpansion()); + if (production instanceof BNFProduction + && !((BNFProduction) production).getDeclarationTokens().isEmpty()) { + value.caveats.add("java-declarations"); + } + if (!value.signature().equals(facts.first.get(name).signature())) { + facts.first.put(name, value); + changed = true; + } + } + } while (changed); + } + + private static Map firstValue(LookaheadAnalysis.First first) { + Map result = new TreeMap<>(); + result.put("tokens", first.tokens); + result.put("nullable", first.nullable); + result.put("caveats", first.caveats); + return result; + } + + @SuppressWarnings("unchecked") + private static LookaheadAnalysis.First first(Map value) { + LookaheadAnalysis.First result = new LookaheadAnalysis.First(); + if (value != null) { + result.tokens.addAll((Set) value.get("tokens")); + result.nullable = (Boolean) value.get("nullable"); + result.caveats.addAll((Set) value.get("caveats")); + } + return result; + } + + private void computeRules() { + int recomputed = 0; + for (String name : definitions.keySet()) { + String signature = LookaheadAnalysis.sha(LookaheadAnalysis.json(List.of( + current.productions.get(name).get("hash"), firstDependencies(name, Set.of())))); + current.ruleInputs.put(name, signature); + if (previous != null && signature.equals(previous.ruleInputs.get(name))) { + current.followRules.put(name, previous.followRules.get(name)); + } else { + List> rules = new ArrayList<>(); + collectRules(definitions.get(name).getExpansion(), new Suffix(), name, rules); + current.followRules.put(name, rules); + recomputed++; + } + } + current.statistics.put("ruleProductionsRecomputed", recomputed); + } + + private void collectRules(Expansion expansion, Suffix after, String caller, + List> rules) { + if (expansion == null) { + return; + } + if (expansion instanceof NonTerminal) { + Map rule = new TreeMap<>(); + rule.put("caller", caller); + rule.put("target", ((NonTerminal) expansion).getName()); + rule.put("tokens", after.tokens); + rule.put("inherits", after.inherits); + rule.put("suffix", after.shape); + rule.put("calls", after.calls); + rules.add(rule); + } else if (expansion instanceof Sequence) { + Suffix next = after; + List children = LookaheadAnalysis.children(expansion); + for (int i = children.size() - 1; i >= 0; i--) { + Expansion child = children.get(i); + collectRules(child, next, caller, rules); + next = after(child, next, false); + } + } else { + Suffix next = expansion instanceof OneOrMore || expansion instanceof ZeroOrMore + ? after(expansion, after, true) + : after; + for (Expansion child : LookaheadAnalysis.children(expansion)) { + collectRules(child, next, caller, rules); + } + } + } + + private Suffix after(Expansion expansion, Suffix following, boolean loop) { + LookaheadAnalysis.First first = facts.inspect(expansion); + Suffix result = new Suffix(); + result.tokens.addAll(first.tokens); + if (loop || first.nullable) { + result.tokens.addAll(following.tokens); + } + result.inherits = (loop || first.nullable) && following.inherits; + result.shape.add((loop ? "repeat:" : "") + + LookaheadAnalysis.sha(LookaheadSnapshot.continuationShape(expansion))); + result.shape.addAll(following.shape); + result.calls.addAll(following.calls); + result.calls.addAll(LookaheadGraph.calls(expansion, true)); + return result; + } + + private static final class Suffix { + private final Set tokens = new TreeSet<>(); + private boolean inherits = true; + private final List shape = new ArrayList<>(); + private final Set calls = new TreeSet<>(); + } + + private void computeFollows() { + definitions.keySet().forEach(name -> facts.follows.put(name, + previous == null ? new TreeSet<>(Set.of(-1)) + : new TreeSet<>(previous.followFacts.getOrDefault(name, Set.of(-1))))); + List> order = new ArrayList<>(graph.components); + Collections.reverse(order); + List> all = new ArrayList<>(); + current.followRules.values().forEach(all::addAll); + int recomputed = 0; + for (Set component : order) { + List> incoming = new ArrayList<>(); + List inputs = new ArrayList<>(); + for (Map rule : all) { + if (component.contains(rule.get("target"))) { + incoming.add(rule); + inputs.add(followInput(rule, component)); + } + } + Collections.sort(inputs); + String key = componentKey(component); + String signature = LookaheadAnalysis.sha(LookaheadAnalysis.json(inputs)); + current.followInputs.put(key, signature); + if (previous == null || !signature.equals(previous.followInputs.get(key))) { + component.forEach(name -> facts.follows.put(name, new TreeSet<>(Set.of(-1)))); + solveFollows(incoming); + recomputed += component.size(); + } + } + facts.follows.forEach((name, value) -> current.followFacts.put(name, new TreeSet<>(value))); + current.statistics.put("followProductionsRecomputed", recomputed); + current.statistics.put("followProductionsReused", definitions.size() - recomputed); + } + + private String followInput(Map rule, Set component) { + String caller = rule.get("caller").toString(); + return LookaheadAnalysis.json(List.of(caller, rule.get("target"), rule.get("tokens"), + rule.get("inherits"), component.contains(caller) ? "internal" + : facts.follows.get(caller))); + } + + @SuppressWarnings("unchecked") + private void solveFollows(List> incoming) { + boolean changed; + do { + changed = false; + for (Map rule : incoming) { + Set value = facts.follows.get(rule.get("target")); + changed |= value.addAll((Set) rule.get("tokens")); + if (Boolean.TRUE.equals(rule.get("inherits"))) { + changed |= value.addAll(facts.follows.get(rule.get("caller"))); + } + } + } while (changed); + } + + private void computeRows() { + int recomputed = 0; + int reused = 0; + for (String name : definitions.keySet()) { + String signature = LookaheadAnalysis.sha(LookaheadAnalysis.json(List.of( + current.productions.get(name).get("hash"), rowFirstDependencies(name), + facts.follows.get(name)))); + current.rowInputs.put(name, signature); + if (previous != null && signature.equals(previous.rowInputs.get(name))) { + reused += current.reuseRows(name, previous, facts.rows); + } else { + NormalProduction production = definitions.get(name); + facts.propagate(production.getExpansion(), facts.follows.get(name)); + int before = facts.rows.size(); + facts.visit(production.getExpansion(), name, + Collections.newSetFromMap(new IdentityHashMap<>())); + recomputed += facts.rows.size() - before; + } + } + current.statistics.put("sitesRecomputed", recomputed); + current.statistics.put("sitesReused", reused); + } + + @SuppressWarnings("unchecked") + private Map rowFirstDependencies(String name) { + Map result = new TreeMap<>(); + for (String target : (Set) current.productions.get(name).get("calls")) { + result.put(target, facts.first.get(target).signature()); + } + return result; + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadIncrementalTest.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadIncrementalTest.java new file mode 100644 index 000000000..1cc95e991 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadIncrementalTest.java @@ -0,0 +1,396 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import static org.junit.jupiter.api.Assertions.*; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.TreeSet; +import java.util.stream.Stream; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.io.TempDir; + +class LookaheadIncrementalTest { + private static final String HEADER = "options { STATIC=false; }\n" + + "PARSER_BEGIN(Fixture) public class Fixture {} PARSER_END(Fixture)\n" + + "TOKEN: { | | | }\n"; + @TempDir + Path directory; + + private static LookaheadSnapshot parse(String grammar) throws Exception { + return LookaheadSnapshot.parse(grammar, "external", "engine"); + } + + private static Map binding(LookaheadSnapshot snapshot, String production, + String fingerprint) { + return snapshot.contexts().entrySet().stream() + .filter(entry -> entry.getKey().startsWith(production + ":")) + .collect(java.util.stream.Collectors.toMap(Map.Entry::getKey, + entry -> entry.getValue().get(0).get(fingerprint).toString())); + } + + private static LookaheadSnapshot update(LookaheadSnapshot before, String grammar) + throws Exception { + LookaheadSnapshot incremental = + LookaheadSnapshot.parse(grammar, "external", "engine", before); + LookaheadSnapshot full = parse(grammar); + assertEquals(full.firstFacts, incremental.firstFacts, "FIRST must equal a cold solve"); + assertEquals(full.followFacts, incremental.followFacts, "FOLLOW must equal a cold solve"); + assertEquals(full.sites, incremental.sites, "site facts and locations must agree"); + assertEquals(full.contexts(), incremental.contexts(), "bindings must agree"); + return incremental; + } + + @Test + void unrelatedPureProductionPreservesFullContextEvidence() throws Exception { + String grammar = HEADER + "void Entry():{} { (LOOKAHEAD(2) | ) }\n" + + "void Other():{} { }"; + LookaheadSnapshot before = parse(grammar); + assertTrue(before.pureGrammar); + LookaheadSnapshot after = update(before, + grammar.replace("void Other():{} { ", "void Other():{} { ")); + assertEquals(1, after.statistics.get("firstProductionsRecomputed")); + assertEquals(0, after.statistics.get("followProductionsRecomputed")); + assertEquals(0, after.statistics.get("sitesRecomputed")); + Map evidence = binding(before, "Entry", "fingerprint"); + Map report = LookaheadSnapshot.compare(before, after, evidence); + assertEquals(false, report.get("needsReview")); + assertEquals(List.copyOf(evidence.keySet()), report.get("contextEvidenceReused")); + } + + @Test + void unchangedFirstDoesNotHideAChangedCalleeBody() throws Exception { + String grammar = HEADER + "void Entry():{} { (LOOKAHEAD(2) Leaf() | ) }\n" + + "void Leaf():{} { }"; + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, grammar.replace(" }", " }")); + assertEquals(1, after.statistics.get("firstProductionsRecomputed")); + assertEquals(1, after.statistics.get("sitesReused")); + Map evidence = binding(before, "Entry", "fingerprint"); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, after, evidence).get("invalidatedEvidence")); + } + + @Test + void continuationBeyondFollowOneInvalidatesEvenWhenRowsAreReused() throws Exception { + String grammar = HEADER + "void Entry():{} { Opt() }\n" + + "void Opt():{} { [LOOKAHEAD(2) ] }"; + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, + grammar.replace("Opt() ", "Opt() ")); + assertEquals(before.followFacts, after.followFacts); + assertEquals(1, after.statistics.get("sitesReused")); + Map evidence = binding(before, "Opt", "fingerprint"); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, after, evidence).get("invalidatedEvidence")); + } + + @Test + void shortNonNullableSuffixStillDependsOnHigherCallerTokens() throws Exception { + String grammar = HEADER + "void Entry():{} { Middle() }\n" + + "void Middle():{} { Opt() }\n" + + "void Opt():{} { [LOOKAHEAD(3) ] }"; + String changed = grammar.replace("Middle() ", "Middle() "); + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, changed); + Map evidence = binding(before, "Opt", "fingerprint"); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, after, evidence).get("invalidatedEvidence")); + try (GeneratedLookaheadParser original = + new GeneratedLookaheadParser(directory.resolve("original"), grammar); + GeneratedLookaheadParser removed = new GeneratedLookaheadParser( + directory.resolve("removed"), grammar.replace("LOOKAHEAD(3)", "")); + GeneratedLookaheadParser current = + new GeneratedLookaheadParser(directory.resolve("current"), changed); + GeneratedLookaheadParser currentRemoved = new GeneratedLookaheadParser( + directory.resolve("current-removed"), + changed.replace("LOOKAHEAD(3)", ""))) { + assertFalse(original.accepts("abc")); + assertFalse(removed.accepts("abc")); + assertTrue(original.accepts("abcabc")); + assertTrue(removed.accepts("abcabc")); + assertTrue(current.accepts("abd")); + assertFalse(currentRemoved.accepts("abd")); + assertTrue(current.accepts("abcabd")); + assertTrue(currentRemoved.accepts("abcabd")); + } + } + + @Test + void recursiveFirstAndFollowFactsCanShrink() throws Exception { + String grammar = HEADER + "void Entry():{} { P() }\n" + + "void P():{} { ( [P()] | ) }"; + LookaheadSnapshot after = update(parse(grammar), + grammar.replace("P() ", "P() ") + .replace("| )", "| )")); + assertEquals(Set.of(1, 3), after.firstFacts.get("P").get("tokens")); + assertEquals(Set.of(-1, 3), after.followFacts.get("P")); + } + + @Test + void deletingAnActionRemovesRecursiveCaveats() throws Exception { + String grammar = HEADER + "void Entry():{} { P() }\n" + + "void P():{} { [Q()] }\nvoid Q():{} { [P()] {int x=1;} }"; + LookaheadSnapshot after = update(parse(grammar), grammar.replace("{int x=1;}", "")); + assertEquals(Set.of(), after.firstFacts.get("P").get("caveats")); + assertEquals(Set.of(), after.firstFacts.get("Q").get("caveats")); + } + + @Test + void deletedAndAddedCallSitesDoNotLeaveOldFollowContributions() throws Exception { + String grammar = HEADER + "void Entry():{} { P() Q() }\n" + + "void P():{} { }\nvoid Q():{} { }"; + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot removed = update(before, grammar.replace("P() Q() ", "P() ")); + assertEquals(Set.of(-1), removed.followFacts.get("Q")); + update(removed, grammar); + } + + @Test + void movedLinesRelocateReusedDuplicateAnchors() throws Exception { + String grammar = HEADER + "void Entry():{} { [LOOKAHEAD(2) ] [LOOKAHEAD(2) ] }"; + LookaheadSnapshot after = update(parse(grammar), grammar.replace( + "[LOOKAHEAD(2) ] [", "\n[LOOKAHEAD(2)\n]\n[")); + assertEquals(0, after.statistics.get("sitesRecomputed")); + assertEquals(2, after.statistics.get("sitesReused")); + } + + @Test + void implicitLiteralChangesInvalidateTheTokenNamespace() throws Exception { + String grammar = HEADER + "void Entry():{} { [LOOKAHEAD() ] }\n" + + "void Other():{} { }"; + LookaheadSnapshot after = update(parse(grammar), + grammar.replace("void Other():{} { ", "void Other():{} { \"z\"")); + assertEquals("lexer-options-or-engine-changed", after.statistics.get("fallback")); + } + + @Test + void tokenCertificateDoesNotInheritAnOpaqueContextClaim() throws Exception { + String grammar = HEADER.replace("class Fixture {}", "class Fixture {boolean flag;}") + + "void Entry():{} { [LOOKAHEAD() ] }\n" + + "void Other():{} { }"; + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, + grammar.replace("void Other():{} { ", "void Other():{} { ")); + assertFalse(before.pureGrammar); + Map context = binding(before, "Entry", "fingerprint"); + assertEquals(List.copyOf(context.keySet()), + LookaheadSnapshot.compare(before, after, context).get("invalidatedEvidence")); + Map certificate = binding(before, "Entry", "tokenFingerprint"); + String key = certificate.keySet().iterator().next(); + Map scoped = LookaheadSnapshot.compare(before, after, certificate, + Map.of(key, "token-probe")); + assertEquals(List.of(key), scoped.get("tokenCertificatesReused")); + assertEquals(List.of(), scoped.get("contextEvidenceReused")); + } + + @Test + void probeOnlyCallersAreNotRuntimeContinuations() throws Exception { + String grammar = HEADER + "void Entry():{} { Opt() }\n" + + "void Opt():{} { (LOOKAHEAD(2) | ) }\n" + + "void Other():{} { }"; + LookaheadSnapshot before = parse(grammar); + Map evidence = binding(before, "Opt", "fingerprint"); + String probing = grammar.replace("void Other():{} { ", + "void Other():{} { (LOOKAHEAD(Opt()) | )"); + LookaheadSnapshot after = update(before, probing); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, after, evidence).get("contextEvidenceReused")); + String calling = grammar.replace("void Other():{} { ", "void Other():{} { Opt()"); + LookaheadSnapshot actual = update(before, calling); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, actual, evidence).get("invalidatedEvidence")); + } + + @Test + void probeCalleesInCallerSuffixesInvalidateAnUnchangedCallee() throws Exception { + String grammar = HEADER + "void Entry():{} { Opt() " + + "(LOOKAHEAD(Probe()) | ) }\n" + + "void Probe():{} { }\n" + + "void Opt():{} { [LOOKAHEAD(2) ] }"; + String changed = grammar.replace("void Probe():{} { ", "void Probe():{} { "); + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, changed); + Map evidence = binding(before, "Opt", "fingerprint"); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, after, evidence).get("invalidatedEvidence")); + try (GeneratedLookaheadParser original = + new GeneratedLookaheadParser(directory.resolve("before"), grammar); + GeneratedLookaheadParser current = + new GeneratedLookaheadParser(directory.resolve("after"), changed)) { + assertTrue(original.accepts("ab")); + assertFalse(current.accepts("ab")); + assertTrue(original.accepts("ddac")); + assertFalse(current.accepts("ddac")); + } + } + + @Test + void changedProbeFirstRecomputesRowsWithoutAnActualCallEdge() throws Exception { + String grammar = HEADER + "void Entry():{} { " + + "(LOOKAHEAD(Probe()) | ) }\n" + + "void Probe():{} { }"; + LookaheadSnapshot after = update(parse(grammar), + grammar.replace("void Probe():{} { ", "void Probe():{} { ")); + assertEquals(1, after.statistics.get("sitesRecomputed")); + assertEquals(List.of(""), after.sites.get(0).get("probeFirst")); + } + + @Test + void inlineNamedRegexChangesTheGlobalLexerEvenWhenItsLabelIsStable() throws Exception { + String grammar = HEADER + "void Entry():{} { (LOOKAHEAD(2) | ) }\n" + + "void Other():{} { }"; + String changed = grammar.replace("", ""); + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, changed); + assertNotEquals(before.globalHash, after.globalHash); + assertEquals("lexer-options-or-engine-changed", after.statistics.get("fallback")); + try (GeneratedLookaheadParser original = + new GeneratedLookaheadParser(directory.resolve("before"), grammar); + GeneratedLookaheadParser current = + new GeneratedLookaheadParser(directory.resolve("after"), changed)) { + assertTrue(original.accepts("aa")); + assertFalse(current.accepts("aa")); + } + } + + @Test + void inlineRegexRangesNegationAndRepetitionAreAllBound() throws Exception { + for (List pair : List.of(List.of("([\"x\"-\"z\"])+", "([\"w\"-\"z\"])+"), + List.of("~[\"x\"]", "~[\"y\"]"), + List.of("(\"x\"){2,3}", "(\"x\"){2,4}"))) { + String grammar = HEADER + "void Entry():{} { [LOOKAHEAD() ] }\n" + + "void Other():{} { }"; + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, grammar.replace(pair.get(0), pair.get(1))); + assertNotEquals(before.globalHash, after.globalHash, pair.toString()); + } + String grammar = HEADER + "void Entry():{} { }"; + assertEquals(parse(grammar).globalHash, + parse(grammar.replace(" TOKEN:{:S|:T}\n" + + " TOKEN:{|}\n TOKEN:{}\n" + + "void Entry():{} {

Opt() (|)}\n" + + "void Opt():{} {[LOOKAHEAD(2)]}"; + String changed = grammar.replace("{

Opt()", "{ Opt()"); + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, changed); + assertFalse(before.pureGrammar); + assertFalse(after.pureGrammar); + Map evidence = binding(before, "Opt", "fingerprint"); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, after, evidence).get("invalidatedEvidence")); + try (GeneratedLookaheadParser original = + new GeneratedLookaheadParser(directory.resolve("before"), grammar); + GeneratedLookaheadParser removed = new GeneratedLookaheadParser( + directory.resolve("removed"), grammar.replace("LOOKAHEAD(2)", "")); + GeneratedLookaheadParser current = + new GeneratedLookaheadParser(directory.resolve("after"), changed); + GeneratedLookaheadParser currentRemoved = new GeneratedLookaheadParser( + directory.resolve("after-removed"), changed.replace("LOOKAHEAD(2)", ""))) { + assertTrue(original.accepts("pb")); + assertTrue(removed.accepts("pb")); + assertFalse(original.accepts("pac")); + assertFalse(removed.accepts("pac")); + assertTrue(current.accepts("qac")); + assertFalse(currentRemoved.accepts("qac")); + } + } + + static Stream callerPrefixChanges() { + String opt = "void Opt():{} { [LOOKAHEAD(2) ] }"; + String direct = "void Entry():{} { (LOOKAHEAD() Opt() | ) }"; + String probe = "void Entry():{} { (LOOKAHEAD(Probe()) Opt() | ) }" + + "void Probe():{} { }"; + String preceding = "void Entry():{} { (LOOKAHEAD() Gate() Opt() | ) }" + + "void Gate():{} { [LOOKAHEAD() ] }"; + String competitor = "void Entry():{} { (LOOKAHEAD() | Opt() ) }"; + return Stream.of( + Arguments.of("direct-prefix", direct + opt, + direct.replace("LOOKAHEAD()", "LOOKAHEAD()") + opt, + "ac", Set.of("abac", "d")), + Arguments.of("same-FIRST-probe-target", probe + opt, + probe.replace("void Probe():{} { }", "void Probe():{} { }") + + opt, + "ac", Set.of("abac", "d")), + Arguments.of("preceding-callee", preceding + opt, + preceding.replace("LOOKAHEAD()", "LOOKAHEAD()") + opt, + "dac", Set.of("b", "dabac")), + Arguments.of("earlier-competitor", competitor + opt, + competitor.replace("LOOKAHEAD()", "LOOKAHEAD()") + opt, + "ac", Set.of("abac", "ac"))); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("callerPrefixChanges") + void callersCanChangeReachableInputsBeforeAnUnchangedCall(String name, String original, + String changed, String witness, Set language) throws Exception { + String grammar = HEADER + original; + String current = HEADER + changed; + LookaheadSnapshot before = parse(grammar); + LookaheadSnapshot after = update(before, current); + assertTrue(before.pureGrammar); + assertTrue(after.pureGrammar); + Map evidence = binding(before, "Opt", "fingerprint"); + assertEquals(List.copyOf(evidence.keySet()), + LookaheadSnapshot.compare(before, after, evidence).get("invalidatedEvidence")); + try (GeneratedLookaheadParser old = + new GeneratedLookaheadParser(directory.resolve(name + "-old"), grammar); + GeneratedLookaheadParser oldDeleted = new GeneratedLookaheadParser( + directory.resolve(name + "-old-deleted"), + grammar.replace("LOOKAHEAD(2)", "")); + GeneratedLookaheadParser revised = + new GeneratedLookaheadParser(directory.resolve(name + "-current"), current); + GeneratedLookaheadParser revisedDeleted = new GeneratedLookaheadParser( + directory.resolve(name + "-current-deleted"), + current.replace("LOOKAHEAD(2)", ""))) { + Set accepted = new TreeSet<>(); + for (String input : finiteInputs()) { + boolean expected = old.accepts(input); + assertEquals(expected, oldDeleted.accepts(input), input); + if (expected) { + accepted.add(input); + } + } + assertEquals(language, accepted); + assertTrue(revised.accepts(witness)); + assertFalse(revisedDeleted.accepts(witness)); + } + } + + private static List finiteInputs() { + List result = new ArrayList<>(List.of("")); + List previous = List.of(""); + for (int length = 0; length < 5; length++) { + List next = new ArrayList<>(); + for (String prefix : previous) { + for (String token : List.of("a", "b", "c", "d")) { + next.add(prefix + token); + } + } + result.addAll(next); + previous = next; + } + return result; + } + +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadLexer.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadLexer.java new file mode 100644 index 000000000..1be884e57 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadLexer.java @@ -0,0 +1,77 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.util.*; +import org.javacc.parser.*; + +/** Binds the complete source of token definitions, including named inline regular expressions. */ +final class LookaheadLexer { + final Map namespace = new TreeMap<>(); + boolean fallback; + + LookaheadLexer(Context context, Token first) { + Map positions = new HashMap<>(); + for (Token token = first; token != null; token = token.next) { + positions.put(position(token.beginLine, token.beginColumn), token); + } + context.globals().rexps_of_tokens.forEach((ordinal, expression) -> { + if (ordinal == 0) { + namespace.put(ordinal, "EOF"); + } else { + Token start = positions.get(position(expression.getLine(), expression.getColumn())); + List definition = definition(start); + if (definition == null || expression.tpContext == null) { + fallback = true; + namespace.put(ordinal, List.of(expression.label, "unmapped-definition")); + } else { + TokenProduction owner = expression.tpContext; + namespace.put(ordinal, List.of(expression.label, definition, owner.kind, + owner.ignoreCase, owner.lexStates == null ? List.of("ALL-STATES") + : Arrays.asList(owner.lexStates))); + } + } + }); + if (fallback) { + List all = new ArrayList<>(); + for (Token token = first; token != null && token.kind != 0; token = token.next) { + all.add(List.of(token.kind, token.image)); + } + namespace.put(-1, LookaheadAnalysis.sha(LookaheadAnalysis.json(all))); + } + } + + private static String position(int line, int column) { + return line + ":" + column; + } + + private static List definition(Token start) { + if (start == null) { + return null; + } + if (start.image.startsWith("\"")) { + return List.of(List.of(start.kind, start.image)); + } + if (!start.image.equals("<")) { + return null; + } + List tokens = new ArrayList<>(); + int depth = 0; + for (Token token = start; token != null && token.kind != 0; token = token.next) { + tokens.add(List.of(token.kind, token.image)); + if (token.image.equals("<")) { + depth++; + } else if (token.image.equals(">") && --depth == 0) { + return tokens; + } + } + return null; + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadPurity.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadPurity.java new file mode 100644 index 000000000..170e31ea9 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadPurity.java @@ -0,0 +1,136 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.util.*; +import java.util.stream.Collectors; +import org.javacc.parser.*; + +/** A deliberately small effect-free grammar subset. Anything else keeps the Java fallback. */ +final class LookaheadPurity { + private LookaheadPurity() {} + + static boolean isPure(Context context) { + if (!plainParser(context) || !plainLexer(context) || customRuntime()) { + return false; + } + for (NormalProduction production : context.globals().bnfproductions) { + if (!(production instanceof BNFProduction) + || !((BNFProduction) production).getDeclarationTokens().isEmpty() + || !production.getParameterListTokens().isEmpty() + || !production.getThrowsList().isEmpty() + || !tokens(production.getReturnTypeTokens()).equals("void") + || !pureExpansion(production.getExpansion(), + Collections.newSetFromMap(new IdentityHashMap<>()))) { + return false; + } + } + return true; + } + + private static boolean plainParser(Context context) { + List java = new ArrayList<>(context.globals().cu_to_insertion_point_1); + java.addAll(context.globals().cu_to_insertion_point_2); + java.addAll(context.globals().cu_from_insertion_point_2); + String body = tokens(java); + String name = context.globals().cu_name; + return body.equals("public class " + name + " { }") + || body.equals("class " + name + " { }"); + } + + private static String tokens(List tokens) { + return tokens.stream().map(t -> t.image).collect(Collectors.joining(" ")); + } + + private static boolean customRuntime() { + return Options.getUserCharStream() || Options.getUserTokenManager() + || Options.getTokenManagerUsesParser() || !blank(Options.getTokenFactory()) + || !blank(Options.getTokenExtends()) || !blank(Options.getTokenManagerSuperClass()) + || !blank(Options.getTokenClass()) && !Options.getTokenClass().equals("Token"); + } + + private static boolean blank(String value) { + return value == null || value.isEmpty(); + } + + private static boolean plainLexer(Context context) { + if (context.globals().lexstate_I2S.size() != 1 + || !context.globals().lexstate_I2S.containsValue("DEFAULT") + || context.globals().nextStateForEof != null + || context.globals().token_mgr_decls != null + && !context.globals().token_mgr_decls.isEmpty() + || context.globals().actForEof != null + && !context.globals().actForEof.getActionTokens().isEmpty()) { + return false; + } + for (TokenProduction production : context.globals().rexprlist) { + if (!plainTokenProduction(production)) { + return false; + } + } + return true; + } + + private static boolean plainTokenProduction(TokenProduction production) { + for (RegExprSpec specification : production.respecs) { + if (specification.nsTok != null) { + return false; + } + } + int braces = 0; + Set end = Collections.newSetFromMap(new IdentityHashMap<>()); + end.add(production.lastToken); + for (Token token = production.firstToken; token != null; token = token.next) { + if (token.image.equals("{")) { + braces++; + } + if (end.contains(token)) { + break; + } + } + return braces <= 1; + } + + private static boolean pureExpansion(Expansion expansion, Set seen) { + if (expansion == null || !seen.add(expansion)) { + return expansion != null; + } + if (expansion instanceof Action || expansion instanceof TryBlock) { + return false; + } + if (expansion instanceof NonTerminal) { + return pureCall((NonTerminal) expansion); + } + if (expansion instanceof RegularExpression) { + return pureToken((RegularExpression) expansion); + } + if (expansion instanceof Lookahead + && !pureLookahead((Lookahead) expansion, seen)) { + return false; + } + return LookaheadAnalysis.children(expansion).stream() + .allMatch(child -> pureExpansion(child, seen)); + } + + private static boolean pureCall(NonTerminal call) { + return call.getArgumentTokens().isEmpty() && call.getLhsTokens().isEmpty() + && call.getParametrizedTypeTokens().isEmpty(); + } + + private static boolean pureToken(RegularExpression token) { + return token.lhsTokens.isEmpty() && token.rhsToken == null; + } + + private static boolean pureLookahead(Lookahead lookahead, Set seen) { + return lookahead.getActionTokens().isEmpty() + && pureExpansion(lookahead.getLaExpansion(), seen); + } + +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadSnapshot.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadSnapshot.java new file mode 100644 index 000000000..dc4118a99 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadSnapshot.java @@ -0,0 +1,671 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import java.io.StringReader; +import java.util.*; +import java.util.stream.Collectors; +import org.javacc.parser.*; + +/** Validated syntax plus versioned lexical facts; Java effects remain conservative. */ +final class LookaheadSnapshot { + final Map> productions = new TreeMap<>(); + final List> sites = new ArrayList<>(); + final Map anchors = new HashMap<>(); + final Map>> positions = new TreeMap<>(); + final Map> firstFacts = new TreeMap<>(); + final Map> followFacts = new TreeMap<>(); + final Map>> followRules = new TreeMap<>(); + final Map firstInputs = new TreeMap<>(); + final Map ruleInputs = new TreeMap<>(); + final Map followInputs = new TreeMap<>(); + final Map rowInputs = new TreeMap<>(); + final Map statistics = new TreeMap<>(); + final Map> dependencyCache = new TreeMap<>(); + final Map dependencyDigests = new TreeMap<>(); + String globalHash; + String opaqueHash; + String grammarHash; + String externalHash; + String engineHash; + boolean pureGrammar; + + static LookaheadSnapshot parse(String grammar, String external, String engine) + throws Exception { + return parse(grammar, external, engine, null); + } + + static LookaheadSnapshot parse(String grammar, String external, String engine, + LookaheadSnapshot previous) throws Exception { + Context context = new Context(); + JavaCCParser parser = new JavaCCParser(new StringReader(grammar)); + Token first = parser.getToken(1); + parser.javacc_input(context); + Semanticize.start(context); + if (context.errors().get_error_count() != 0) { + throw new IllegalArgumentException("Invalid grammar"); + } + LookaheadSnapshot snapshot = new LookaheadSnapshot(); + snapshot.externalHash = external; + snapshot.engineHash = engine; + Set owned = Collections.newSetFromMap(new IdentityHashMap<>()); + List opaque = new ArrayList<>(); + LookaheadAnalysis facts = new LookaheadAnalysis(context); + for (NormalProduction production : context.globals().bnfproductions) { + snapshot.recordProduction(production, owned, opaque, facts); + } + Collections.sort(opaque); + LookaheadLexer lexer = new LookaheadLexer(context, first); + snapshot.globalHash = LookaheadAnalysis.sha(globalFingerprint(first, owned) + "\n" + + LookaheadAnalysis.json(lexer.namespace)); + snapshot.statistics.put("lexicalNamespaceFallback", lexer.fallback); + snapshot.opaqueHash = LookaheadAnalysis.sha(String.join("\n", opaque)); + snapshot.grammarHash = LookaheadAnalysis.sha(LookaheadAnalysis.json(snapshot.productions)); + snapshot.pureGrammar = LookaheadPurity.isPure(context); + LookaheadSnapshot usable = previous != null + && previous.globalHash.equals(snapshot.globalHash) + && previous.engineHash.equals(engine) ? previous : null; + new LookaheadIncremental(snapshot, usable, facts).compute(); + snapshot.statistics.put("javaCCParses", 1); + snapshot.statistics.put("parsedProductions", snapshot.productions.size()); + if (previous != null && usable == null) { + snapshot.statistics.put("fallback", "lexer-options-or-engine-changed"); + } + snapshot.recordSites(facts.rows, grammar.startsWith("/*@bgen")); + snapshot.prepareDependencies(usable); + return snapshot; + } + + private static List productionBody(NormalProduction production, Set owned) { + List body = new ArrayList<>(); + Token token = production.getFirstToken(); + if (token == null || production.getLastToken() == null) { + throw new IllegalStateException("Missing production bounds"); + } + Set end = Collections.newSetFromMap(new IdentityHashMap<>()); + end.add(production.getLastToken()); + while (true) { + owned.add(token); + body.add(token.image); + if (end.contains(token)) { + return body; + } + token = token.next; + if (token == null) { + throw new IllegalStateException("Unclosed production"); + } + } + } + + private void recordProduction(NormalProduction production, Set owned, + List opaque, LookaheadAnalysis facts) { + List body = productionBody(production, owned); + Set calls = new TreeSet<>(); + List code = new ArrayList<>(); + if (production instanceof BNFProduction) { + addCode(code, "declarations", ((BNFProduction) production).getDeclarationTokens()); + } else { + code.add("opaque-production:" + String.join(" ", body)); + } + code.add("access:" + production.getAccessMod()); + addCode(code, "return-type", production.getReturnTypeTokens()); + addCode(code, "parameters", production.getParameterListTokens()); + for (List exception : production.getThrowsList()) { + addCode(code, "throws", exception); + } + walk(production.getExpansion(), "", production.getLhs(), null, + Collections.newSetFromMap(new IdentityHashMap<>()), calls, code); + if (!code.isEmpty()) { + opaque.add(production.getLhs() + ":" + LookaheadAnalysis.sha(String.join("\n", code))); + } + Map value = new TreeMap<>(); + value.put("hash", LookaheadAnalysis.sha(String.join(" ", body))); + value.put("calls", calls); + value.put("actualCalls", LookaheadGraph.calls(production.getExpansion(), false)); + value.put("nullable", production.getExpansion() == null ? "unknown" + : Semanticize.emptyExpansionExists(production.getExpansion())); + productions.put(production.getLhs(), value); + facts.recordParents(production.getExpansion(), null); + } + + private static String globalFingerprint(Token first, Set owned) { + List globals = new ArrayList<>(); + for (Token token = first; token != null && token.kind != 0; token = token.next) { + if (!owned.contains(token)) { + globals.add(token.image); + } + } + return LookaheadAnalysis.sha(String.join(" ", globals)); + } + + private void recordSites(List> rows, boolean generated) { + for (Map row : rows) { + String anchor = anchors.get(location(row)); + if (anchor == null) { + throw new IllegalStateException("Unmapped explicit lookahead " + row); + } + row.put("anchor", anchor); + if (generated) { + row.put("line", ((Integer) row.get("line")) - 1); + } + sites.add(row); + } + int siteCount = sites.size(); + int anchorCount = anchors.size(); + if (siteCount != anchorCount) { + throw new IllegalStateException("Lookahead count mismatch"); + } + } + + private static String tokens(List tokens) { + return tokens.stream().map(t -> t.image).collect(Collectors.joining(" ")); + } + + private static void addCode(List target, String kind, List tokens) { + if (!tokens.isEmpty()) { + target.add(kind + ":" + tokens(tokens)); + } + } + + static String shape(Expansion expansion) { + if (expansion instanceof Lookahead) { + return ""; + } + if (expansion instanceof RegularExpression) { + RegularExpression re = (RegularExpression) expansion; + if (!re.label.isEmpty()) { + return "T:" + re.label; + } + return re instanceof RStringLiteral + ? "L:" + LookaheadAnalysis.json(((RStringLiteral) re).image) + : "R:" + re.getClass().getSimpleName(); + } + if (expansion instanceof NonTerminal) { + NonTerminal nt = (NonTerminal) expansion; + return "N:" + nt.getName() + "(" + tokens(nt.getArgumentTokens()) + ")"; + } + if (expansion instanceof Action) { + return "Action"; + } + return expansion.getClass().getSimpleName() + LookaheadAnalysis.children(expansion).stream() + .map(LookaheadSnapshot::shape).filter(s -> !s.isEmpty()) + .collect(Collectors.joining(",", "[", "]")); + } + + static String continuationShape(Expansion expansion) { + return continuationShape(expansion, Collections.newSetFromMap(new IdentityHashMap<>())); + } + + private static String continuationShape(Expansion expansion, Set path) { + if (expansion == null) { + return "unknown"; + } + if (!path.add(expansion)) { + return "actual-branch"; + } + try { + if (expansion instanceof Lookahead) { + Lookahead lookahead = (Lookahead) expansion; + return lookahead.isExplicit() ? "lookahead:" + lookahead.getAmount() + ":" + + tokens(lookahead.getActionTokens()) + ":" + + continuationShape(lookahead.getLaExpansion(), path) : ""; + } + if (expansion instanceof RegularExpression || expansion instanceof NonTerminal) { + return shape(expansion); + } + if (expansion instanceof Action) { + return "action:" + tokens(((Action) expansion).getActionTokens()); + } + List children = new ArrayList<>(); + for (Expansion child : LookaheadAnalysis.children(expansion)) { + children.add(continuationShape(child, path)); + } + return expansion.getClass().getSimpleName() + LookaheadAnalysis.json(children); + } finally { + path.remove(expansion); + } + } + + private void walk(Expansion expansion, String path, String name, Expansion parent, + Set seen, Set calls, List code) { + if (expansion == null || !seen.add(expansion)) { + return; + } + recordExpansionCode(expansion, calls, code); + if (expansion instanceof Lookahead) { + Lookahead la = (Lookahead) expansion; + recordLookahead(la, path, name, parent, code); + walk(la.getLaExpansion(), path + "/probe", name, expansion, seen, calls, code); + } + for (Expansion child : LookaheadAnalysis.children(expansion)) { + walk(child, path + "/" + expansion.getClass().getSimpleName(), name, expansion, seen, + calls, code); + } + } + + private static void recordExpansionCode(Expansion expansion, Set calls, + List code) { + if (expansion instanceof NonTerminal) { + NonTerminal nt = (NonTerminal) expansion; + calls.add(nt.getName()); + addCode(code, "assignment", nt.getLhsTokens()); + addCode(code, "type-arguments", nt.getParametrizedTypeTokens()); + addCode(code, "arguments:" + nt.getName(), nt.getArgumentTokens()); + } + if (expansion instanceof RegularExpression) { + RegularExpression token = (RegularExpression) expansion; + addCode(code, "assignment", token.lhsTokens); + if (token.rhsToken != null) { + addCode(code, "token-field", List.of(token.rhsToken)); + } + } + if (expansion instanceof Action) { + addCode(code, "action", ((Action) expansion).getActionTokens()); + } + if (expansion instanceof TryBlock) { + recordRecoveryCode((TryBlock) expansion, code); + } + } + + private static void recordRecoveryCode(TryBlock block, List code) { + for (List type : block.types) { + addCode(code, "catch-type", type); + } + for (List body : block.catchblks) { + addCode(code, "catch-body", body); + } + if (block.finallyblk != null) { + addCode(code, "finally-body", block.finallyblk); + } + } + + private void recordLookahead(Lookahead la, String path, String name, Expansion parent, + List code) { + if (!la.isExplicit()) { + return; + } + addCode(code, "predicate", la.getActionTokens()); + String key = name + ":" + la.getLine() + ":" + la.getColumn(); + String anchor = name + ":" + + LookaheadAnalysis.sha(path + "|" + shape(parent)).substring(0, 16); + if (anchors.put(key, anchor) != null) { + throw new IllegalStateException("Duplicate lookahead location"); + } + Map position = new TreeMap<>(); + position.put("line", la.getLine()); + position.put("column", la.getColumn()); + positions.computeIfAbsent(anchor, k -> new ArrayList<>()).add(position); + } + + int reuseRows(String production, LookaheadSnapshot previous, List> rows) { + Map indexes = new HashMap<>(); + int reused = 0; + for (Map old : previous.sites) { + if (old.get("production").equals(production)) { + String anchor = old.get("anchor").toString(); + int index = indexes.getOrDefault(anchor, 0); + Map row = new TreeMap<>(old); + row.remove("anchor"); + row.putAll(positions.get(anchor).get(index)); + indexes.put(anchor, index + 1); + rows.add(row); + reused++; + } + } + return reused; + } + + Map cacheValue() { + Map value = new TreeMap<>(); + value.put("productions", productions); + value.put("sites", sites); + value.put("anchors", anchors); + value.put("positions", positions); + value.put("firstFacts", firstFacts); + value.put("followFacts", followFacts); + value.put("followRules", followRules); + value.put("firstInputs", firstInputs); + value.put("ruleInputs", ruleInputs); + value.put("followInputs", followInputs); + value.put("rowInputs", rowInputs); + value.put("dependencies", dependencyCache); + value.put("dependencyDigests", dependencyDigests); + value.put("global", globalHash); + value.put("opaque", opaqueHash); + value.put("grammar", grammarHash); + value.put("engine", engineHash); + value.put("pure", pureGrammar); + return value; + } + + @SuppressWarnings("unchecked") + static LookaheadSnapshot fromCache(Object value) { + Map data = (Map) value; + LookaheadSnapshot result = new LookaheadSnapshot(); + result.productions.putAll((Map>) data.get("productions")); + result.sites.addAll((List>) data.get("sites")); + result.anchors.putAll((Map) data.get("anchors")); + result.positions.putAll((Map>>) data.get("positions")); + result.firstFacts.putAll((Map>) data.get("firstFacts")); + result.followFacts.putAll((Map>) data.get("followFacts")); + result.followRules.putAll((Map>>) data.get("followRules")); + result.firstInputs.putAll((Map) data.get("firstInputs")); + result.ruleInputs.putAll((Map) data.get("ruleInputs")); + result.followInputs.putAll((Map) data.get("followInputs")); + result.rowInputs.putAll((Map) data.get("rowInputs")); + result.dependencyCache.putAll( + (Map>) data.get("dependencies")); + result.dependencyDigests.putAll((Map) data.get("dependencyDigests")); + result.globalHash = Objects.requireNonNull((String) data.get("global")); + result.opaqueHash = Objects.requireNonNull((String) data.get("opaque")); + result.grammarHash = Objects.requireNonNull((String) data.get("grammar")); + result.engineHash = Objects.requireNonNull((String) data.get("engine")); + result.pureGrammar = (Boolean) data.get("pure"); + return result; + } + + private static String location(Map row) { + return row.get("production") + ":" + row.get("line") + ":" + row.get("column"); + } + + @SuppressWarnings("unchecked") + Map>> contexts() { + Map>> result = new TreeMap<>(); + for (Map site : sites) { + String production = site.get("production").toString(); + Map roots = dependencyCache.get(production); + Map facts = new TreeMap<>(site); + facts.keySet().removeAll(Set.of("line", "column", "anchor")); + Map local = new TreeMap<>(); + local.put("facts", LookaheadAnalysis.sha(LookaheadAnalysis.json(facts))); + local.put("dependencies", dependencyDigests.get(production)); + String localFingerprint = LookaheadAnalysis.sha(LookaheadAnalysis.json(local)); + Map context = new TreeMap<>(); + context.put("local", localFingerprint); + context.put("global", globalHash); + context.put("opaque", pureGrammar ? "effect-free-grammar" : opaqueHash); + context.put("grammar", pureGrammar ? "dependent-productions" : grammarHash); + context.put("external", externalHash); + context.put("engine", engineHash); + Map binding = new TreeMap<>(); + binding.put("production", site.get("production")); + binding.put("line", site.get("line")); + binding.put("triage", site.get("triage")); + binding.put("facts", facts); + binding.put("dependencies", roots); + binding.put("localFingerprint", localFingerprint); + binding.put("fingerprint", LookaheadAnalysis.sha(LookaheadAnalysis.json(context))); + if (Boolean.TRUE.equals(site.get("tokenProbe"))) { + Map certificate = new TreeMap<>(); + certificate.put("scope", "token-probe-v1"); + certificate.put("amount", site.get("amount")); + certificate.put("tokens", site.get("probeFirst")); + certificate.put("global", globalHash); + certificate.put("external", externalHash); + certificate.put("engine", engineHash); + binding.put("tokenFingerprint", + LookaheadAnalysis.sha(LookaheadAnalysis.json(certificate))); + } + result.computeIfAbsent(site.get("anchor").toString(), k -> new ArrayList<>()) + .add(binding); + } + return result; + } + + private void prepareDependencies(LookaheadSnapshot previous) { + LookaheadGraph graph = new LookaheadGraph(productions); + Map> actualCallers = + new LookaheadGraph(productions, "actualCalls").callers; + Map>> incoming = new TreeMap<>(); + productions.keySet().forEach(name -> incoming.put(name, new ArrayList<>())); + followRules.values().forEach(rules -> rules.forEach( + rule -> incoming.get(rule.get("target")).add(rule))); + Map continuationDigests = new TreeMap<>(); + incoming.forEach((name, rules) -> { + List descriptions = new ArrayList<>(); + for (Map rule : rules) { + descriptions.add(LookaheadAnalysis.json(List.of(rule.get("caller"), + rule.get("suffix"), rule.get("calls")))); + } + Collections.sort(descriptions); + continuationDigests.put(name, + LookaheadAnalysis.sha(LookaheadAnalysis.json(descriptions))); + }); + int recomputed = 0; + Set owners = new TreeSet<>(); + sites.forEach(site -> owners.add(site.get("production").toString())); + for (String owner : owners) { + Map old = previous == null ? null : previous.dependencyCache.get(owner); + if (old != null && dependenciesMatch(old, continuationDigests)) { + dependencyCache.put(owner, old); + dependencyDigests.put(owner, previous.dependencyDigests.get(owner)); + } else { + Map value = + dependencyHashes(owner, graph, actualCallers, continuationDigests); + dependencyCache.put(owner, value); + dependencyDigests.put(owner, LookaheadAnalysis.sha(LookaheadAnalysis.json(value))); + recomputed++; + } + } + statistics.put("dependencyContextsRecomputed", recomputed); + statistics.put("dependencyContextsReused", owners.size() - recomputed); + } + + private boolean dependenciesMatch(Map old, + Map continuations) { + for (Map.Entry entry : old.entrySet()) { + String name = entry.getKey(); + Object current = name.startsWith("caller-continuations:") + ? continuations.get(name.substring("caller-continuations:".length())) + : productions.getOrDefault(name, Map.of()).get("hash"); + if (!entry.getValue().equals(current)) { + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") + private Map dependencyHashes(String production, LookaheadGraph graph, + Map> actualCallers, + Map continuationDigests) { + // Prefix probes and competing alternatives can restrict which inputs reach this call. + Set ancestors = LookaheadGraph.closure(Set.of(production), actualCallers); + Set needed = LookaheadGraph.closure(ancestors, graph.calls); + Map result = new TreeMap<>(); + for (String name : needed) { + if (!productions.containsKey(name)) { + throw new IllegalStateException("Unresolved production " + name); + } + result.put(name, productions.get(name).get("hash")); + } + ancestors.forEach(target -> result.put("caller-continuations:" + target, + continuationDigests.get(target))); + return result; + } + + static Map compare(LookaheadSnapshot before, LookaheadSnapshot after, + Map evidence) { + return compare(before, after, evidence, Map.of()); + } + + static Map compare(LookaheadSnapshot before, LookaheadSnapshot after, + Map evidence, Map scopes) { + if (!Set.of("context", "token-probe").containsAll(scopes.values())) { + throw new IllegalArgumentException("Unknown evidence scope"); + } + Comparison comparison = new Comparison(before.contexts(), after.contexts(), evidence, + scopes); + comparison.collect(); + return comparison.report(before.sites.size(), after.sites.size(), + globalChanges(before, after)); + } + + private static List globalChanges(LookaheadSnapshot before, LookaheadSnapshot after) { + List globals = new ArrayList<>(); + if (!before.globalHash.equals(after.globalHash)) { + globals.add("parser-options-helpers-or-lexer"); + } + boolean pure = before.pureGrammar && after.pureGrammar; + if (!pure && !before.opaqueHash.equals(after.opaqueHash)) { + globals.add("opaque-Java-or-predicate"); + } + if (!pure && !before.grammarHash.equals(after.grammarHash)) { + globals.add("grammar-visible-to-opaque-Java"); + } + if (!before.externalHash.equals(after.externalHash)) { + globals.add("production-Java-resources-or-build"); + } + if (!before.engineHash.equals(after.engineHash)) { + globals.add("generator-or-analysis-version"); + } + return globals; + } + + private static final class Comparison { + private final Map>> old; + private final Map>> current; + private final Map evidence; + private final Map scopes; + private final List> changed = new ArrayList<>(); + private final List reused = new ArrayList<>(); + private final List contextReused = new ArrayList<>(); + private final List tokensReused = new ArrayList<>(); + private final List stale = new ArrayList<>(); + private final List ambiguous = new ArrayList<>(); + private final Set direct = new TreeSet<>(); + private final Set roots = new TreeSet<>(); + + private Comparison(Map>> old, + Map>> current, Map evidence, + Map scopes) { + this.old = old; + this.current = current; + this.evidence = evidence; + this.scopes = scopes; + } + + private void collect() { + Set keys = new TreeSet<>(old.keySet()); + keys.addAll(current.keySet()); + keys.addAll(evidence.keySet()); + for (String key : keys) { + List> before = old.getOrDefault(key, List.of()); + List> after = current.getOrDefault(key, List.of()); + String kind = changeKind(key, before, after); + if (kind != null) { + recordFinding(key, kind, before, after); + } + if (evidence.containsKey(key)) { + recordEvidence(key, after); + } + } + } + + private String changeKind(String key, List> before, + List> after) { + if (before.isEmpty() && after.isEmpty()) { + return "unbound-evidence"; + } + if (before.size() > 1 || after.size() > 1) { + ambiguous.add(key); + return localFingerprints(before).equals(localFingerprints(after)) ? null + : "ambiguous-context-changed"; + } + if (before.isEmpty()) { + return "added"; + } + if (after.isEmpty()) { + return "removed"; + } + return before.get(0).get("localFingerprint") + .equals(after.get(0).get("localFingerprint")) + ? null + : "context-changed"; + } + + private static List localFingerprints(List> bindings) { + return bindings.stream().map(x -> x.get("localFingerprint").toString()).sorted() + .collect(Collectors.toList()); + } + + private void recordFinding(String key, String kind, List> before, + List> after) { + Map finding = new TreeMap<>(); + finding.put("anchor", key); + finding.put("kind", kind); + finding.put("beforeLines", + before.stream().map(x -> x.get("line")).collect(Collectors.toList())); + finding.put("afterLines", + after.stream().map(x -> x.get("line")).collect(Collectors.toList())); + List> available = after.isEmpty() ? before : after; + if (!available.isEmpty()) { + finding.put("triage", available.get(0).get("triage")); + } + if (before.size() == 1 && after.size() == 1) { + recordFacts(key, finding, before.get(0), after.get(0)); + } + if (kind.equals("added") || kind.equals("removed")) { + direct.add(key); + } + changed.add(finding); + } + + @SuppressWarnings("unchecked") + private void recordFacts(String key, Map finding, + Map before, + Map after) { + boolean factChange = !before.get("facts").equals(after.get("facts")); + finding.put("localFactsChanged", factChange); + if (factChange) { + direct.add(key); + } + Map oldDependencies = (Map) before.get("dependencies"); + Map newDependencies = (Map) after.get("dependencies"); + Set names = new TreeSet<>(oldDependencies.keySet()); + names.addAll(newDependencies.keySet()); + List differences = names.stream() + .filter(n -> !Objects.equals(oldDependencies.get(n), newDependencies.get(n))) + .collect(Collectors.toList()); + roots.addAll(differences); + finding.put("dependencyChanges", differences); + } + + private void recordEvidence(String key, List> bindings) { + boolean token = scopes.getOrDefault(key, "context").equals("token-probe"); + if (bindings.size() == 1 + && evidence.get(key).equals(bindings.get(0) + .get(token ? "tokenFingerprint" : "fingerprint"))) { + reused.add(key); + (token ? tokensReused : contextReused).add(key); + } else { + stale.add(key); + } + } + + private Map report(int beforeCount, int afterCount, List globals) { + Map report = new TreeMap<>(); + report.put("needsReview", !changed.isEmpty() || !stale.isEmpty() || !globals.isEmpty()); + report.put("changes", changed); + report.put("directDecisions", direct); + report.put("dependencyRoots", roots); + report.put("reusableEvidence", reused); + report.put("contextEvidenceReused", contextReused); + report.put("tokenCertificatesReused", tokensReused); + report.put("invalidatedEvidence", stale); + report.put("globalSignals", globals); + report.put("ambiguousAnchors", ambiguous); + report.put("beforeSites", beforeCount); + report.put("afterSites", afterCount); + report.put("automaticGrammarChanges", 0); + return report; + } + } +} diff --git a/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadTokenProbeTest.java b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadTokenProbeTest.java new file mode 100644 index 000000000..2ac3b31b2 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/parser/audit/LookaheadTokenProbeTest.java @@ -0,0 +1,100 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2019 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.parser.audit; + +import static org.junit.jupiter.api.Assertions.*; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class LookaheadTokenProbeTest { + private static final String HEADER = "options { STATIC=false; }\n" + + "PARSER_BEGIN(Fixture) public class Fixture {} PARSER_END(Fixture)\n" + + "TOKEN: { | | | | }\n"; + @TempDir + Path directory; + + static Stream contexts() { + return Stream.of( + Arguments.of("choice", "", "(PROBE Wide() | ) "), + Arguments.of("optional", "", "[PROBE Wide()] "), + Arguments.of("repeat", "", "(PROBE Wide())* "), + Arguments.of("token-set", "(|)", "[PROBE Wide()] "), + Arguments.of("nested", "", + "(LOOKAHEAD(3) [PROBE Wide()] | ) "), + Arguments.of("nested-set", "(|)", + "(LOOKAHEAD(3) [PROBE Wide()] | ) ")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("contexts") + void boundedSyntacticProbesPreserveAcceptanceAndRestriction(String name, String probe, + String expansion) throws Exception { + String grammar = HEADER + "void Entry():{} { " + expansion.replace("PROBE", + "LOOKAHEAD(" + probe + ")") + " }\n" + + "void Wide():{} { (|||) }"; + String bounded = grammar.replace("LOOKAHEAD(" + probe + ")", + "LOOKAHEAD(1," + probe + ")"); + String deleted = grammar.replace("LOOKAHEAD(" + probe + ")", ""); + int changesWithoutGuard = 0; + try (GeneratedLookaheadParser before = + new GeneratedLookaheadParser(directory.resolve(name + "-before"), grammar); + GeneratedLookaheadParser after = + new GeneratedLookaheadParser(directory.resolve(name + "-after"), bounded); + GeneratedLookaheadParser removed = + new GeneratedLookaheadParser(directory.resolve(name + "-deleted"), + deleted)) { + for (String input : inputs(4)) { + boolean accepted = before.accepts(input); + assertEquals(accepted, after.accepts(input), input); + if (accepted != removed.accepts(input)) { + changesWithoutGuard++; + } + } + } + assertTrue(changesWithoutGuard > 0, "the mutation must expose a necessary restriction"); + } + + @Test + void certificatesRejectSemanticZeroDepthNestedAndMultiTokenProbes() throws Exception { + for (String probe : List.of("LOOKAHEAD(0,)", "LOOKAHEAD(1,,{true})", + "LOOKAHEAD()", "LOOKAHEAD(2,(LOOKAHEAD() ))")) { + LookaheadSnapshot snapshot = LookaheadSnapshot.parse(HEADER + + "void Entry():{} { (" + probe + " | ) }", "external", "engine"); + assertTrue(snapshot.sites.stream() + .noneMatch(site -> Boolean.TRUE.equals(site.get("tokenProbe"))), probe); + assertTrue(snapshot.contexts().values().stream().flatMap(List::stream) + .noneMatch(binding -> binding.containsKey("tokenFingerprint")), probe); + } + } + + private static List inputs(int depth) { + List result = new ArrayList<>(); + result.add(""); + List previous = List.of(""); + for (int i = 0; i < depth; i++) { + List next = new ArrayList<>(); + for (String prefix : previous) { + for (String token : List.of("a", "b", "c", "d", "e")) { + next.add(prefix + token); + } + } + result.addAll(next); + previous = next; + } + return result; + } +}