diff --git a/internal/ast/parseoptions.go b/internal/ast/parseoptions.go index 73e8bfc6c5..a697eceb75 100644 --- a/internal/ast/parseoptions.go +++ b/internal/ast/parseoptions.go @@ -31,8 +31,8 @@ func GetSourceFileAffectingCompilerOptions(fileName string, options *core.Compil } type ExternalModuleIndicatorOptions struct { - jsx bool - force bool + JSX bool + Force bool } func GetExternalModuleIndicatorOptions(fileName string, options *core.CompilerOptions, metadata SourceFileMetaData) ExternalModuleIndicatorOptions { @@ -43,7 +43,7 @@ func GetExternalModuleIndicatorOptions(fileName string, options *core.CompilerOp switch options.GetEmitModuleDetectionKind() { case core.ModuleDetectionKindForce: // All non-declaration files are modules, declaration files still do the usual isFileProbablyExternalModule - return ExternalModuleIndicatorOptions{force: true} + return ExternalModuleIndicatorOptions{Force: true} case core.ModuleDetectionKindLegacy: // Files are modules if they have imports, exports, or import.meta return ExternalModuleIndicatorOptions{} @@ -52,8 +52,8 @@ func GetExternalModuleIndicatorOptions(fileName string, options *core.CompilerOp // If jsx is react-jsx or react-jsxdev then jsx tags force module-ness // otherwise, the presence of import or export statments (or import.meta) implies module-ness return ExternalModuleIndicatorOptions{ - jsx: options.Jsx == core.JsxEmitReactJSX || options.Jsx == core.JsxEmitReactJSXDev, - force: isFileForcedToBeModuleByFormat(fileName, options, metadata), + JSX: options.Jsx == core.JsxEmitReactJSX || options.Jsx == core.JsxEmitReactJSXDev, + Force: isFileForcedToBeModuleByFormat(fileName, options, metadata), } default: return ExternalModuleIndicatorOptions{} @@ -89,13 +89,13 @@ func getExternalModuleIndicator(file *SourceFile, opts ExternalModuleIndicatorOp return nil } - if opts.jsx { + if opts.JSX { if node := isFileModuleFromUsingJSXTag(file); node != nil { return node } } - if opts.force { + if opts.Force { return file.AsNode() } diff --git a/internal/parser/jsdoc.go b/internal/parser/jsdoc.go index db6a24f4d8..477628c0d5 100644 --- a/internal/parser/jsdoc.go +++ b/internal/parser/jsdoc.go @@ -918,8 +918,14 @@ func (p *Parser) parseTypedefTag(start int, tagName *ast.IdentifierNode, indent if childTypeTag != nil && childTypeTag.TypeExpression != nil && !isObjectOrObjectArrayTypeReference(childTypeTag.TypeExpression.Type()) { typeExpression = childTypeTag.TypeExpression } else { - typeExpression = p.finishNode(jsdocTypeLiteral, jsdocPropertyTags[0].Pos()) + // !!! This differs from Strada but prevents a crash + pos := start + if len(jsdocPropertyTags) > 0 { + pos = jsdocPropertyTags[0].Pos() + } + typeExpression = p.finishNode(jsdocTypeLiteral, pos) } + end = typeExpression.End() } } diff --git a/internal/parser/parser.go b/internal/parser/parser.go index bb59c4fc90..7f9e570c83 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -503,7 +503,7 @@ func (p *Parser) parseListIndex(kind ParsingContext, parseElement func(p *Parser list := make([]*ast.Node, 0, 16) for i := 0; !p.isListTerminator(kind); i++ { if p.isListElement(kind, false /*inErrorRecovery*/) { - elt := parseElement(p, i) + elt := parseElement(p, len(list)) if len(p.reparseList) > 0 { for _, e := range p.reparseList { // Propagate @typedef type alias declarations outwards to a context that permits them. @@ -1029,8 +1029,12 @@ func (p *Parser) parseDeclaration() *ast.Statement { func (p *Parser) parseDeclarationWorker(pos int, hasJSDoc bool, modifiers *ast.ModifierList) *ast.Statement { switch p.token { - case ast.KindVarKeyword, ast.KindLetKeyword, ast.KindConstKeyword, ast.KindUsingKeyword, ast.KindAwaitKeyword: + case ast.KindVarKeyword, ast.KindLetKeyword, ast.KindConstKeyword, ast.KindUsingKeyword: return p.parseVariableStatement(pos, hasJSDoc, modifiers) + case ast.KindAwaitKeyword: + if p.isAwaitUsingDeclaration() { + return p.parseVariableStatement(pos, hasJSDoc, modifiers) + } case ast.KindFunctionKeyword: return p.parseFunctionDeclaration(pos, hasJSDoc, modifiers) case ast.KindClassKeyword: @@ -6397,6 +6401,9 @@ func skipNonBlanks(text string, pos int) int { } func skipTo(text string, pos int, s string) int { + if pos >= len(text) { + return -1 + } i := strings.Index(text[pos:], s) if i < 0 { return -1 diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 529d257949..b2f2b9a6e4 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -1,4 +1,4 @@ -package parser +package parser_test import ( "io/fs" @@ -10,7 +10,9 @@ import ( "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/parser" "github.com/microsoft/typescript-go/internal/repo" + "github.com/microsoft/typescript-go/internal/testrunner" "github.com/microsoft/typescript-go/internal/testutil/fixtures" "github.com/microsoft/typescript-go/internal/tspath" "github.com/microsoft/typescript-go/internal/vfs/osvfs" @@ -46,7 +48,7 @@ func BenchmarkParse(b *testing.B) { } for b.Loop() { - ParseSourceFile(opts, sourceText, scriptKind) + parser.ParseSourceFile(opts, sourceText, scriptKind) } }) } @@ -94,7 +96,6 @@ func FuzzParser(f *testing.F) { "src", "scripts", "Herebyfile.mjs", - // "tests/cases", } var extensions collections.Set[string] @@ -111,19 +112,53 @@ func FuzzParser(f *testing.F) { sourceText, err := os.ReadFile(file.path) assert.NilError(f, err) extension := tspath.TryGetExtensionFromPath(file.path) - f.Add(extension, string(sourceText), int32(core.ScriptTargetESNext), uint8(ast.JSDocParsingModeParseAll)) + f.Add(extension, string(sourceText), uint8(ast.JSDocParsingModeParseAll), false, false) } } - f.Fuzz(func(t *testing.T, extension string, sourceText string, scriptTarget_ int32, jsdocParsingMode_ uint8) { - scriptTarget := core.ScriptTarget(scriptTarget_) - jsdocParsingMode := ast.JSDocParsingMode(jsdocParsingMode_) + testDirs := []string{ + filepath.Join(repo.TypeScriptSubmodulePath, "tests/cases/compiler"), + filepath.Join(repo.TypeScriptSubmodulePath, "tests/cases/conformance"), + filepath.Join(repo.TestDataPath, "tests/cases/compiler"), + } - if !extensions.Has(extension) { - t.Skip() + for _, testDir := range testDirs { + if _, err := os.Stat(testDir); os.IsNotExist(err) { + continue + } + + for file := range allParsableFiles(f, testDir) { + sourceText, err := os.ReadFile(file.path) + assert.NilError(f, err) + + type testFile struct { + content string + name string + } + + testUnits, _, _, _, err := testrunner.ParseTestFilesAndSymlinks( + string(sourceText), + file.path, + func(filename string, content string, fileOptions map[string]string) (testFile, error) { + return testFile{content: content, name: filename}, nil + }, + ) + assert.NilError(f, err) + + for _, unit := range testUnits { + extension := tspath.TryGetExtensionFromPath(unit.name) + if extension == "" { + continue + } + f.Add(extension, unit.content, uint8(ast.JSDocParsingModeParseAll), false, false) + } } + } + + f.Fuzz(func(t *testing.T, extension string, sourceText string, jsdocParsingMode_ uint8, externalModuleIndicatorOptionsJSX bool, externalModuleIndicatorOptionsForce bool) { + jsdocParsingMode := ast.JSDocParsingMode(jsdocParsingMode_) - if scriptTarget < core.ScriptTargetNone || scriptTarget > core.ScriptTargetLatest { + if !extensions.Has(extension) { t.Skip() } @@ -138,8 +173,12 @@ func FuzzParser(f *testing.F) { FileName: fileName, Path: path, JSDocParsingMode: jsdocParsingMode, + ExternalModuleIndicatorOptions: ast.ExternalModuleIndicatorOptions{ + JSX: externalModuleIndicatorOptionsJSX, + Force: externalModuleIndicatorOptionsForce, + }, } - ParseSourceFile(opts, sourceText, core.GetScriptKindFromFileName(fileName)) + parser.ParseSourceFile(opts, sourceText, core.GetScriptKindFromFileName(fileName)) }) } diff --git a/internal/parser/reparser.go b/internal/parser/reparser.go index b1498ca68b..46e5598c50 100644 --- a/internal/parser/reparser.go +++ b/internal/parser/reparser.go @@ -134,6 +134,10 @@ func (p *Parser) reparseJSDocSignature(jsSignature *ast.Node, fun *ast.Node, jsD signature = p.factory.NewFunctionDeclaration(clonedModifiers, nil, p.factory.DeepCloneReparse(fun.Name()), nil, nil, nil, nil, nil) case ast.KindMethodDeclaration, ast.KindMethodSignature: signature = p.factory.NewMethodDeclaration(clonedModifiers, nil, p.factory.DeepCloneReparse(fun.Name()), nil, nil, nil, nil, nil, nil) + case ast.KindGetAccessor: + signature = p.factory.NewGetAccessorDeclaration(clonedModifiers, p.factory.DeepCloneReparse(fun.Name()), nil, nil, nil, nil, nil) + case ast.KindSetAccessor: + signature = p.factory.NewSetAccessorDeclaration(clonedModifiers, p.factory.DeepCloneReparse(fun.Name()), nil, nil, nil, nil, nil) case ast.KindConstructor: signature = p.factory.NewConstructorDeclaration(clonedModifiers, nil, nil, nil, nil, nil) case ast.KindJSDocCallbackTag: @@ -319,7 +323,7 @@ func (p *Parser) reparseHosted(tag *ast.Node, parent *ast.Node, jsDoc *ast.Node) } } case ast.KindReturnStatement, ast.KindParenthesizedExpression: - if tag.AsJSDocTypeTag().TypeExpression != nil { + if parent.Expression() != nil && tag.AsJSDocTypeTag().TypeExpression != nil { parent.AsMutable().SetExpression(p.makeNewCast( p.factory.DeepCloneReparse(tag.AsJSDocTypeTag().TypeExpression.Type()), p.factory.DeepCloneReparse(parent.Expression()), @@ -352,7 +356,7 @@ func (p *Parser) reparseHosted(tag *ast.Node, parent *ast.Node, jsDoc *ast.Node) } case ast.KindVariableDeclaration, ast.KindCommonJSExport, - ast.KindPropertyDeclaration, ast.KindPropertyAssignment, ast.KindShorthandPropertyAssignment: + ast.KindPropertyDeclaration, ast.KindPropertyAssignment: if parent.Initializer() != nil && tag.AsJSDocSatisfiesTag().TypeExpression != nil { parent.AsMutable().SetInitializer(p.makeNewCast( p.factory.DeepCloneReparse(tag.AsJSDocSatisfiesTag().TypeExpression.Type()), @@ -360,6 +364,15 @@ func (p *Parser) reparseHosted(tag *ast.Node, parent *ast.Node, jsDoc *ast.Node) false /*isAssertion*/)) p.finishMutatedNode(parent) } + case ast.KindShorthandPropertyAssignment: + shorthand := parent.AsShorthandPropertyAssignment() + if shorthand.ObjectAssignmentInitializer != nil && tag.AsJSDocSatisfiesTag().TypeExpression != nil { + shorthand.ObjectAssignmentInitializer = p.makeNewCast( + p.factory.DeepCloneReparse(tag.AsJSDocSatisfiesTag().TypeExpression.Type()), + p.factory.DeepCloneReparse(shorthand.ObjectAssignmentInitializer), + false /*isAssertion*/) + p.finishMutatedNode(parent) + } case ast.KindReturnStatement, ast.KindParenthesizedExpression, ast.KindExportAssignment, ast.KindJSExportAssignment: if parent.Expression() != nil && tag.AsJSDocSatisfiesTag().TypeExpression != nil { diff --git a/internal/parser/testdata/fuzz/FuzzParser/02b74efe61495c2a b/internal/parser/testdata/fuzz/FuzzParser/02b74efe61495c2a index 6bd4a7bcdb..dff3eb9368 100644 --- a/internal/parser/testdata/fuzz/FuzzParser/02b74efe61495c2a +++ b/internal/parser/testdata/fuzz/FuzzParser/02b74efe61495c2a @@ -1,5 +1,6 @@ go test fuzz v1 string(".ts") -string("/**@0\n * */0") -int32(99) -uint8(0) +string("") +byte('\x00') +bool(false) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/0ec0d5de7f0264d9 b/internal/parser/testdata/fuzz/FuzzParser/0ec0d5de7f0264d9 new file mode 100644 index 0000000000..3199b97092 --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/0ec0d5de7f0264d9 @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".js") +string("00000000000000000000000000000000000000000000000000000000000\"00000000000000000000000\n/**@type */return") +byte('\x00') +bool(true) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/3d59c16f3abc20e1 b/internal/parser/testdata/fuzz/FuzzParser/3d59c16f3abc20e1 new file mode 100644 index 0000000000..7025036252 --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/3d59c16f3abc20e1 @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".ts") +string("/**@typedef @type object00") +byte('\x00') +bool(false) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/4c0324cd37d955ff b/internal/parser/testdata/fuzz/FuzzParser/4c0324cd37d955ff new file mode 100644 index 0000000000..fb1b7aa812 --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/4c0324cd37d955ff @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".ts") +string("@00await") +byte('\x00') +bool(false) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/6944735deac09149 b/internal/parser/testdata/fuzz/FuzzParser/6944735deac09149 new file mode 100644 index 0000000000..c9caf93eba --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/6944735deac09149 @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".ts") +string("/*/") +byte('\x00') +bool(false) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/7d4c688e1df61349 b/internal/parser/testdata/fuzz/FuzzParser/7d4c688e1df61349 new file mode 100644 index 0000000000..8a57001ea2 --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/7d4c688e1df61349 @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".js") +string("0%{\n/**@satisfies */A") +byte('\x00') +bool(true) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/9ce2d994c65c7bfe b/internal/parser/testdata/fuzz/FuzzParser/9ce2d994c65c7bfe index 7ba034ca18..dff3eb9368 100644 --- a/internal/parser/testdata/fuzz/FuzzParser/9ce2d994c65c7bfe +++ b/internal/parser/testdata/fuzz/FuzzParser/9ce2d994c65c7bfe @@ -1,5 +1,6 @@ go test fuzz v1 string(".ts") -string("/") -int32(99) -uint8(1) +string("") +byte('\x00') +bool(false) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/c5dc3279768e5a11 b/internal/parser/testdata/fuzz/FuzzParser/c5dc3279768e5a11 new file mode 100644 index 0000000000..505e4516a4 --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/c5dc3279768e5a11 @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".ts") +string("/**") +byte('\x00') +bool(false) +bool(false) diff --git a/internal/parser/testdata/fuzz/FuzzParser/f48591d3b8f41eca b/internal/parser/testdata/fuzz/FuzzParser/f48591d3b8f41eca new file mode 100644 index 0000000000..14b3a7639d --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/f48591d3b8f41eca @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".js") +string("c(0000{\n/**@overload */get 0") +byte('\x00') +bool(false) +bool(true) diff --git a/internal/parser/testdata/fuzz/FuzzParser/f6dbdaa8568c9488 b/internal/parser/testdata/fuzz/FuzzParser/f6dbdaa8568c9488 new file mode 100644 index 0000000000..315b16bfac --- /dev/null +++ b/internal/parser/testdata/fuzz/FuzzParser/f6dbdaa8568c9488 @@ -0,0 +1,6 @@ +go test fuzz v1 +string(".ts") +string(")import A,await") +byte('\x00') +bool(false) +bool(false) diff --git a/internal/parser/utilities.go b/internal/parser/utilities.go index ee31ae4704..2e143a835b 100644 --- a/internal/parser/utilities.go +++ b/internal/parser/utilities.go @@ -41,7 +41,9 @@ func GetJSDocCommentRanges(f *ast.NodeFactory, commentRanges []ast.CommentRange, } // Keep if the comment starts with '/**' but not if it is '/**/' return slices.DeleteFunc(commentRanges, func(comment ast.CommentRange) bool { - return comment.End() > node.End() || text[comment.Pos()+1] != '*' || text[comment.Pos()+2] != '*' || text[comment.Pos()+3] == '/' + commentStart := comment.Pos() + commentLen := comment.End() - commentStart + return comment.End() > node.End() || commentLen < 4 || text[commentStart+1] != '*' || text[commentStart+2] != '*' || text[commentStart+3] == '/' }) }