diff --git a/pkgs/test_analyzer_plugin/README.md b/pkgs/test_analyzer_plugin/README.md new file mode 100644 index 000000000..e0540b9a2 --- /dev/null +++ b/pkgs/test_analyzer_plugin/README.md @@ -0,0 +1,21 @@ +# test_analyzer_plugin + +This package is an analyzer plugin that provides additional static analysis for +usage of the test package. + +This analyzer plugin provides the following additional analysis: + +* Report a warning when a `test` or a `group` is declared inside a `test` + declaration. This can _sometimes_ be detected at runtime. This warning is + reported statically. + +* Report a warning when a non-nullable value is matched against `isNotNull` or + `isNull`. + +* Report a warning when an `expect` expectation of `true`, `false`, `isTrue`, + or `isFalse` is paired with a `.isEmpty` or `.isNotEmpty` property access on + an actual value. Instead, the `isEmpty` or `isNotEmpty` Matcher should be + used. + +* Offer a quick fix in the IDE for the above warning, which moves the violating + `test` or `group` declaration below the containing `test` declaration. \ No newline at end of file diff --git a/pkgs/test_analyzer_plugin/lib/main.dart b/pkgs/test_analyzer_plugin/lib/main.dart new file mode 100644 index 000000000..343e2c5f8 --- /dev/null +++ b/pkgs/test_analyzer_plugin/lib/main.dart @@ -0,0 +1,39 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:analysis_server_plugin/plugin.dart'; +import 'package:analysis_server_plugin/registry.dart'; + +import 'src/fixes.dart'; +import 'src/rules/non_nullable_is_not_null_rule.dart'; +import 'src/rules/test_in_test_rule.dart'; +import 'src/rules/use_is_empty_matcher.dart'; + +final plugin = TestPackagePlugin(); + +class TestPackagePlugin extends Plugin { + String get name => 'Test package plugin'; + + @override + void register(PluginRegistry registry) { + registry.registerWarningRule(TestInTestRule()); + registry.registerFixForRule( + TestInTestRule.code, + MoveBelowEnclosingTestCall.new, + ); + + registry.registerWarningRule(NonNullableIsNotNullRule()); + registry.registerWarningRule(UseIsEmptyMatcherRule()); + // Should we register a fix for this rule? The only automatic fix I can + // think of would be to delete the entire statement: + // `expect(a, isNotNull);` or `expect(a, isNull);`. + + // TODO(srawlins): More rules to catch: + // * `expect(7, contains(6))` - should only use `hasLength` with `Iterable` + // or `String`. + // * `expect(7, hasLength(3))` - should only use `hasLength` with `Iterable` + // or `String`. + // * `expect([].contains(7), isFalse)` - should use `contains` matcher. + } +} diff --git a/pkgs/test_analyzer_plugin/lib/src/fixes.dart b/pkgs/test_analyzer_plugin/lib/src/fixes.dart new file mode 100644 index 000000000..f2d36f8a1 --- /dev/null +++ b/pkgs/test_analyzer_plugin/lib/src/fixes.dart @@ -0,0 +1,61 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:analysis_server_plugin/edit/dart/correction_producer.dart'; +import 'package:analysis_server_plugin/edit/dart/dart_fix_kind_priority.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer_plugin/utilities/change_builder/change_builder_core.dart'; +import 'package:analyzer_plugin/utilities/fixes/fixes.dart'; +import 'package:analyzer_plugin/utilities/range_factory.dart'; + +import 'utilities.dart'; + +class MoveBelowEnclosingTestCall extends ResolvedCorrectionProducer { + static const _moveBelowEnclosingTestCallKind = FixKind( + 'dart.fix.moveBelowEnclosingTestCall', + DartFixKindPriority.standard, + "Move below the enclosing 'test' call", + ); + + MoveBelowEnclosingTestCall({required super.context}); + + @override + CorrectionApplicability get applicability => + // This fix may break code by moving references to variables away from the + // scope in which they are declared. + CorrectionApplicability.singleLocation; + + @override + FixKind get fixKind => _moveBelowEnclosingTestCallKind; + + @override + Future compute(ChangeBuilder builder) async { + var methodCall = node; + if (methodCall is! MethodInvocation) return; + AstNode? enclosingTestCall = findEnclosingTestCall(methodCall); + if (enclosingTestCall == null) return; + + if (enclosingTestCall.parent is ExpressionStatement) { + // Move the 'test' call to below the outer 'test' call _statement_. + enclosingTestCall = enclosingTestCall.parent!; + } + + if (methodCall.parent is ExpressionStatement) { + // Move the whole statement (don't leave the semicolon dangling). + methodCall = methodCall.parent!; + } + + await builder.addDartFileEdit(file, (builder) { + var indent = utils.getLinePrefix(enclosingTestCall!.offset); + var source = utils.getRangeText(range.node(methodCall)); + + // Move the source for `methodCall` wholsale to be just after `enclosingTestCall`. + builder.addDeletion(range.deletionRange(methodCall)); + builder.addSimpleInsertion( + enclosingTestCall.end, + '$defaultEol$defaultEol$indent$source', + ); + }); + } +} diff --git a/pkgs/test_analyzer_plugin/lib/src/rules/non_nullable_is_not_null_rule.dart b/pkgs/test_analyzer_plugin/lib/src/rules/non_nullable_is_not_null_rule.dart new file mode 100644 index 000000000..4c826ae1e --- /dev/null +++ b/pkgs/test_analyzer_plugin/lib/src/rules/non_nullable_is_not_null_rule.dart @@ -0,0 +1,88 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:analyzer/analysis_rule/analysis_rule.dart'; +import 'package:analyzer/analysis_rule/rule_context.dart'; +import 'package:analyzer/analysis_rule/rule_visitor_registry.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:analyzer/dart/element/type_system.dart'; +import 'package:analyzer/error/error.dart'; + +import '../utilities.dart'; + +class NonNullableIsNotNullRule extends MultiAnalysisRule { + static const LintCode nonNullableIsNotNullCode = LintCode( + 'non_nullable_is_not_null', + 'Do not check whether a non-nullable value isNotNull', + correctionMessage: 'Try changing the expectation, or removing it', + ); + + static const LintCode nonNullableIsNullCode = LintCode( + 'non_nullable_is_null', + 'Do not check whether a non-nullable value isNull', + correctionMessage: 'Try changing the expectation, or removing it', + ); + + NonNullableIsNotNullRule() + : super( + name: 'non_nullable_is_not_null', + description: + "Non-nullable values will always pass an 'isNotNull' " + "expectation and never pass an 'isNull' expectation.", + ); + + @override + List get diagnosticCodes => [ + nonNullableIsNotNullCode, + nonNullableIsNullCode, + ]; + + @override + void registerNodeProcessors( + RuleVisitorRegistry registry, + RuleContext context, + ) { + var visitor = _Visitor(this, context.typeSystem); + registry.addMethodInvocation(this, visitor); + } +} + +class _Visitor extends SimpleAstVisitor { + final MultiAnalysisRule rule; + + final TypeSystem typeSystem; + + _Visitor(this.rule, this.typeSystem); + + @override + void visitMethodInvocation(MethodInvocation node) { + if (!node.methodName.isExpect) { + return; + } + + if (node.argumentList.arguments case [ + var actual, + SimpleIdentifier matcher, + ]) { + var actualType = actual.staticType; + if (actualType == null) return; + if (typeSystem.isNonNullable(actualType)) { + if (matcher.isNotNull) { + // The actual value will always match this matcher. + rule.reportAtNode( + matcher, + diagnosticCode: NonNullableIsNotNullRule.nonNullableIsNotNullCode, + ); + } else if (matcher.isNull) { + // The actual value will never match this matcher. + rule.reportAtNode( + matcher, + diagnosticCode: NonNullableIsNotNullRule.nonNullableIsNullCode, + ); + } + } + } + } +} diff --git a/pkgs/test_analyzer_plugin/lib/src/rules/test_in_test_rule.dart b/pkgs/test_analyzer_plugin/lib/src/rules/test_in_test_rule.dart new file mode 100644 index 000000000..05f639daa --- /dev/null +++ b/pkgs/test_analyzer_plugin/lib/src/rules/test_in_test_rule.dart @@ -0,0 +1,57 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:analyzer/analysis_rule/analysis_rule.dart'; +import 'package:analyzer/analysis_rule/rule_context.dart'; +import 'package:analyzer/analysis_rule/rule_visitor_registry.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:analyzer/error/error.dart'; + +import '../utilities.dart'; + +class TestInTestRule extends AnalysisRule { + static const LintCode code = LintCode( + 'test_in_test', + "Do not declare a 'test' or a 'group' inside a 'test'", + correctionMessage: "Try moving 'test' or 'group' outside of 'test'", + ); + + TestInTestRule() + : super( + name: 'test_in_test', + description: + 'Tests and groups declared inside of a test are not properly ' + 'registered in the test framework.', + ); + + @override + LintCode get diagnosticCode => code; + + @override + void registerNodeProcessors( + RuleVisitorRegistry registry, + RuleContext context, + ) { + var visitor = _Visitor(this); + registry.addMethodInvocation(this, visitor); + } +} + +class _Visitor extends SimpleAstVisitor { + final AnalysisRule rule; + + _Visitor(this.rule); + + @override + void visitMethodInvocation(MethodInvocation node) { + if (!node.methodName.isTest && !node.methodName.isGroup) { + return; + } + var enclosingTestCall = findEnclosingTestCall(node); + if (enclosingTestCall != null) { + rule.reportAtNode(node); + } + } +} diff --git a/pkgs/test_analyzer_plugin/lib/src/rules/use_is_empty_matcher.dart b/pkgs/test_analyzer_plugin/lib/src/rules/use_is_empty_matcher.dart new file mode 100644 index 000000000..99035e53c --- /dev/null +++ b/pkgs/test_analyzer_plugin/lib/src/rules/use_is_empty_matcher.dart @@ -0,0 +1,111 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:analyzer/analysis_rule/analysis_rule.dart'; +import 'package:analyzer/analysis_rule/rule_context.dart'; +import 'package:analyzer/analysis_rule/rule_visitor_registry.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:analyzer/dart/element/type_system.dart'; +import 'package:analyzer/error/error.dart'; + +import '../utilities.dart'; + +class UseIsEmptyMatcherRule extends MultiAnalysisRule { + static const LintCode useIsEmptyMatcherCode = LintCode( + 'use_is_empty_matcher', + "Use the 'isEmpty' matcher.", + ); + + static const LintCode useIsNotEmptyMatcherCode = LintCode( + 'use_is_not_empty_matcher', + "Use the 'isNotEmpty' matcher.", + ); + + UseIsEmptyMatcherRule() + : super( + name: 'use_is_empty_matcher', + description: "Use the built-in 'isEmpty' and 'isNotEmpty' matchers.", + ); + + @override + List get diagnosticCodes => [ + useIsEmptyMatcherCode, + useIsNotEmptyMatcherCode, + ]; + + @override + void registerNodeProcessors( + RuleVisitorRegistry registry, + RuleContext context, + ) { + var visitor = _Visitor(this, context.typeSystem); + registry.addMethodInvocation(this, visitor); + } +} + +class _Visitor extends SimpleAstVisitor { + final MultiAnalysisRule rule; + + final TypeSystem typeSystem; + + _Visitor(this.rule, this.typeSystem); + + @override + void visitMethodInvocation(MethodInvocation node) { + if (!node.methodName.isExpect) return; + + var arguments = node.argumentList.arguments; + if (arguments.isEmpty || arguments.length > 2) return; + var [actual, matcher] = arguments; + + bool actualIsIsEmpty; + if (actual is PrefixedIdentifier) { + if (actual.identifier.name == 'isEmpty') { + actualIsIsEmpty = true; + } else if (actual.identifier.name == 'isNotEmpty') { + actualIsIsEmpty = false; + } else { + return; + } + } else if (actual is PropertyAccess) { + if (actual.propertyName.name == 'isEmpty') { + actualIsIsEmpty = true; + } else if (actual.propertyName.name == 'isNotEmpty') { + actualIsIsEmpty = false; + } else { + return; + } + } else { + return; + } + + bool matcherValue; + if (matcher is BooleanLiteral) { + matcherValue = matcher.value; + } else if (matcher is SimpleIdentifier && matcher.isIsFalse) { + matcherValue = false; + } else if (matcher is SimpleIdentifier && matcher.isIsTrue) { + matcherValue = true; + } else { + return; + } + + if (actualIsIsEmpty == matcherValue) { + // Either `expect(a.isEmpty, isTrue|true)` or + // `expect(a.isNotEmpty, isFalse|false)`. + rule.reportAtNode( + matcher, + diagnosticCode: UseIsEmptyMatcherRule.useIsEmptyMatcherCode, + ); + } else { + // Either `expect(a.isEmpty, isFalse|false)` or + // `expect(a.isNotEmpty, isTrue|true)`. + rule.reportAtNode( + matcher, + diagnosticCode: UseIsEmptyMatcherRule.useIsNotEmptyMatcherCode, + ); + } + } +} diff --git a/pkgs/test_analyzer_plugin/lib/src/utilities.dart b/pkgs/test_analyzer_plugin/lib/src/utilities.dart new file mode 100644 index 000000000..bb110d294 --- /dev/null +++ b/pkgs/test_analyzer_plugin/lib/src/utilities.dart @@ -0,0 +1,82 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:analyzer/dart/ast/ast.dart'; + +/// Finds an enclosing call to the 'test' function, if there is one. +MethodInvocation? findEnclosingTestCall(MethodInvocation node) { + var ancestor = node.parent?.thisOrAncestorOfType(); + while (ancestor != null) { + if (ancestor.methodName.isTest) { + return ancestor; + } + ancestor = ancestor.parent?.thisOrAncestorOfType(); + } + return null; +} + +extension SimpleIdentifierExtension on SimpleIdentifier { + /// Whether this identifier represents the 'test' function from the + /// 'test_core' package. + bool get isTest { + final element = this.element; + if (element == null) return false; + if (element.name != 'test') return false; + return element.library?.uri.path.startsWith('test_core/') ?? false; + } + + /// Whether this identifier represents the 'group' function from the + /// 'test_core' package. + bool get isGroup { + final element = this.element; + if (element == null) return false; + if (element.name != 'group') return false; + return element.library?.uri.path.startsWith('test_core/') ?? false; + } + + /// Whether this identifier represents the 'expect' function from the + /// 'matcher' package. + bool get isExpect { + final element = this.element; + if (element == null) return false; + if (element.name != 'expect') return false; + return element.library?.uri.path.startsWith('matcher/') ?? false; + } + + /// Whether this identifier represents the 'isFalse' matcher from the + /// 'matcher' package. + bool get isIsFalse { + final element = this.element; + if (element == null) return false; + if (element.name != 'isFalse') return false; + return element.library?.uri.path.startsWith('matcher/') ?? false; + } + + /// Whether this identifier represents the 'isTrue' matcher from the + /// 'matcher' package. + bool get isIsTrue { + final element = this.element; + if (element == null) return false; + if (element.name != 'isTrue') return false; + return element.library?.uri.path.startsWith('matcher/') ?? false; + } + + /// Whether this identifier represents the 'isNotNull' constant from the + /// 'matcher' package. + bool get isNotNull { + final element = this.element; + if (element == null) return false; + if (element.name != 'isNotNull') return false; + return element.library?.uri.path.startsWith('matcher/') ?? false; + } + + /// Whether this identifier represents the 'isNull' constant from the + /// 'matcher' package. + bool get isNull { + final element = this.element; + if (element == null) return false; + if (element.name != 'isNull') return false; + return element.library?.uri.path.startsWith('matcher/') ?? false; + } +} diff --git a/pkgs/test_analyzer_plugin/pubspec.yaml b/pkgs/test_analyzer_plugin/pubspec.yaml new file mode 100644 index 000000000..a9a276c4d --- /dev/null +++ b/pkgs/test_analyzer_plugin/pubspec.yaml @@ -0,0 +1,16 @@ +name: test_analyzer_plugin +description: An analyzer plugin to report improper usage of the test package. +version: 0.1.0 + +environment: + sdk: ^3.10.0 + +dependencies: + analysis_server_plugin: ^0.3.0 + analyzer: ^8.4.0 + analyzer_plugin: ^0.13.5 + +dev_dependencies: + analyzer_testing: ^0.1.2 + test: any + test_reflective_loader: any diff --git a/pkgs/test_analyzer_plugin/test/rules/non_nullable_is_not_null_test.dart b/pkgs/test_analyzer_plugin/test/rules/non_nullable_is_not_null_test.dart new file mode 100644 index 000000000..abac2b0cf --- /dev/null +++ b/pkgs/test_analyzer_plugin/test/rules/non_nullable_is_not_null_test.dart @@ -0,0 +1,84 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// ignore_for_file: non_constant_identifier_names + +import 'package:analyzer/src/lint/registry.dart'; +import 'package:analyzer/utilities/package_config_file_builder.dart'; +import 'package:analyzer_testing/analysis_rule/analysis_rule.dart'; +import 'package:test_analyzer_plugin/src/rules/non_nullable_is_not_null_rule.dart'; +import 'package:test_reflective_loader/test_reflective_loader.dart'; + +void main() { + defineReflectiveSuite(() { + defineReflectiveTests(NonNullableIsNotNullTest); + }); +} + +@reflectiveTest +class NonNullableIsNotNullTest extends AnalysisRuleTest { + @override + void setUp() { + Registry.ruleRegistry.registerLintRule(NonNullableIsNotNullRule()); + + super.setUp(); + + var matcherPath = '/packages/matcher'; + newFile('$matcherPath/lib/matcher.dart', ''' +void expect(dynamic actual, dynamic matcher) {} + +const isNotNull = 0; +const isNull = 0; +'''); + writeTestPackageConfig( + PackageConfigFileBuilder() + ..add(name: 'matcher', rootPath: convertPath(matcherPath)), + ); + } + + @override + String get analysisRule => 'non_nullable_is_not_null'; + + void test_nullableValue_isNotNullMatcher() async { + await assertNoDiagnostics(r''' +import 'package:matcher/matcher.dart'; +void f(String? p) { + expect(p, isNotNull); +} +'''); + } + + void test_nonNullableValue_isNotNullMatcher() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(123, isNotNull); +} +''', + [lint(64, 9)], + ); + } + + void test_nullableValue_isNullMatcher() async { + await assertNoDiagnostics(r''' +import 'package:matcher/matcher.dart'; +void f(String? p) { + expect(p, isNull); +} +'''); + } + + void test_nonNullableValue_isNullMatcher() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(123, isNull); +} +''', + [lint(64, 6, name: 'non_nullable_is_null')], + ); + } +} diff --git a/pkgs/test_analyzer_plugin/test/rules/use_is_empty_matcher_test.dart b/pkgs/test_analyzer_plugin/test/rules/use_is_empty_matcher_test.dart new file mode 100644 index 000000000..68d898919 --- /dev/null +++ b/pkgs/test_analyzer_plugin/test/rules/use_is_empty_matcher_test.dart @@ -0,0 +1,152 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// ignore_for_file: non_constant_identifier_names + +import 'package:analyzer/src/lint/registry.dart'; +import 'package:analyzer/utilities/package_config_file_builder.dart'; +import 'package:analyzer_testing/analysis_rule/analysis_rule.dart'; +import 'package:test_analyzer_plugin/src/rules/use_is_empty_matcher.dart'; +import 'package:test_reflective_loader/test_reflective_loader.dart'; + +void main() { + defineReflectiveSuite(() { + defineReflectiveTests(UseIsEmptyMatcherTest); + }); +} + +@reflectiveTest +class UseIsEmptyMatcherTest extends AnalysisRuleTest { + @override + String get analysisRule => 'use_is_empty_matcher'; + + @override + void setUp() { + Registry.ruleRegistry.registerLintRule(UseIsEmptyMatcherRule()); + + super.setUp(); + + var matcherPath = '/packages/matcher'; + newFile('$matcherPath/lib/matcher.dart', ''' +void expect(dynamic actual, dynamic matcher) {} + +const isNotNull = 0; +const isNull = 0; + +const isEmpty = 0; +const isFalse = 0; +const isNotEmpty = 0; +const isTrue = 0; +'''); + writeTestPackageConfig( + PackageConfigFileBuilder() + ..add(name: 'matcher', rootPath: convertPath(matcherPath)), + ); + } + + void test_isEmpty_false() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isEmpty, false); +} +''', + [lint(71, 5, name: 'use_is_not_empty_matcher')], + ); + } + + void test_isEmpty_isFalse() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isEmpty, isFalse); +} +''', + [lint(71, 7, name: 'use_is_not_empty_matcher')], + ); + } + + void test_isEmpty_isTrue() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isEmpty, isTrue); +} +''', + [lint(71, 6, messageContains: 'isEmpty')], + ); + } + + void test_isEmpty_true() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isEmpty, true); +} +''', + [lint(71, 4, messageContains: 'isEmpty')], + ); + } + + void test_isEmptyMatcher() async { + await assertNoDiagnostics(r''' +import 'package:matcher/matcher.dart'; +void f() { + expect('', isEmpty); +} +'''); + } + + void test_isNotEmpty_false() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isNotEmpty, false); +} +''', + [lint(74, 5, messageContains: 'isEmpty')], + ); + } + + void test_isNotEmpty_isFalse() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isNotEmpty, isFalse); +} +''', + [lint(74, 7, messageContains: 'isEmpty')], + ); + } + + void test_isNotEmpty_isTrue() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isNotEmpty, isTrue); +} +''', + [lint(74, 6, name: 'use_is_not_empty_matcher')], + ); + } + + void test_isNotEmpty_true() async { + await assertDiagnostics( + r''' +import 'package:matcher/matcher.dart'; +void f() { + expect(''.isNotEmpty, true); +} +''', + [lint(74, 4, name: 'use_is_not_empty_matcher')], + ); + } +}