SWIFT-871, SWIFT-1340 Reduce Evergreen matrix size, add TSan tasks (#674)

Author: kmahar

.evergreen/compile.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+# variables
+SWIFT_VERSION=${SWIFT_VERSION:-5.4.2}
+INSTALL_DIR="${PROJECT_DIRECTORY}/opt"
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+# enable swiftenv
+export SWIFTENV_ROOT="${INSTALL_DIR}/swiftenv"
+export PATH="${SWIFTENV_ROOT}/bin:$PATH"
+eval "$(swiftenv init -)"
+
+if [ "$OS" == "darwin" ]; then
+    # 5.1, 5.2 require an older version of Xcode/Command Line Tools
+    if [[ "$SWIFT_VERSION" == 5.1.* || "$SWIFT_VERSION" == 5.2.* ]]; then
+        sudo xcode-select -s /Applications/Xcode11.3.app
+    else
+        sudo xcode-select -s /Applications/Xcode12.app
+    fi
+fi
+
+# switch swift version and build
+swiftenv local $SWIFT_VERSION
+swift build

.evergreen/config.yml

@@ -160,6 +160,8 @@ functions:
           SWIFT_VERSION=${SWIFT_VERSION} \
           MONGODB_API_VERSION=${MONGODB_API_VERSION} \
           TEST_FILTER=${TEST_FILTER} \
+          SANITIZE=${SANITIZE} \
+          CHECK_LEAKS=${CHECK_LEAKS} \
             ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
     
   "upload test results":
@@ -356,6 +358,17 @@ functions:
           LINUX_SOURCEKIT_LIB_PATH=${PROJECT_DIRECTORY}/opt/swiftenv/versions/${SWIFT_VERSION}/usr/lib \
             ${PROJECT_DIRECTORY}/opt/swiftlint/.build/release/swiftlint --strict --quiet
   
+  "compile":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+
+          SWIFT_VERSION=${SWIFT_VERSION} \
+            ${PROJECT_DIRECTORY}/.evergreen/compile.sh
+  
   start-load-balancer:
     - command: shell.exec
       params:
@@ -416,6 +429,10 @@ post:
   - func: "cleanup"
 
 tasks:
+    - name: "compile"
+      commands:
+        - func: "compile"
+
     - name: "test-3.6-standalone"
       tags: ["3.6", "standalone"]
       commands:
@@ -1085,7 +1102,7 @@ axes:
       - id: "5.4"
         display_name: "Swift 5.4"
         variables:
-          SWIFT_VERSION: "5.4"
+          SWIFT_VERSION: "5.4.2"
       - id: "5.5-dev"
         display_name: "Swift 5.5-dev"
         variables:
@@ -1138,6 +1155,28 @@ axes:
           ORCHESTRATION_FILE: "versioned-api-testing.json"
           TEST_FILTER: "VersionedAPITests"
 
+  - id: sanitize
+    display_name: Sanitize
+    values:
+      - id: tsan
+        display_name: TSan
+        variables:
+          SANITIZE: "thread"
+      # ASan is unfortunately known to produce false positives in Swift, see
+      # https://forums.swift.org/t/test-for-memory-leaks-in-ci/36526/9 
+      # - id: asan
+      #   display_name: ASan
+      #   variables:
+      #     SANITIZE: "address"
+
+  - id: check-leaks
+    display_name: Check Leaks
+    values:
+      - id: leaks
+        display_name: Leaks
+        variables:
+          CHECK_LEAKS: leaks
+
 task_groups:
   - name: serverless_task_group
     setup_group_can_fail_task: true
@@ -1174,40 +1213,22 @@ task_groups:
       - ".serverless"
 
 buildvariants:
-# there are no 5.1 toolchains for Ubuntu 20.04. unfortunately as of now we cannot remove
-# entire buildvariants via rule from the main matrix so we need to split this up. see EVG-13092
-- matrix_name: "tests-5.1"
+
+- matrix_name: "min-version-compile"
   matrix_spec:
+  # Ubuntu 20.04 does not have Swift 5.1 toolchains.
     os-fully-featured:
-      - "ubuntu-18.04"
       - "macos-10.15"
+      - "ubuntu-18.04"
     swift-version: "5.1"
-    ssl-auth: "*"
-  display_name: "${swift-version} ${os-fully-featured} ${ssl-auth}"
+  display_name: "Compile ${swift-version} ${os-fully-featured}"
   tasks:
-     - ".latest !.load-balancer"
-     - ".5.0"
-     - ".4.4"
-     - ".4.2"
-     - ".4.0"
-     - ".3.6"
-  rules:
-  # MongoDB 3.6 does not have Ubuntu 18.04-specific Linux releases, so we just use
-  # generic Linux there. This do not link to OpenSSL, so we have to skip SSL tests.
-  - if:
-      os-fully-featured:
-        - "ubuntu-18.04"
-      swift-version: "*"
-      ssl-auth: "ssl-auth"
-    then:
-      remove_tasks:
-      - ".3.6"
+    - "compile"
 
 - matrix_name: "tests-all"
   matrix_spec:
     os-fully-featured: "*"
     swift-version:
-      - "5.2"
       - "5.3"
       - "5.4"
       - "5.5-dev"
@@ -1245,23 +1266,10 @@ buildvariants:
         - ".4.0"
         - ".4.2"
 
-# there are no 5.1 toolchains for Ubuntu 20.04. unfortunately as of now we cannot remove
-# entire buildvariants via rule from the main matrix so we need to split this up. see EVG-13092
-- matrix_name: "atlas-connect-5.1"
-  matrix_spec:
-    os-fully-featured:
-      - "macos-10.15"
-      - "ubuntu-18.04"
-    swift-version: "5.1"
-  display_name: "Atlas Connectivity ${swift-version} ${os-fully-featured}"
-  tasks:
-    - ".atlas-connect"
-
 - matrix_name: "atlas-connect"
   matrix_spec:
     os-fully-featured: "*"
     swift-version:
-      - "5.2"
       - "5.3"
       - "5.4"
       - "5.5-dev"
@@ -1276,7 +1284,6 @@ buildvariants:
       - "ubuntu-20.04"
     ssl-auth: "*"
     swift-version:
-      - "5.2"
       - "5.3"
       - "5.4"
       - "5.5-dev"
@@ -1303,8 +1310,7 @@ buildvariants:
       - latest
       - 5.0
       - 4.4
-    swift-version:
-      - 5.2
+    swift-version: "5.4"
   display_name: "OCSP ${swift-version} ${os-fully-featured} ${versions}"
   batchtime: 20160 # 14 days
   tasks:
@@ -1321,10 +1327,8 @@ buildvariants:
 
 - matrix_name: "ocsp-macos"
   matrix_spec:
-    os-fully-featured:
-      - macos-10.15
-    swift-version:
-      - 5.2
+    os-fully-featured: "macos-10.15"
+    swift-version: "5.4"
     versions:
       - latest
       - 5.0
@@ -1338,7 +1342,7 @@ buildvariants:
 - matrix_name: "versioned-api-tests"
   matrix_spec:
     os-fully-featured: ubuntu-18.04
-    swift-version: "5.3"
+    swift-version: "5.4"
     versionedAPI: "*"
   display_name: "Versioned API ${versionedAPI} ${swift-version} ${os-fully-featured}"
   batchtime: 10080  # 7 days
@@ -1350,7 +1354,7 @@ buildvariants:
   display_name: "Format and Lint"
   matrix_spec:
     os-fully-featured: "ubuntu-18.04"
-    swift-version: "5.3"
+    swift-version: "5.4"
   tasks:
     - name: "check-format"
     - name: "check-lint"
@@ -1359,6 +1363,26 @@ buildvariants:
   display_name: "Check Sourcery"
   matrix_spec:
     os-fully-featured: "macos-10.15"
-    swift-version: "5.2"
+    swift-version: "5.4"
   tasks:
     - name: "check-sourcery"
+
+- matrix_name: "sanitize"
+  display_name: "${sanitize} ${os-fully-featured} ${ssl-auth}"
+  matrix_spec:
+    os-fully-featured: "ubuntu-18.04"
+    swift-version: "5.4"
+    ssl-auth: "*"
+    sanitize: "tsan"
+  tasks:
+     - ".5.0"
+
+- matrix_name: "leaks"
+  display_name: "Leak Checker ${os-fully-featured} ${ssl-auth}"
+  matrix_spec:
+    os-fully-featured: "macos-10.15"
+    swift-version: "5.4"
+    ssl-auth: "*"
+    check-leaks: "leaks"
+  tasks:
+     - ".5.0"

.evergreen/install-tools.sh

@@ -39,7 +39,7 @@ swiftenv local $SWIFT_VERSION
 
 if [ $1 == "swiftlint" ]
 then
-    build_from_gh swiftlint https://github.com/realm/SwiftLint "0.41.0"
+    build_from_gh swiftlint https://github.com/realm/SwiftLint "0.43.1"
 elif [ $1 == "swiftformat" ]
 then
     build_from_gh swiftformat https://github.com/nicklockwood/SwiftFormat "0.47.13"

.evergreen/run-tests.sh

@@ -15,6 +15,7 @@ RAW_TEST_RESULTS="${PROJECT_DIRECTORY}/rawTestResults"
 XML_TEST_RESULTS="${PROJECT_DIRECTORY}/testResults.xml"
 INSTALL_DEPS=${INSTALL_DEPS:-"false"}
 TEST_FILTER=${TEST_FILTER:-"NO_FILTER"}
+SANITIZE=${SANITIZE:-"false"}
 
 # ssl setup
 SSL=${SSL:-nossl}
@@ -47,8 +48,13 @@ fi
 # switch swift version, and run tests
 swiftenv local $SWIFT_VERSION
 
+SANITIZE_STATEMENT=""
+if [ "$SANITIZE" != "false" ]; then
+    SANITIZE_STATEMENT="--sanitize ${SANITIZE}"
+fi
+
 # build the driver
-swift build
+swift build $SANITIZE_STATEMENT
 
 # test the driver
 set +o errexit # even if tests fail we want to parse the results, so disable errexit
@@ -62,7 +68,7 @@ fi
 
 MONGODB_TOPOLOGY=${TOPOLOGY} MONGODB_URI=$MONGODB_URI SINGLE_MONGOS_LB_URI=$SINGLE_MONGOS_LB_URI \
 MULTI_MONGOS_LB_URI=$MULTI_MONGOS_LB_URI MONGODB_API_VERSION=$MONGODB_API_VERSION \
-swift test $FILTER_STATEMENT 2>&1 | tee ${RAW_TEST_RESULTS}
+swift test $FILTER_STATEMENT $SANITIZE_STATEMENT 2>&1 | tee ${RAW_TEST_RESULTS}
 
 # save tests exit code
 EXIT_CODE=$?

.swift-version

@@ -1 +1 @@
-5.2
+5.4.2

Package.swift

@@ -20,7 +20,7 @@ let package = Package(
         .target(name: "AtlasConnectivity", dependencies: ["MongoSwiftSync"]),
         .target(name: "TestsCommon", dependencies: ["MongoSwift", "Nimble"]),
         .testTarget(name: "BSONTests", dependencies: ["MongoSwift", "TestsCommon", "Nimble", "CLibMongoC"]),
-        .testTarget(name: "MongoSwiftTests", dependencies: ["MongoSwift", "TestsCommon", "Nimble", "NIO"]),
+        .testTarget(name: "MongoSwiftTests", dependencies: ["MongoSwift", "TestsCommon", "Nimble", "NIO", "NIOConcurrencyHelpers"]),
         .testTarget(name: "MongoSwiftSyncTests", dependencies: ["MongoSwiftSync", "TestsCommon", "Nimble", "MongoSwift"]),
         .target(
             name: "CLibMongoC",

Tests/LinuxMain.swift

@@ -112,6 +112,12 @@ extension EventLoopBoundMongoClientTests {
     ]
 }
 
+extension LeakCheckTests {
+    static var allTests = [
+        ("testLeaks", testLeaks),
+    ]
+}
+
 extension LoadBalancerTests {
     static var allTests = [
         ("testLoadBalancers", testLoadBalancers),
@@ -406,6 +412,7 @@ XCTMain([
     testCase(CrudTests.allTests),
     testCase(DNSSeedlistTests.allTests),
     testCase(EventLoopBoundMongoClientTests.allTests),
+    testCase(LeakCheckTests.allTests),
     testCase(LoadBalancerTests.allTests),
     testCase(MongoClientTests.allTests),
     testCase(MongoCollectionTests.allTests),

Tests/MongoSwiftSyncTests/SyncMongoClientTests.swift

@@ -1,6 +1,7 @@
 import Foundation
 import MongoSwiftSync
 import Nimble
+import NIOConcurrencyHelpers
 import TestsCommon
 import XCTest
 
@@ -272,21 +273,22 @@ final class SyncMongoClientTests: MongoSwiftTestCase {
     func testAPMCallbacks() throws {
         let client = try MongoClient.makeTestClient()
 
-        var commandEvents: [CommandEvent] = []
-        client.addCommandEventHandler { event in
-            commandEvents.append(event)
+        let commandEventCount = NIOAtomic<Int>.makeAtomic(value: 0)
+        let sdamEventCount = NIOAtomic<Int>.makeAtomic(value: 0)
+
+        client.addCommandEventHandler { _ in
+            _ = commandEventCount.add(1)
         }
 
-        var sdamEvents: [SDAMEvent] = []
-        client.addSDAMEventHandler { event in
-            sdamEvents.append(event)
+        client.addSDAMEventHandler { _ in
+            _ = sdamEventCount.add(1)
         }
 
         // don't care if command fails, just testing that the events were emitted
         _ = try? client.listDatabases()
 
-        expect(commandEvents).toEventually(haveCount(2)) // wait for started and succeeded / failed
-        expect(sdamEvents.isEmpty).toEventually(beFalse())
+        expect(commandEventCount.load()).toEventually(equal(2)) // wait for started and succeeded / failed
+        expect(sdamEventCount.load()).toEventuallyNot(equal(0))
     }
 
     func testCertificateVerificationOptions() throws {

Tests/MongoSwiftTests/ChangeStreamTests.swift

@@ -2,6 +2,7 @@ import Foundation
 import MongoSwift
 import Nimble
 import NIO
+import NIOConcurrencyHelpers
 import TestsCommon
 
 final class ChangeStreamTests: MongoSwiftTestCase {
@@ -131,8 +132,8 @@ final class ChangeStreamTests: MongoSwiftTestCase {
     }
 
     func testChangeStreamForEach() throws {
-        var count = 0
-        let increment: (ChangeStreamEvent<BSONDocument>) -> Void = { _ in count += 1 }
+        let count = NIOAtomic<Int>.makeAtomic(value: 0)
+        let increment: (ChangeStreamEvent<BSONDocument>) -> Void = { _ in count.add(1) }
 
         try self.withTestClient { client in
             let unmetRequirement = try client.getUnmetRequirement(.changeStreamOnCollectionSupport)
@@ -151,10 +152,10 @@ final class ChangeStreamTests: MongoSwiftTestCase {
             let future = stream.forEach(increment)
 
             _ = try coll.insertOne(["x": 1]).wait()
-            expect(count).toEventually(equal(1), timeout: 10)
+            expect(count.load()).toEventually(equal(1), timeout: 10)
 
             _ = try coll.insertOne(["x": 2]).wait()
-            expect(count).toEventually(equal(2), timeout: 10)
+            expect(count.load()).toEventually(equal(2), timeout: 10)
 
             try stream.kill().wait()
             expect(try future.wait()).toNot(throwError())