Refactor mpp/native build, introduce "concurrent" source set, test launcher (#2074)

New source sets:
* "concurrent" source set is shared between "jvm" and "native"
* "native" source set is subdivided into "nativeDarwin" (Apple) and "nativeOther" (Linux, etc)

Native tests are launched in two variants:
* A default "test" task runs tests with memory leak checker from "mainNoExit" entry point.
* A special "backgroundTest" task runs tests in a background worker from "mainBackground" entry point.

Other build improvement:
* Modernize old-style IDEA-active hacks to kts helper.
* Extract versions of JS test runner dependencies.
* Remove redundant google repo reference from android tests.

Author: elizarov

build.gradle

@@ -89,8 +89,8 @@ buildscript {
 
 import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType
 
-// Hierarchical project structures are not fully supported in 1.3.7x MPP, enable conditionally for 1.4.x
-if (VersionNumber.parse(kotlin_version) > VersionNumber.parse("1.3.79")) {
+// todo:KLUDGE: Hierarchical project structures are not fully supported in IDEA, enable only for a regular built
+if (!Idea.active) {
     ext.set("kotlin.mpp.enableGranularSourceSetsMetadata", "true")
 }
 

gradle.properties

@@ -36,10 +36,12 @@ kotlin.js.compiler=both
 gradle_node_version=1.2.0
 node_version=8.9.3
 npm_version=5.7.1
-mocha_version=4.1.0
+mocha_version=6.2.2
 mocha_headless_chrome_version=1.8.2
-mocha_teamcity_reporter_version=2.2.2
-source_map_support_version=0.5.3
+mocha_teamcity_reporter_version=3.0.0
+source_map_support_version=0.5.16
+jsdom_version=15.2.1
+jsdom_global_version=3.0.2
 
 # Settings
 kotlin.incremental.multiplatform=true
@@ -56,7 +58,6 @@ org.gradle.jvmargs=-Xmx2g
 # https://github.com/gradle/gradle/issues/11412
 systemProp.org.gradle.internal.publish.checksums.insecure=true
 
-# This is commented out, and the property is set conditionally in build.gradle, because 1.3.71 doesn't work with it.
-# Once this property is set by default in new versions or 1.3.71 is dropped, either uncomment or remove this line.
+# todo:KLUDGE: This is commented out, and the property is set conditionally in build.gradle, because IDEA doesn't work with it.
 #kotlin.mpp.enableGranularSourceSetsMetadata=true
 kotlin.mpp.enableCompatibilityMetadataVariant=true

gradle/compile-native-multiplatform.gradle

@@ -13,36 +13,24 @@ kotlin {
     }
 
     targets {
-        if (project.ext.ideaActive) {
-            fromPreset(project.ext.ideaPreset, 'native')
-        } else {
-            addTarget(presets.linuxX64)
-            addTarget(presets.iosArm64)
-            addTarget(presets.iosArm32)
-            addTarget(presets.iosX64)
-            addTarget(presets.macosX64)
-            addTarget(presets.mingwX64)
-            addTarget(presets.tvosArm64)
-            addTarget(presets.tvosX64)
-            addTarget(presets.watchosArm32)
-            addTarget(presets.watchosArm64)
-            addTarget(presets.watchosX86)
-        }
+        addTarget(presets.linuxX64)
+        addTarget(presets.iosArm64)
+        addTarget(presets.iosArm32)
+        addTarget(presets.iosX64)
+        addTarget(presets.macosX64)
+        addTarget(presets.mingwX64)
+        addTarget(presets.tvosArm64)
+        addTarget(presets.tvosX64)
+        addTarget(presets.watchosArm32)
+        addTarget(presets.watchosArm64)
+        addTarget(presets.watchosX86)
     }
 
     sourceSets {
         nativeMain { dependsOn commonMain }
-        // Empty source set is required in order to have native tests task
-        nativeTest {}
+        nativeTest { dependsOn commonTest }
 
-        if (!project.ext.ideaActive) {
-            configure(nativeMainSets) {
-                dependsOn nativeMain
-            }
-
-            configure(nativeTestSets) {
-                dependsOn nativeTest
-            }
-        }
+        configure(nativeMainSets) { dependsOn nativeMain }
+        configure(nativeTestSets) { dependsOn nativeTest }
     }
 }

gradle/targets.gradle

@@ -86,8 +86,8 @@ task testMochaChrome(type: NodeTask, dependsOn: prepareMochaChrome) {
 task installDependenciesMochaJsdom(type: NpmTask, dependsOn: [npmInstall]) {
     args = ['install',
             "mocha@$mocha_version",
-            'jsdom@15.2.1',
-            'jsdom-global@3.0.2',
+            "jsdom@$jsdom_version",
+            "jsdom-global@$jsdom_global_version",
             "source-map-support@$source_map_support_version",
             '--no-save']
     if (project.hasProperty("teamcity")) args += ["mocha-teamcity-reporter@$mocha_teamcity_reporter_version"]

gradle/test-mocha-js.gradle

@@ -3,13 +3,60 @@
  */
 
 apply plugin: 'org.jetbrains.kotlin.multiplatform'
-apply from: rootProject.file("gradle/targets.gradle")
 apply from: rootProject.file("gradle/compile-jvm-multiplatform.gradle")
 apply from: rootProject.file("gradle/compile-common.gradle")
 apply from: rootProject.file("gradle/compile-js-multiplatform.gradle")
 apply from: rootProject.file("gradle/compile-native-multiplatform.gradle")
 apply from: rootProject.file('gradle/publish-npm-js.gradle')
 
+/* ==========================================================================
+  Configure source sets structure for kotlinx-coroutines-core:
+
+  TARGETS                            SOURCE SETS
+  -------         ----------------------------------------------
+
+     js -----------------------------------------------------+
+                                                             |
+                                                             V
+     jvm -------------------------------> concurrent ---> common
+                                              ^
+     ios     \                                |
+     macos   | ---> nativeDarwin ---> native --+
+     tvos    |                         ^
+     watchos /                         |
+                                       |
+     linux  \  ---> nativeOther -------+
+     mingw  /
+
+   ========================================================================== */
+
+project.ext.sourceSetSuffixes = ["Main", "Test"]
+
+void defineSourceSet(newName, dependsOn, includedInPred) {
+    for (suffix in project.ext.sourceSetSuffixes) {
+        def newSS = kotlin.sourceSets.maybeCreate(newName + suffix)
+        for (dep in dependsOn) {
+            newSS.dependsOn(kotlin.sourceSets[dep + suffix])
+        }
+        for (curSS in kotlin.sourceSets) {
+            def curName = curSS.name
+            if (curName.endsWith(suffix)) {
+                def prefix = curName.substring(0, curName.length() - suffix.length())
+                if (includedInPred(prefix)) curSS.dependsOn(newSS)
+            }
+        }
+    }
+}
+
+static boolean isNativeDarwin(String name) { return ["ios", "macos", "tvos", "watchos"].any { name.startsWith(it) } }
+static boolean isNativeOther(String name) { return ["linux", "mingw"].any { name.startsWith(it) } }
+
+defineSourceSet("concurrent", ["common"]) { it in ["jvm", "native"] }
+defineSourceSet("nativeDarwin", ["native"]) { isNativeDarwin(it) }
+defineSourceSet("nativeOther", ["native"]) { isNativeOther(it) }
+
+/* ========================================================================== */
+
 /*
  * All platform plugins and configuration magic happens here instead of build.gradle
  * because JMV-only projects depend on core, thus core should always be initialized before configuration.
@@ -18,7 +65,7 @@ kotlin {
     configure(sourceSets) {
         def srcDir = name.endsWith('Main') ? 'src' : 'test'
         def platform = name[0..-5]
-        kotlin.srcDir "$platform/$srcDir"
+        kotlin.srcDirs = ["$platform/$srcDir"]
         if (name == "jvmMain") {
             resources.srcDirs = ["$platform/resources"]
         } else if (name == "jvmTest") {
@@ -31,12 +78,18 @@ kotlin {
     }
 
     configure(targets) {
-        def targetName = it.name
-        compilations.all { compilation ->
-            def compileTask = tasks.getByName(compilation.compileKotlinTaskName)
-            // binary compatibility support
-            if (targetName.contains("jvm") && compilation.compilationName == "main") {
-                compileTask.kotlinOptions.freeCompilerArgs += ["-Xdump-declarations-to=${buildDir}/visibilities.json"]
+        // Configure additional binaries and test runs -- one for each OS
+        if (["macos", "linux", "mingw"].any { name.startsWith(it) }) {
+            binaries {
+                // Test for memory leaks using a special entry point that does not exit but returns from main
+                binaries.getTest("DEBUG").freeCompilerArgs += ["-e", "kotlinx.coroutines.mainNoExit"]
+                // Configure a separate test where code runs in background
+                test("background", [org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType.DEBUG]) {
+                    freeCompilerArgs += ["-e", "kotlinx.coroutines.mainBackground"]
+                }
+            }
+            testRuns {
+                background { setExecutionSourceFrom(binaries.backgroundDebugTest) }
             }
         }
     }
@@ -54,23 +107,52 @@ compileKotlinMetadata {
     }
 }
 
+// :KLUDGE: Idea.active: This is needed to workaround resolve problems after importing this project to IDEA
+def configureNativeSourceSetPreset(name, preset) {
+    def hostMainCompilation = project.kotlin.targetFromPreset(preset).compilations.main
+    // Look for platform libraries in "implementation" for default source set
+    def implementationConfiguration = configurations[hostMainCompilation.defaultSourceSet.implementationMetadataConfigurationName]
+    // Now find the libraries: Finds platform libs & stdlib, but platform declarations are still not resolved due to IDE bugs
+    def hostNativePlatformLibs = files(
+        provider {
+            implementationConfiguration.findAll {
+                it.path.endsWith(".klib") || it.absolutePath.contains("klib${File.separator}platform") || it.absolutePath.contains("stdlib")
+            }
+        }
+    )
+    // Add all those dependencies
+    for (suffix in sourceSetSuffixes) {
+        configure(kotlin.sourceSets[name + suffix]) {
+            dependencies.add(implementationMetadataConfigurationName, hostNativePlatformLibs)
+        }
+    }
+}
+
+// :KLUDGE: Idea.active: Configure platform libraries for native source sets when working in IDEA
+if (Idea.active) {
+    def manager = project.ext.hostManager
+    def linuxPreset = kotlin.presets.linuxX64
+    def macosPreset = kotlin.presets.macosX64
+    // linux should be always available (cross-compilation capable) -- use it as default
+    assert manager.isEnabled(linuxPreset.konanTarget)
+    // use macOS libs for nativeDarwin if available
+    def macosAvailable = manager.isEnabled(macosPreset.konanTarget)
+    // configure source sets
+    configureNativeSourceSetPreset("native", linuxPreset)
+    configureNativeSourceSetPreset("nativeOther", linuxPreset)
+    configureNativeSourceSetPreset("nativeDarwin", macosAvailable ? macosPreset : linuxPreset)
+}
+
 kotlin.sourceSets {
     jvmMain.dependencies {
         compileOnly "com.google.android:annotations:4.1.1.4"
     }
 
     jvmTest.dependencies {
-        // This is a workaround for https://youtrack.jetbrains.com/issue/KT-39037
-        def excludingCurrentProject = { dependency ->
-            def result = project.dependencies.create(dependency)
-            result.exclude(module: project.name)
-            return result
-        }
-
-        api(excludingCurrentProject("org.jetbrains.kotlinx:lincheck:$lincheck_version"))
+        api "org.jetbrains.kotlinx:lincheck:$lincheck_version"
         api "org.jetbrains.kotlinx:kotlinx-knit-test:$knit_version"
         api "com.esotericsoftware:kryo:4.0.0"
-        implementation(excludingCurrentProject(project(":android-unit-tests")))
+        implementation project(":android-unit-tests")
     }
 }
 
@@ -97,7 +179,7 @@ jvmTest {
     enableAssertions = true
     systemProperty 'java.security.manager', 'kotlinx.coroutines.TestSecurityManager'
     // 'stress' is required to be able to run all subpackage tests like ":jvmTests --tests "*channels*" -Pstress=true"
-    if (!project.ext.ideaActive && rootProject.properties['stress'] == null) {
+    if (!Idea.active && rootProject.properties['stress'] == null) {
         exclude '**/*StressTest.*'
     }
     systemProperty 'kotlinx.coroutines.scheduler.keep.alive.sec', '100000' // any unpark problem hangs test

kotlinx-coroutines-core/build.gradle

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.coroutines
+
+// It is used in the main sources of native-mt branch
+internal expect inline fun workerMain(block: () -> Unit)

kotlinx-coroutines-core/native/src/WorkerMain.native.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package kotlinx.coroutines
@@ -19,6 +19,7 @@ class WorkerTest : TestBase() {
                 delay(1)
             }
         }.result
+        worker.requestTermination()
     }
 
     @Test
@@ -31,5 +32,6 @@ class WorkerTest : TestBase() {
                 }.join()
             }
         }.result
+        worker.requestTermination()
     }
 }

kotlinx-coroutines-core/native/test/WorkerTest.kt

@@ -0,0 +1,13 @@
+/*
+ * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.coroutines
+
+import kotlinx.cinterop.*
+
+internal actual inline fun workerMain(block: () -> Unit) {
+    autoreleasepool {
+        block()
+    }
+}

kotlinx-coroutines-core/nativeDarwin/src/WorkerMain.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.coroutines
+
+import platform.CoreFoundation.*
+import kotlin.native.concurrent.*
+import kotlin.native.internal.test.*
+import kotlin.system.*
+
+// This is a separate entry point for tests in background
+fun mainBackground(args: Array<String>) {
+    val worker = Worker.start(name = "main-background")
+    worker.execute(TransferMode.SAFE, { args.freeze() }) {
+        val result = testLauncherEntryPoint(it)
+        exitProcess(result)
+    }
+    CFRunLoopRun()
+    error("CFRunLoopRun should never return")
+}
+
+// This is a separate entry point for tests with leak checker
+fun mainNoExit(args: Array<String>) {
+    workerMain { // autoreleasepool to make sure interop objects are properly freed
+        testLauncherEntryPoint(args)
+    }
+}