Fix deadlock (#1541)

[utbot-rd]
Fix deadlock when IDEA process under readlock asks Engine process, which then asks IDEA process for isCancalled which also tries to take readlock, but some write action occured and readlock for isCancelled cant be taken
Fix #1213
Fix #1539

Author: Domonion

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/CodeGenerationController.kt

@@ -8,6 +8,7 @@ import com.intellij.ide.fileTemplates.FileTemplateUtil
 import com.intellij.ide.fileTemplates.JavaTemplateUtil
 import com.intellij.ide.highlighter.JavaFileType
 import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.application.readAction
 import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.command.WriteCommandAction.runWriteCommandAction
@@ -225,7 +226,7 @@ object CodeGenerationController {
             .doInspections(AnalysisScope(model.project))
     }
 
-    private fun proceedTestReport(proc: EngineProcess, model: GenerateTestsModel) = runReadAction {
+    private fun proceedTestReport(proc: EngineProcess, model: GenerateTestsModel) {
         try {
             // Parametrized tests are not supported in tests report yet
             // TODO JIRA:1507
@@ -675,8 +676,7 @@ object CodeGenerationController {
         run(THREAD_POOL, indicator, "Rendering test code") {
             val (generatedTestsCode, utilClassKind) = try {
                 val paramNames = try {
-                    DumbService.getInstance(model.project)
-                        .runReadActionInSmartMode(Computable { proc.findMethodParamNames(classUnderTest, classMethods) })
+                    proc.findMethodParamNames(classUnderTest, classMethods)
                 } catch (e: Exception) {
                     logger.warn(e) { "Cannot find method param names for ${classUnderTest.name}" }
                     reportsCountDown.countDown()
@@ -833,24 +833,26 @@ object CodeGenerationController {
     }
 
 
-    private fun eventLogMessage(project: Project): String? {
-        if (ToolWindowManager.getInstance(project).getToolWindow("Event Log") != null)
-            return     """
+    private fun eventLogMessage(project: Project): String? = runReadAction {
+        return@runReadAction if (ToolWindowManager.getInstance(project).getToolWindow("Event Log") != null)
+            """
             <a href="${TestReportUrlOpeningListener.prefix}${TestReportUrlOpeningListener.eventLogSuffix}">See details in Event Log</a>.
         """.trimIndent()
-        return null
+        else null
     }
 
     private fun showTestsReport(proc: EngineProcess, model: GenerateTestsModel) {
         val (notifyMessage, statistics, hasWarnings) = proc.generateTestsReport(model, eventLogMessage(model.project))
 
-        if (hasWarnings) {
-            WarningTestsReportNotifier.notify(notifyMessage)
-        } else {
-            TestsReportNotifier.notify(notifyMessage)
-        }
+        runReadAction {
+            if (hasWarnings) {
+                WarningTestsReportNotifier.notify(notifyMessage)
+            } else {
+                TestsReportNotifier.notify(notifyMessage)
+            }
 
-        statistics?.let { DetailsTestsReportNotifier.notify(it) }
+            statistics?.let { DetailsTestsReportNotifier.notify(it) }
+        }
     }
 
     @Suppress("unused")

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt

@@ -192,28 +192,30 @@ object UtTestsDialogProcessor {
                                 }
 
                                 val (methods, className) = process.executeWithTimeoutSuspended {
+                                    var canonicalName = ""
+                                    var srcMethods: List<MemberInfo> = emptyList()
                                     DumbService.getInstance(project)
-                                    .runReadActionInSmartMode(Computable {
-                                        val canonicalName = srcClass.canonicalName
-                                        val classId = process.obtainClassId(canonicalName)
-                                        psi2KClass[srcClass] = classId
-                                        val srcMethods = if (model.extractMembersFromSrcClasses) {
-                                            val chosenMethods =
-                                                model.selectedMembers.filter { it.member is PsiMethod }
-                                            val chosenNestedClasses =
-                                                model.selectedMembers.mapNotNull { it.member as? PsiClass }
-                                            chosenMethods + chosenNestedClasses.flatMap {
-                                                it.extractClassMethodsIncludingNested(false)
+                                        .runReadActionInSmartMode(Computable {
+                                            canonicalName = srcClass.canonicalName
+                                            srcMethods = if (model.extractMembersFromSrcClasses) {
+                                                val chosenMethods =
+                                                    model.selectedMembers.filter { it.member is PsiMethod }
+                                                val chosenNestedClasses =
+                                                    model.selectedMembers.mapNotNull { it.member as? PsiClass }
+                                                chosenMethods + chosenNestedClasses.flatMap {
+                                                    it.extractClassMethodsIncludingNested(false)
+                                                }
+                                            } else {
+                                                srcClass.extractClassMethodsIncludingNested(false)
                                             }
-                                        } else {
-                                            srcClass.extractClassMethodsIncludingNested(false)
-                                        }
-                                        process.findMethodsInClassMatchingSelected(
-                                            classId,
-                                            srcMethods
-                                        ) to srcClass.name
-                                    })
-                            }
+                                        })
+                                    val classId = process.obtainClassId(canonicalName)
+                                    psi2KClass[srcClass] = classId
+                                    process.findMethodsInClassMatchingSelected(
+                                        classId,
+                                        srcMethods
+                                    ) to srcClass.name
+                                }
 
                                 if (methods.isEmpty()) {
                                     logger.error { "No methods matching selected found in class $className." }

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/process/EngineProcess.kt

@@ -1,8 +1,10 @@
 package org.utbot.intellij.plugin.process
 
 import com.intellij.ide.plugins.cl.PluginClassLoader
+import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.Computable
 import com.intellij.psi.PsiMethod
 import com.intellij.psi.impl.file.impl.JavaFileManager
 import com.intellij.psi.search.GlobalSearchScope
@@ -30,8 +32,7 @@ import org.utbot.instrumentation.util.KryoHelper
 import org.utbot.intellij.plugin.UtbotBundle
 import org.utbot.intellij.plugin.models.GenerateTestsModel
 import org.utbot.intellij.plugin.ui.TestReportUrlOpeningListener
-import org.utbot.intellij.plugin.util.assertIsNonDispatchThread
-import org.utbot.intellij.plugin.util.assertIsReadAccessAllowed
+import org.utbot.intellij.plugin.util.assertReadAccessNotAllowed
 import org.utbot.intellij.plugin.util.methodDescription
 import org.utbot.rd.*
 import org.utbot.rd.exceptions.InstantProcessDeathException
@@ -164,7 +165,8 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
     private val sourceFindingStrategies = ConcurrentHashMap<Long, SourceFindingStrategy>()
 
     fun setupUtContext(classpathForUrlsClassloader: List<String>) {
-        engineModel.setupUtContext.start(lifetime, SetupContextParams(classpathForUrlsClassloader))
+        assertReadAccessNotAllowed()
+        engineModel.setupUtContext.startBlocking(SetupContextParams(classpathForUrlsClassloader))
     }
 
     private fun computeSourceFileByClass(params: ComputeSourceFileByClassArguments): String =
@@ -185,6 +187,8 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
         jdkInfo: JdkInfo,
         isCancelled: (Unit) -> Boolean
     ) {
+        assertReadAccessNotAllowed()
+
         engineModel.isCancelled.set(handler = isCancelled)
         instrumenterAdapterModel.computeSourceFileByClass.set(handler = this::computeSourceFileByClass)
 
@@ -194,19 +198,18 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
             dependencyPaths,
             JdkInfo(jdkInfo.path.pathString, jdkInfo.version)
         )
-        engineModel.createTestGenerator.start(lifetime, params)
+        engineModel.createTestGenerator.startBlocking(params)
     }
 
     fun obtainClassId(canonicalName: String): ClassId {
-        assertIsNonDispatchThread()
+        assertReadAccessNotAllowed()
         return kryoHelper.readObject(engineModel.obtainClassId.startBlocking(canonicalName))
     }
 
     fun findMethodsInClassMatchingSelected(clazzId: ClassId, srcMethods: List<MemberInfo>): List<ExecutableId> {
-        assertIsNonDispatchThread()
-        assertIsReadAccessAllowed()
+        assertReadAccessNotAllowed()
 
-        val srcDescriptions = srcMethods.map { it.methodDescription() }
+        val srcDescriptions = runReadAction { srcMethods.map { it.methodDescription() } }
         val rdDescriptions = srcDescriptions.map { MethodDescription(it.name, it.containingClass, it.parameterTypes) }
         val binaryClassId = kryoHelper.writeObject(clazzId)
         val arguments = FindMethodsInClassMatchingSelectedArguments(binaryClassId, rdDescriptions)
@@ -216,10 +219,13 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
     }
 
     fun findMethodParamNames(classId: ClassId, methods: List<MemberInfo>): Map<ExecutableId, List<String>> {
-        assertIsNonDispatchThread()
-        assertIsReadAccessAllowed()
+        assertReadAccessNotAllowed()
 
-        val bySignature = methods.associate { it.methodDescription() to it.paramNames() }
+        val bySignature = executeWithTimeoutSuspended {
+            DumbService.getInstance(project).runReadActionInSmartMode(Computable {
+                methods.associate { it.methodDescription() to it.paramNames() }
+            })
+        }
         val arguments = FindMethodParamNamesArguments(
             kryoHelper.writeObject(classId),
             kryoHelper.writeObject(bySignature)
@@ -254,7 +260,7 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
         fuzzingValue: Double,
         searchDirectory: String
     ): RdTestGenerationResult {
-        assertIsNonDispatchThread()
+        assertReadAccessNotAllowed()
         val params = GenerateParams(
             mockInstalled,
             staticsMockingIsConfigured,
@@ -291,7 +297,7 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
         enableTestsTimeout: Boolean,
         testClassPackageName: String
     ): Pair<String, UtilClassKind?> {
-        assertIsNonDispatchThread()
+        assertReadAccessNotAllowed()
         val params = RenderParams(
             testSetsId,
             kryoHelper.writeObject(classUnderTest),
@@ -353,7 +359,7 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
         generatedTestsCode: String,
         sourceFindingStrategy: SourceFindingStrategy
     ): String {
-        assertIsNonDispatchThread()
+        assertReadAccessNotAllowed()
 
         val params = WriteSarifReportArguments(testSetsId, reportFilePath.pathString, generatedTestsCode)
 
@@ -362,7 +368,7 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
     }
 
     fun generateTestsReport(model: GenerateTestsModel, eventLogMessage: String?): Triple<String, String?, Boolean> {
-        assertIsNonDispatchThread()
+        assertReadAccessNotAllowed()
 
         val forceMockWarning = UtbotBundle.takeIf(
             "test.report.force.mock.warning",
@@ -411,10 +417,12 @@ class EngineProcess private constructor(val project: Project, rdProcess: Process
 
     fun <T> executeWithTimeoutSuspended(block: () -> T): T {
         try {
+            assertReadAccessNotAllowed()
             protocol.synchronizationModel.suspendTimeoutTimer.startBlocking(true)
             return block()
         }
         finally {
+            assertReadAccessNotAllowed()
             protocol.synchronizationModel.suspendTimeoutTimer.startBlocking(false)
         }
     }

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/util/IdeaThreadingUtil.kt

@@ -10,14 +10,18 @@ fun assertIsWriteThread() {
     ApplicationManager.getApplication().isWriteThread()
 }
 
-fun assertIsReadAccessAllowed() {
+fun assertReadAccessAllowed() {
     ApplicationManager.getApplication().assertReadAccessAllowed()
 }
 
-fun assertIsWriteAccessAllowed() {
+fun assertWriteAccessAllowed() {
     ApplicationManager.getApplication().assertWriteAccessAllowed()
 }
 
 fun assertIsNonDispatchThread() {
     ApplicationManager.getApplication().assertIsNonDispatchThread()
+}
+
+fun assertReadAccessNotAllowed() {
+    ApplicationManager.getApplication().assertReadAccessNotAllowed()
 }