Python type hints in a separate module (#2510)

Author: tochilinak

settings.gradle.kts

@@ -65,6 +65,7 @@ if (pythonIde.split(",").contains(ideType)) {
     include("utbot-cli-python")
     include("utbot-intellij-python")
     include("utbot-python-parser")
+    include("utbot-python-types")
 }
 
 include("utbot-spring-sample")
@@ -85,4 +86,3 @@ if (projectType == ultimateEdition) {
 }
 
 include("utbot-light")
-

utbot-python-types/.gitignore

@@ -0,0 +1 @@
+dist/

utbot-python-types/build.gradle.kts

@@ -0,0 +1,114 @@
+val kotlinLoggingVersion: String? by rootProject
+
+dependencies {
+    implementation("com.squareup.moshi:moshi:1.11.0")
+    implementation("com.squareup.moshi:moshi-kotlin:1.11.0")
+    implementation("com.squareup.moshi:moshi-adapters:1.11.0")
+    implementation(group = "io.github.microutils", name = "kotlin-logging", version = kotlinLoggingVersion)
+}
+
+val utbotMypyRunnerVersion = File(project.projectDir, "src/main/resources/utbot_mypy_runner_version").readText()
+// these two properties --- from GRADLE_USER_HOME/gradle.properties
+val pypiToken: String? by project
+val pythonInterpreter: String? by project
+val utbotMypyRunnerPath = File(project.projectDir, "src/main/python/utbot_mypy_runner")
+val localMypyPath = File(utbotMypyRunnerPath, "dist")
+
+tasks.register("cleanDist") {
+    group = "python"
+    delete(localMypyPath.canonicalPath)
+}
+
+val setMypyRunnerVersion =
+    if (pythonInterpreter != null)
+        tasks.register<Exec>("setVersion") {
+        group = "python"
+        workingDir = utbotMypyRunnerPath
+        commandLine(pythonInterpreter, "-m", "poetry", "version", utbotMypyRunnerVersion)
+    } else {
+        null
+    }
+
+val buildMypyRunner =
+    if (pythonInterpreter != null) {
+        tasks.register<Exec>("buildUtbotMypyRunner") {
+            dependsOn(setMypyRunnerVersion!!)
+            group = "python"
+            workingDir = utbotMypyRunnerPath
+            commandLine(pythonInterpreter, "-m", "poetry", "build")
+        }
+    } else {
+        null
+    }
+
+if (pythonInterpreter != null && pypiToken != null) {
+    tasks.register<Exec>("publishUtbotMypyRunner") {
+        dependsOn(buildMypyRunner!!)
+        group = "python"
+        workingDir = utbotMypyRunnerPath
+        commandLine(
+            pythonInterpreter,
+            "-m",
+            "poetry",
+            "publish",
+            "-u",
+            "__token__",
+            "-p",
+            pypiToken
+        )
+    }
+}
+
+val installMypyRunner =
+    if (pythonInterpreter != null) {
+        tasks.register<Exec>("installUtbotMypyRunner") {
+            dependsOn(buildMypyRunner!!)
+            group = "python"
+            environment("PIP_FIND_LINKS" to localMypyPath.canonicalPath)
+            commandLine(
+                pythonInterpreter,
+                "-m",
+                "pip",
+                "install",
+                "utbot_mypy_runner==$utbotMypyRunnerVersion"
+            )
+        }
+    } else {
+        null
+    }
+
+val samplesDir = File(project.projectDir, "src/test/resources/samples")
+val buildDir = File(project.buildDir, "samples_builds")
+val jsonDir = File(project.projectDir, "src/test/resources")
+
+fun getParamsForSample(sampleName: String): List<String> =
+    listOf(
+        File(samplesDir, "$sampleName.py").canonicalPath,
+        sampleName,
+        buildDir.canonicalPath,
+        File(jsonDir, "$sampleName.json").canonicalPath
+    )
+
+val samplesInfo = listOf(
+    getParamsForSample("annotation_tests"),
+    getParamsForSample("boruvka"),
+    getParamsForSample("import_test"),
+    getParamsForSample("subtypes"),
+)
+
+if (pythonInterpreter != null) {
+    val subtasks = samplesInfo.mapIndexed { index, params ->
+        tasks.register<JavaExec>("regenerateJsonForTests_$index") {
+            dependsOn(installMypyRunner!!)
+            group = "python_subtasks"
+            classpath = sourceSets.test.get().runtimeClasspath
+            args = listOf(pythonInterpreter) + params
+            mainClass.set("org.utbot.python.newtyping.samples.GenerateMypyInfoBuildKt")
+        }
+    }
+
+    tasks.register("regenerateJsonForTests") {
+        subtasks.forEach { dependsOn(it) }
+        group = "python"
+    }
+}

utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonDefinition.kt renamed to utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/PythonDefinition.kt

@@ -1,9 +1,9 @@
 package org.utbot.python.newtyping
 
 import org.utbot.python.newtyping.general.FunctionType
-import org.utbot.python.newtyping.general.Type
+import org.utbot.python.newtyping.general.UtType
 
-open class PythonDefinition(open val meta: PythonDefinitionDescription, open val type: Type) {
+open class PythonDefinition(open val meta: PythonDefinitionDescription, open val type: UtType) {
     override fun toString(): String =
         "${meta.name}: ${type.pythonTypeRepresentation()}"
 }

utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt renamed to utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt

@@ -5,15 +5,15 @@ import org.utbot.python.newtyping.general.Name
 import org.utbot.python.newtyping.utils.isRequired
 
 sealed class PythonTypeDescription(name: Name) : TypeMetaDataWithName(name) {
-    open fun castToCompatibleTypeApi(type: Type): Type = type
-    open fun getNamedMembers(type: Type): List<PythonDefinition> = emptyList()  // direct members (without inheritance)
-    open fun getAnnotationParameters(type: Type): List<Type> = emptyList()
-    open fun getMemberByName(storage: PythonTypeStorage, type: Type, name: String): PythonDefinition? =
+    open fun castToCompatibleTypeApi(type: UtType): UtType = type
+    open fun getNamedMembers(type: UtType): List<PythonDefinition> = emptyList()  // direct members (without inheritance)
+    open fun getAnnotationParameters(type: UtType): List<UtType> = emptyList()
+    open fun getMemberByName(storage: PythonTypeHintsStorage, type: UtType, name: String): PythonDefinition? =
         // overridden for some types
         getNamedMembers(type).find { it.meta.name == name }
-    open fun createTypeWithNewAnnotationParameters(like: Type, newParams: List<Type>): Type =  // overriden for Callable
+    open fun createTypeWithNewAnnotationParameters(like: UtType, newParams: List<UtType>): UtType =  // overriden for Callable
         DefaultSubstitutionProvider.substituteAll(like.getOrigin(), newParams)
-    open fun getTypeRepresentation(type: Type): String {  // overriden for Callable
+    open fun getTypeRepresentation(type: UtType): String {  // overriden for Callable
         if (name.prefix == listOf("builtins") && name.name == "tuple") {
             return "${getTypeName()}[${type.parameters.first().pythonTypeRepresentation()}, ...]"
         }
@@ -35,7 +35,7 @@ sealed class PythonTypeDescription(name: Name) : TypeMetaDataWithName(name) {
     fun getName(): String {
         return name.name
     }
-    fun getModules(type: Type): Set<String> {
+    fun getModules(type: UtType): Set<String> {
         val cur = if (name.prefix.isNotEmpty())
             setOf(name.prefix.joinToString(separator = "."))
         else
@@ -50,12 +50,12 @@ sealed class PythonCompositeTypeDescription(
     name: Name,
     private val memberDescriptions: List<PythonDefinitionDescription>
 ): PythonTypeDescription(name) {
-    override fun castToCompatibleTypeApi(type: Type): CompositeType {
+    override fun castToCompatibleTypeApi(type: UtType): CompositeType {
         return type as? CompositeType
             ?: error("Got unexpected type PythonCompositeTypeDescription: $type")
     }
 
-    override fun getNamedMembers(type: Type): List<PythonDefinition> {
+    override fun getNamedMembers(type: UtType): List<PythonDefinition> {
         val compositeType = castToCompatibleTypeApi(type)
         assert(compositeType.members.size == memberDescriptions.size)
         return (memberDescriptions zip compositeType.members).map { (descr, typ) ->
@@ -66,8 +66,8 @@ sealed class PythonCompositeTypeDescription(
         }
     }
 
-    override fun getAnnotationParameters(type: Type): List<Type> = type.parameters
-    fun mro(storage: PythonTypeStorage, type: Type): List<Type> {
+    override fun getAnnotationParameters(type: UtType): List<UtType> = type.parameters
+    fun mro(storage: PythonTypeHintsStorage, type: UtType): List<UtType> {
         val compositeType = castToCompatibleTypeApi(type)
         var bases = compositeType.supertypes
         if (bases.isEmpty() && !type.isPythonObjectType())
@@ -82,7 +82,7 @@ sealed class PythonCompositeTypeDescription(
             linBases.removeIf { it.isEmpty() }
             if (linBases.isEmpty())
                 break
-            lateinit var addAtThisIteration: Type
+            lateinit var addAtThisIteration: UtType
             for (seq in linBases) {
                 val head = seq.first()
                 val isContainedSomewhereElse = linBases.any {
@@ -102,7 +102,7 @@ sealed class PythonCompositeTypeDescription(
         return result
     }
 
-    override fun getMemberByName(storage: PythonTypeStorage, type: Type, name: String): PythonDefinition? {
+    override fun getMemberByName(storage: PythonTypeHintsStorage, type: UtType, name: String): PythonDefinition? {
         for (parent in mro(storage, type)) {
             val cur = parent.getPythonAttributes().find { it.meta.name == name }
             if (cur != null)
@@ -119,7 +119,7 @@ class PythonTypeVarDescription(
     val variance: Variance,
     val parameterKind: ParameterKind
 ) : PythonTypeDescription(name) {
-    override fun castToCompatibleTypeApi(type: Type): TypeParameter {
+    override fun castToCompatibleTypeApi(type: UtType): TypeParameter {
         return type as? TypeParameter
             ?: error("Got unexpected type PythonTypeVarDescription: $type")
     }
@@ -154,17 +154,17 @@ class PythonCallableTypeDescription(
     val argumentNames: List<String?>  // like in mypy's CallableType: https://github.com/python/mypy/blob/master/mypy/types.py#L1672
 ): PythonTypeDescription(pythonCallableName) {
     val numberOfArguments = argumentKinds.size
-    override fun castToCompatibleTypeApi(type: Type): FunctionType {
+    override fun castToCompatibleTypeApi(type: UtType): FunctionType {
         return type as? FunctionType
             ?: error("Got unexpected type PythonCallableTypeDescription: $type")
     }
 
-    override fun getNamedMembers(type: Type): List<PythonDefinition> {
+    override fun getNamedMembers(type: UtType): List<PythonDefinition> {
         val functionType = castToCompatibleTypeApi(type)
         return listOf(PythonDefinition(PythonVariableDescription("__call__"), functionType))
     }
 
-    override fun getAnnotationParameters(type: Type): List<Type> {
+    override fun getAnnotationParameters(type: UtType): List<UtType> {
         val functionType = castToCompatibleTypeApi(type)
         return functionType.arguments + listOf(functionType.returnValue)
     }
@@ -178,7 +178,7 @@ class PythonCallableTypeDescription(
         ARG_NAMED_OPT
     }
 
-    override fun createTypeWithNewAnnotationParameters(like: Type, newParams: List<Type>): Type {
+    override fun createTypeWithNewAnnotationParameters(like: UtType, newParams: List<UtType>): UtType {
         val args = newParams.dropLast(1)
         val returnValue = newParams.last()
         return createPythonCallableType(
@@ -200,15 +200,15 @@ class PythonCallableTypeDescription(
         }
     }
 
-    override fun getTypeRepresentation(type: Type): String {
+    override fun getTypeRepresentation(type: UtType): String {
         val functionType = castToCompatibleTypeApi(type)
         val root = name.prefix.joinToString(".") + "." + name.name
         return "$root[[${
             functionType.arguments.joinToString(separator = ", ") { it.pythonTypeRepresentation() }
         }], ${functionType.returnValue.pythonTypeRepresentation()}]"
     }
 
-    fun removeNonPositionalArgs(type: Type): FunctionType {
+    fun removeNonPositionalArgs(type: UtType): FunctionType {
         val functionType = castToCompatibleTypeApi(type)
         val argsCount = argumentKinds.count { it == ArgKind.ARG_POS }
         return createPythonCallableType(
@@ -228,7 +228,7 @@ class PythonCallableTypeDescription(
         }
     }
 
-    fun removeNotRequiredArgs(type: Type): FunctionType {
+    fun removeNotRequiredArgs(type: UtType): FunctionType {
         val functionType = castToCompatibleTypeApi(type)
         return createPythonCallableType(
             functionType.parameters.size,
@@ -250,7 +250,7 @@ class PythonCallableTypeDescription(
 
 // Special Python annotations
 object PythonAnyTypeDescription : PythonSpecialAnnotation(pythonAnyName) {
-    override fun getMemberByName(storage: PythonTypeStorage, type: Type, name: String): PythonDefinition {
+    override fun getMemberByName(storage: PythonTypeHintsStorage, type: UtType, name: String): PythonDefinition {
         return PythonDefinition(PythonVariableDescription(name), pythonAnyType)
     }
 }
@@ -260,7 +260,7 @@ object PythonNoneTypeDescription : PythonSpecialAnnotation(pythonNoneName) {
 }
 
 object PythonUnionTypeDescription : PythonSpecialAnnotation(pythonUnionName) {
-    override fun getMemberByName(storage: PythonTypeStorage, type: Type, name: String): PythonDefinition? {
+    override fun getMemberByName(storage: PythonTypeHintsStorage, type: UtType, name: String): PythonDefinition? {
         val children = type.parameters.mapNotNull {
             it.getPythonAttributeByName(storage, name)?.type
         }
@@ -273,28 +273,28 @@ object PythonUnionTypeDescription : PythonSpecialAnnotation(pythonUnionName) {
             )
     }
 
-    override fun getAnnotationParameters(type: Type): List<Type> = type.parameters
+    override fun getAnnotationParameters(type: UtType): List<UtType> = type.parameters
 }
 
 object PythonOverloadTypeDescription : PythonSpecialAnnotation(overloadName) {
-    override fun getAnnotationParameters(type: Type): List<Type> = type.parameters
-    override fun getNamedMembers(type: Type): List<PythonDefinition> {
+    override fun getAnnotationParameters(type: UtType): List<UtType> = type.parameters
+    override fun getNamedMembers(type: UtType): List<PythonDefinition> {
         return listOf(PythonDefinition(PythonVariableDescription("__call__"), type))
     }
 }
 
 object PythonTypeAliasDescription : PythonSpecialAnnotation(pythonTypeAliasName) {
-    override fun castToCompatibleTypeApi(type: Type): CompositeType {
+    override fun castToCompatibleTypeApi(type: UtType): CompositeType {
         return type as? CompositeType ?: error("Got unexpected type for PythonTypeAliasDescription: $type")
     }
-    fun getInterior(type: Type): Type {
+    fun getInterior(type: UtType): UtType {
         val casted = castToCompatibleTypeApi(type)
         return casted.members.first()
     }
 }
 
 object PythonTupleTypeDescription : PythonSpecialAnnotation(pythonTupleName) {
-    override fun getAnnotationParameters(type: Type): List<Type> = castToCompatibleTypeApi(type).parameters
+    override fun getAnnotationParameters(type: UtType): List<UtType> = castToCompatibleTypeApi(type).parameters
     // TODO: getMemberByName and/or getNamedMembers
 }
 
@@ -306,13 +306,13 @@ private fun initTypeVar(param: TypeParameter) {
     )
 }
 
-private fun substituteMembers(origin: Type, members: List<Type>): Type =
+private fun substituteMembers(origin: UtType, members: List<UtType>): UtType =
     DefaultSubstitutionProvider.substitute(
         origin,
         (origin.parameters.map { it as TypeParameter } zip members).associate { it }
     )
 
-fun createTypeWithMembers(description: PythonTypeDescription, members: List<Type>): Type {
+fun createTypeWithMembers(description: PythonTypeDescription, members: List<UtType>): UtType {
     val origin = TypeCreator.create(members.size, description) {
         it.parameters.forEach(::initTypeVar)
     }