Skip to content

Commit 5ee3695

Browse files
tochilinaktochilinak
authored
Update of module utbot-python-types (#2575)
* update of utbot_mypy_runner + small fixes * Regenerated tests * Fixed some corner cases * regenerated tests * added timeout for dmypy run
1 parent eb8ece9 commit 5ee3695

File tree

29 files changed

+430
-168
lines changed

29 files changed

+430
-168
lines changed

‎utbot-python-types/build.gradle.kts‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
val kotlinLoggingVersion: String? by rootProject
22

33
dependencies {
4-
implementation("com.squareup.moshi:moshi:1.11.0")
5-
implementation("com.squareup.moshi:moshi-kotlin:1.11.0")
6-
implementation("com.squareup.moshi:moshi-adapters:1.11.0")
4+
implementation("com.squareup.moshi:moshi:1.14.0")
5+
implementation("com.squareup.moshi:moshi-kotlin:1.14.0")
6+
implementation("com.squareup.moshi:moshi-adapters:1.14.0")
77
implementation(group = "io.github.microutils", name = "kotlin-logging", version = kotlinLoggingVersion)
88
}
99

@@ -72,6 +72,29 @@ if (pythonInterpreter != null && pypiToken != null) {
7272
}
7373
}
7474

75+
val uninstallMypyRunner =
76+
if (pythonInterpreter != null) {
77+
tasks.register<Exec>("uninstallMypyRunner") {
78+
group = "python"
79+
commandLine(
80+
pythonInterpreter,
81+
"-m",
82+
"pip",
83+
"uninstall",
84+
"utbot_mypy_runner"
85+
)
86+
commandLine(
87+
pythonInterpreter,
88+
"-m",
89+
"pip",
90+
"cache",
91+
"purge"
92+
)
93+
}
94+
} else {
95+
null
96+
}
97+
7598
val installMypyRunner =
7699
if (pythonInterpreter != null) {
77100
tasks.register<Exec>("installUtbotMypyRunner") {
@@ -96,6 +119,7 @@ val jsonDir = File(project.projectDir, "src/test/resources")
96119

97120
fun getParamsForSample(sampleName: String): List<String> =
98121
listOf(
122+
sampleName,
99123
File(samplesDir, "$sampleName.py").canonicalPath,
100124
sampleName,
101125
buildDir.canonicalPath,
@@ -110,12 +134,12 @@ val samplesInfo = listOf(
110134
)
111135

112136
if (pythonInterpreter != null) {
113-
val subtasks = samplesInfo.mapIndexed { index, params ->
114-
tasks.register<JavaExec>("regenerateJsonForTests_$index") {
137+
val subtasks = samplesInfo.map { params ->
138+
tasks.register<JavaExec>("regenerateJsonForTests_${params.first()}") {
115139
dependsOn(installMypyRunner!!)
116140
group = "python_subtasks"
117141
classpath = sourceSets.test.get().runtimeClasspath
118-
args = listOf(pythonInterpreter) + params
142+
args = listOf(pythonInterpreter) + params.drop(1)
119143
mainClass.set("org.utbot.python.newtyping.samples.GenerateMypyInfoBuildKt")
120144
}
121145
}

‎utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/MypyAnnotations.kt‎

Lines changed: 52 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
package org.utbot.python.newtyping.mypy
22

3-
import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory
43
import org.utbot.python.newtyping.*
54
import org.utbot.python.newtyping.general.*
5+
import org.utbot.python.utils.CustomPolymorphicJsonAdapterFactory
66

77
class MypyAnnotation(
88
val nodeId: String,
@@ -11,14 +11,24 @@ class MypyAnnotation(
1111
var initialized = false
1212
@Transient lateinit var storage: MypyInfoBuild
1313
val node: MypyAnnotationNode
14-
get() = storage.nodeStorage[nodeId]!!
14+
get() {
15+
val result = storage.nodeStorage[nodeId]
16+
require(result != null) {
17+
"Required node is absent in storage: $nodeId"
18+
}
19+
return result
20+
}
1521
val asUtBotType: UtType
1622
get() {
17-
assert(initialized)
23+
require(initialized)
1824
val origin = storage.getUtBotTypeOfNode(node)
25+
if (origin.pythonDescription() is PythonAnyTypeDescription)
26+
return origin
1927
if (args != null) {
20-
assert(origin.parameters.size == args.size)
21-
assert(origin.parameters.all { it is TypeParameter })
28+
require(origin.parameters.size == args.size) {
29+
"Bad arguments on ${origin.pythonTypeRepresentation()}. Expected ${origin.parameters.size} parameters but got ${args.size}"
30+
}
31+
require(origin.parameters.all { it is TypeParameter })
2232
val argTypes = args.map { it.asUtBotType }
2333
return DefaultSubstitutionProvider.substitute(
2434
origin,
@@ -47,7 +57,10 @@ sealed class CompositeAnnotationNode(
4757
fun getInitData(self: CompositeTypeCreator.Original): CompositeTypeCreator.InitializationData {
4858
storage.nodeToUtBotType[this] = self
4959
(typeVars zip self.parameters).forEach { (node, typeParam) ->
50-
val typeVar = node.node as TypeVarNode
60+
val typeVar = node.node as? TypeVarNode
61+
require(typeVar != null) {
62+
"Did not construct type variable"
63+
}
5164
storage.nodeToUtBotType[typeVar] = typeParam
5265
typeParam.meta = PythonTypeVarDescription(Name(emptyList(), typeVar.varName), typeVar.variance, typeVar.kind)
5366
typeParam.constraints = typeVar.constraints
@@ -70,7 +83,7 @@ class ConcreteAnnotation(
7083
val isAbstract: Boolean
7184
): CompositeAnnotationNode(module, simpleName, members, typeVars, bases) {
7285
override fun initializeType(): UtType {
73-
assert(storage.nodeToUtBotType[this] == null)
86+
require(storage.nodeToUtBotType[this] == null)
7487
return createPythonConcreteCompositeType(
7588
Name(module.split('.'), simpleName),
7689
typeVars.size,
@@ -117,7 +130,10 @@ class FunctionNode(
117130
) { self ->
118131
storage.nodeToUtBotType[this] = self
119132
(typeVars zip self.parameters).forEach { (nodeId, typeParam) ->
120-
val typeVar = storage.nodeStorage[nodeId] as TypeVarNode
133+
val typeVar = storage.nodeStorage[nodeId] as? TypeVarNode
134+
require(typeVar != null) {
135+
"Did not construct type variable"
136+
}
121137
storage.nodeToUtBotType[typeVar] = typeParam
122138
typeParam.meta = PythonTypeVarDescription(Name(emptyList(), typeVar.varName), typeVar.variance, typeVar.kind)
123139
typeParam.constraints = typeVar.constraints
@@ -140,9 +156,16 @@ class TypeVarNode(
140156
): MypyAnnotationNode() {
141157
override val children: List<MypyAnnotation>
142158
get() = super.children + values + (upperBound?.let { listOf(it) } ?: emptyList())
143-
override fun initializeType() =
144-
error("Initialization of TypeVar must be done in defining class or function." +
145-
" TypeVar name: $varName, def_id: $def")
159+
override fun initializeType(): UtType {
160+
/*error(
161+
"Initialization of TypeVar must be done in defining class or function." +
162+
" TypeVar name: $varName, def_id: $def"
163+
)*/
164+
// this a rare and bad case:
165+
// https://github.com/sqlalchemy/sqlalchemy/blob/rel_2_0_20/lib/sqlalchemy/sql/sqltypes.py#L2091C5-L2091C23
166+
storage.nodeStorage[def]!!.initializeType()
167+
return storage.nodeToUtBotType[this] ?: error("Error while initializing TypeVar name: $varName, def_id: $def")
168+
}
146169
val constraints: Set<TypeParameterConstraint> by lazy {
147170
val upperBoundConstraint: Set<TypeParameterConstraint> =
148171
upperBound?.let { setOf(TypeParameterConstraint(upperBoundRelation, it.asUtBotType)) } ?: emptySet()
@@ -236,28 +259,22 @@ enum class AnnotationType {
236259
Unknown
237260
}
238261

239-
val annotationAdapter: PolymorphicJsonAdapterFactory<MypyAnnotationNode> =
240-
PolymorphicJsonAdapterFactory.of(MypyAnnotationNode::class.java, "type")
241-
.withSubtype(ConcreteAnnotation::class.java, AnnotationType.Concrete.name)
242-
.withSubtype(Protocol::class.java, AnnotationType.Protocol.name)
243-
.withSubtype(TypeVarNode::class.java, AnnotationType.TypeVar.name)
244-
.withSubtype(OverloadedFunction::class.java, AnnotationType.Overloaded.name)
245-
.withSubtype(FunctionNode::class.java, AnnotationType.Function.name)
246-
.withSubtype(PythonAny::class.java, AnnotationType.Any.name)
247-
//.withSubtype(PythonLiteral::class.java, AnnotationType.Literal.name)
248-
.withSubtype(PythonUnion::class.java, AnnotationType.Union.name)
249-
.withSubtype(PythonTuple::class.java, AnnotationType.Tuple.name)
250-
.withSubtype(PythonNoneType::class.java, AnnotationType.NoneType.name)
251-
.withSubtype(TypeAliasNode::class.java, AnnotationType.TypeAlias.name)
252-
.withSubtype(UnknownAnnotationNode::class.java, AnnotationType.Unknown.name)
253-
254-
object MypyAnnotations {
255-
256-
data class MypyReportLine(
257-
val line: Int,
258-
val type: String,
259-
val message: String,
260-
val file: String
262+
val annotationAdapter = CustomPolymorphicJsonAdapterFactory(
263+
MypyAnnotationNode::class.java,
264+
contentLabel = "content",
265+
keyLabel = "type",
266+
mapOf(
267+
AnnotationType.Concrete.name to ConcreteAnnotation::class.java,
268+
AnnotationType.Protocol.name to Protocol::class.java,
269+
AnnotationType.TypeVar.name to TypeVarNode::class.java,
270+
AnnotationType.Overloaded.name to OverloadedFunction::class.java,
271+
AnnotationType.Function.name to FunctionNode::class.java,
272+
AnnotationType.Any.name to PythonAny::class.java,
273+
// .withSubtype(PythonLiteral::class.java, AnnotationType.Literal.name)
274+
AnnotationType.Union.name to PythonUnion::class.java,
275+
AnnotationType.Tuple.name to PythonTuple::class.java,
276+
AnnotationType.NoneType.name to PythonNoneType::class.java,
277+
AnnotationType.TypeAlias.name to TypeAliasNode::class.java,
278+
AnnotationType.Unknown.name to UnknownAnnotationNode::class.java
261279
)
262-
263-
}
280+
)

‎utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/MypyBuild.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ class ExpressionTypeFromMypy(
3333
class MypyInfoBuild(
3434
val nodeStorage: Map<String, MypyAnnotationNode>,
3535
val definitions: Map<String, Map<String, MypyDefinition>>,
36-
val types: Map<String, List<ExpressionTypeFromMypy>>,
36+
val exprTypes: Map<String, List<ExpressionTypeFromMypy>>,
3737
val names: Map<String, List<Name>>
3838
) {
3939
@Transient lateinit var buildRoot: MypyBuildDirectory
@@ -77,7 +77,7 @@ class MypyInfoBuild(
7777
fillArgNames(it.value)
7878
}
7979
}
80-
types.values.flatten().forEach {
80+
exprTypes.values.flatten().forEach {
8181
initAnnotation(it.type)
8282
}
8383
nodeStorage.values.forEach { node ->

‎utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/MypyDefinitions.kt‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package org.utbot.python.newtyping.mypy
22

3-
import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory
43
import org.utbot.python.newtyping.*
54
import org.utbot.python.newtyping.general.FunctionType
65
import org.utbot.python.newtyping.general.UtType
6+
import org.utbot.python.utils.CustomPolymorphicJsonAdapterFactory
77

88
sealed class MypyDefinition(val type: MypyAnnotation) {
99
fun getUtBotType(): UtType = type.asUtBotType
@@ -69,9 +69,14 @@ enum class MypyDefinitionLabel {
6969
OverloadedFuncDef
7070
}
7171

72-
val definitionAdapter: PolymorphicJsonAdapterFactory<MypyDefinition> =
73-
PolymorphicJsonAdapterFactory.of(MypyDefinition::class.java, "kind")
74-
.withSubtype(Variable::class.java, MypyDefinitionLabel.Variable.name)
75-
.withSubtype(ClassDef::class.java, MypyDefinitionLabel.ClassDef.name)
76-
.withSubtype(FuncDef::class.java, MypyDefinitionLabel.FuncDef.name)
77-
.withSubtype(OverloadedFuncDef::class.java, MypyDefinitionLabel.OverloadedFuncDef.name)
72+
val definitionAdapter = CustomPolymorphicJsonAdapterFactory(
73+
MypyDefinition::class.java,
74+
contentLabel = "content",
75+
keyLabel = "kind",
76+
mapOf(
77+
MypyDefinitionLabel.Variable.name to Variable::class.java,
78+
MypyDefinitionLabel.ClassDef.name to ClassDef::class.java,
79+
MypyDefinitionLabel.FuncDef.name to FuncDef::class.java,
80+
MypyDefinitionLabel.OverloadedFuncDef.name to OverloadedFuncDef::class.java
81+
)
82+
)

‎utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,10 @@ fun buildMypyInfo(
7676
val mypyExitStatus = if (mypyBuildDir.fileForMypyExitStatus.exists()) mypyBuildDir.fileForMypyExitStatus.readText() else null
7777
if (result.exitValue != 0 || mypyExitStatus != "0")
7878
error("Something went wrong in initial mypy run. " +
79-
"\nPython stderr ${result.stderr}" +
80-
"\nMypy stderr: $stderr" +
81-
"\nMypy stdout: $stdout")
79+
"\nPython stdout:\n${result.stdout}" +
80+
"\nPython stderr:\n${result.stderr}" +
81+
"\nMypy stderr:\n$stderr" +
82+
"\nMypy stdout:\n$stdout")
8283
}
8384

8485
fun readMypyAnnotationStorageAndInitialErrors(

‎utbot-python-types/src/main/kotlin/org/utbot/python/utils/CustomJsonAdapter.kt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package org.utbot.python.utils
2+
3+
import com.squareup.moshi.*
4+
import com.squareup.moshi.JsonAdapter.Factory
5+
import java.lang.reflect.Type
6+
7+
class CustomPolymorphicJsonAdapterFactory(
8+
private val baseType: Type,
9+
contentLabel: String,
10+
keyLabel: String,
11+
private val elementAdapters: Map<String, Type>
12+
): Factory {
13+
private val contentOption = JsonReader.Options.of(contentLabel)
14+
private val keyLabelOption = JsonReader.Options.of(keyLabel)
15+
private val labels = elementAdapters.keys.toList()
16+
private val labelOptions = JsonReader.Options.of(*labels.toTypedArray<String>())
17+
18+
class CustomPolymorphicJsonAdapter(
19+
private val contentOption: JsonReader.Options,
20+
private val keyLabelOption: JsonReader.Options,
21+
private val adapters: List<JsonAdapter<Any>>,
22+
private val labelOptions: JsonReader.Options
23+
): JsonAdapter<Any>() {
24+
override fun fromJson(reader: JsonReader): Any? {
25+
reader.beginObject()
26+
27+
val index = reader.selectName(keyLabelOption)
28+
if (index == -1)
29+
return null
30+
31+
val labelIndex = reader.selectString(labelOptions)
32+
if (labelIndex == -1)
33+
return null
34+
35+
val contentIndex = reader.selectName(contentOption)
36+
if (contentIndex == -1)
37+
return null
38+
39+
val result = adapters[labelIndex].fromJson(reader)
40+
reader.endObject()
41+
42+
return result
43+
}
44+
45+
override fun toJson(writer: JsonWriter, value: Any?) {
46+
error("Writing with this Json adapter is not supported")
47+
}
48+
}
49+
50+
override fun create(type: Type, annotations: MutableSet<out Annotation>, moshi: Moshi): JsonAdapter<*>? {
51+
if (Types.getRawType(type) != baseType || annotations.isNotEmpty()) {
52+
return null
53+
}
54+
val adapters: List<JsonAdapter<Any>> = labels.map { moshi.adapter(elementAdapters[it]!!) }
55+
return CustomPolymorphicJsonAdapter(contentOption, keyLabelOption, adapters, labelOptions)
56+
}
57+
}

‎utbot-python-types/src/main/python/utbot_mypy_runner/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
mypy_config.ini
2+
out.json
3+
.vscode/

‎utbot-python-types/src/main/python/utbot_mypy_runner/pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
[tool.poetry]
22
name = "utbot_mypy_runner"
3-
version = "0.2.11"
3+
version = "0.2.15"
44
description = ""
55
authors = ["Ekaterina Tochilina <katerina_t_n@mail.ru>"]
66
readme = "README.md"
77
packages = [{include = "utbot_mypy_runner"}]
88

99
[tool.poetry.dependencies]
1010
python = "^3.8"
11-
mypy = "1.0.0"
11+
mypy = "1.5.1"
1212

1313

1414
[build-system]

‎utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/__main__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,4 +51,4 @@
5151
print("Extracted annotations and wrote to", args.annotations_out)
5252
else:
5353
print("For some reason BuildResult is None")
54-
exit(11)
54+
exit(11)

0 commit comments

Comments
 (0)