New seed selection strategy and time budget for generation

Author: DaniilStepanov

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -537,7 +537,12 @@ class UtBotSymbolicEngine(
             }
 
             try {
-                GreyBoxFuzzer(concreteExecutor.pathsToUserClasses, concreteExecutor.pathsToDependencyClasses, methodUnderTest).fuzz()
+                GreyBoxFuzzer(
+                    concreteExecutor.pathsToUserClasses,
+                    concreteExecutor.pathsToDependencyClasses,
+                    methodUnderTest,
+                    120_000L
+                ).fuzz()
             } catch (e: CancellationException) {
                 logger.debug { "Cancelled by timeout" }
             } catch (e: Throwable) {

utbot-framework/src/main/kotlin/org/utbot/engine/greyboxfuzzer/GreyBoxFuzzer.kt

@@ -10,56 +10,70 @@ import org.utbot.external.api.classIdForType
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.UtModel
 import org.utbot.framework.plugin.api.UtNullModel
+import org.utbot.framework.plugin.api.util.id
 import org.utbot.framework.plugin.api.util.jClass
+import org.utbot.framework.plugin.api.util.objectClassId
+import org.utbot.quickcheck.generator.GenerationState
+import org.utbot.quickcheck.generator.GeneratorContext
 import java.lang.reflect.Parameter
 
 object DataGenerator {
 
     private val generatorRepository = GreyBoxFuzzerGenerators.generatorRepository
 
-    fun generate(
-        clazz: Class<*>,
-        random: SourceOfRandomness,
-        status: GenerationStatus
-    ): UtModel? = generatorRepository.getOrProduceGenerator(clazz)?.generateImpl(random, status)
-
-    fun generate(
+    fun generateUtModel(
         parameterTypeContext: ParameterTypeContext,
+        depth: Int = 0,
+        generatorContext: GeneratorContext,
         random: SourceOfRandomness,
         status: GenerationStatus
-    ): UtModel? = generatorRepository.getOrProduceGenerator(parameterTypeContext, 0)?.generateImpl(random, status)
-
-    fun generate(
-        parameterTypeContext: ParameterTypeContext,
-        random: SourceOfRandomness,
-        status: GenerationStatus,
-        depth: Int
-    ): UtModel? = generatorRepository.getOrProduceGenerator(parameterTypeContext, depth)?.generateImpl(random, status)
+    ): UtModel {
+        logger.debug { "Trying to generate UtModel of type 3 times" }
+        val classId = parameterTypeContext.rawClass.id
+        var generatedInstance: UtModel? = null
+        repeat(3) {
+            generatedInstance =
+                try {
+                    val generator =
+                        generatorRepository.getOrProduceGenerator(parameterTypeContext, generatorContext, depth)
+                            ?: return@repeat
+                    generator.generatorContext.startCheckpoint()
+                    generator.generateImpl(random, status)
+                } catch (_: Throwable) {
+                    null
+                }
+            generatedInstance?.let { if (it !is UtNullModel) return it }
+        }
+        return UtNullModel(classId)
+    }
 
     fun generate(
         parameter: Parameter,
         parameterIndex: Int,
+        generatorContext: GeneratorContext,
         random: SourceOfRandomness,
         status: GenerationStatus
     ): FParameter {
         val generator =
-            generatorRepository.getOrProduceGenerator(parameter, parameterIndex)
+            generatorRepository.getOrProduceGenerator(parameter, parameterIndex, generatorContext)
         return generate(generator, parameter, random, status)
     }
 
     fun generateThis(
         classId: ClassId,
+        generatorContext: GeneratorContext,
         random: SourceOfRandomness,
         status: GenerationStatus
     ): NormalMethodThisInstance {
         val generator =
-            generatorRepository.getOrProduceGenerator(classId.jClass)
-        return generateThis(generator, classId, random, status)
+            generatorRepository.getOrProduceGenerator(classId.jClass, generatorContext)
+        return generateThis(generator, classId, generatorContext, random, status)
     }
 
     private fun generateThis(
         generator: Generator?,
         classId: ClassId,
+        generatorContext: GeneratorContext,
         random: SourceOfRandomness,
         status: GenerationStatus,
         numberOfTries: Int = 3
@@ -70,10 +84,13 @@ object DataGenerator {
             throw FuzzerIllegalStateException("Can't find generator for ${classId.name}")
         }
         var generatedValue: UtModel
-        repeat(numberOfTries) {
-            logger.debug { "Try $it" }
+        repeat(numberOfTries) { iteration ->
+            logger.debug { "Try $iteration" }
             try {
+                generator.generationState = GenerationState.REGENERATE
+                generator.generatorContext.startCheckpoint()
                 generatedValue = generator.generateImpl(random, status)
+                if (generatedValue is UtNullModel && iteration != numberOfTries - 1) return@repeat
                 return NormalMethodThisInstance(
                     generatedValue,
                     generator,
@@ -104,7 +121,10 @@ object DataGenerator {
         repeat(numberOfTries) {
             logger.debug { "Try $it" }
             try {
+                generator.generationState = GenerationState.REGENERATE
+                generator.generatorContext.startCheckpoint()
                 generatedValue = generator.generateImpl(random, status)
+                if (generatedValue is UtNullModel) return@repeat
                 return FParameter(
                     parameter,
                     null,
@@ -120,158 +140,4 @@ object DataGenerator {
         return FParameter(parameter, null, UtNullModel(classId), generator, classId, listOf())
     }
 
-//    //TODO Make it work with type parameters
-//    private fun Type.getFFieldsForClass(value: Any, depth: Int, originalField: Field?): List<FField> {
-//        println("GETTING FFIelds from $value")
-//        createFFieldFromPrimitivesOrBoxedPrimitives(this, value, originalField)?.let { return listOf(it) }
-//        val parameterizedType = this as? ParameterizedType
-//        val genericsContext =
-//            if (this is GenericArrayTypeImpl) {
-//                (this.genericComponentType as? ParameterizedType)?.buildGenericsContext()
-//            } else {
-//                parameterizedType?.buildGenericsContext()
-//            }
-//        if (depth >= GreyBoxFuzzerGenerators.maxDepthOfGeneration) {
-//            return emptyList()
-//        }
-//        val subFields = mutableListOf<FField>()
-//        if (this.toClass()?.isArray == true) {
-//            val arrayContentType = this.toClass()?.componentType ?: return subFields
-//            getFFieldsFromArray(value, subFields, originalField, this, arrayContentType, depth)
-//            return subFields
-//        }
-//        val classFields =
-//            this.toClass()?.getAllDeclaredFields()?.filter { !it.hasModifiers(Modifier.FINAL) } ?: emptyList()
-//        for (field in classFields) {
-//            val resolvedFieldType =
-//                if (genericsContext != null) {
-//                    //TODO make it work for subclasses
-//                    parameterizedType.let { field.resolveFieldType(genericsContext) } ?: field.type
-//                } else {
-//                    field.type
-//                }
-//            assert(resolvedFieldType.toClass() != null)
-////            if (field.hasModifiers(Modifier.FINAL)) {
-////                //subFields.add(FField(field, value))
-////                continue
-////            }
-//            if (resolvedFieldType.toClass()!!.isArray) {
-//                val arrayOfObjects = field.getFieldValue(value)
-//                val arrayContentType =
-//                    (resolvedFieldType as? GenericArrayTypeImpl)?.genericComponentType ?: field.type.componentType
-//                getFFieldsFromArray(arrayOfObjects, subFields, field, resolvedFieldType, arrayContentType, depth)
-//                //TODO!!!!
-//            } else {
-//                field.getFieldValue(value)?.let { fieldValue ->
-//                    try {
-//                        val generatorForField = generatorRepository.getOrProduceGenerator(field)
-//                        if (field.type.isPrimitive) {
-//                            subFields.add(FField(field, fieldValue, resolvedFieldType, generatorForField))
-//                        } else {
-//                            //println("GETTING SUBFIELDS FOR ${field.type} value = ${fieldValue} DEPTH = $depth")
-//                            //TODO resolve type
-//                            val subFFields = resolvedFieldType.getFFieldsForClass(fieldValue, depth + 1, null)
-//                            subFields.add(FField(field, fieldValue, resolvedFieldType, generatorForField, subFFields))
-//                        }
-//                    } catch (e: java.lang.IllegalStateException) {
-//                        subFields.add(FField(field, fieldValue, resolvedFieldType, null, listOf()))
-//                    }
-//                } ?: subFields.add(FField(field, null, resolvedFieldType, null, listOf()))
-//            }
-//        }
-//        return subFields
-//    }
-
-//    private fun createFFieldFromPrimitivesOrBoxedPrimitives(originalType: Type, value: Any?, field: Field?): FField? {
-//        val clazz = originalType.toClass() ?: return null
-//        val listOfPrimitives = listOf(
-//            Byte::class,
-//            Short::class,
-//            Int::class,
-//            Long::class,
-//            Float::class,
-//            Double::class,
-//            Boolean::class,
-//            Char::class,
-//            String::class
-//        )
-//        return if (clazz.kotlin in listOfPrimitives || clazz.isPrimitive) {
-//            FField(field, value, originalType, getGenerator(originalType))
-//        } else null
-//    }
-
-//    private fun getFFieldsFromArray(
-//        array: Any?,
-//        subFields: MutableList<FField>,
-//        field: Field?,
-//        arrayType: Type,
-//        arrayContentType: Type,
-//        depth: Int
-//    ) {
-//        val typedArray =
-//            when (array) {
-//                is BooleanArray -> {
-//                    array.toList()
-//                }
-//                is ByteArray -> {
-//                    array.toList()
-//                }
-//                is CharArray -> {
-//                    array.toList()
-//                }
-//                is ShortArray -> {
-//                    array.toList()
-//                }
-//                is IntArray -> {
-//                    array.toList()
-//                }
-//                is LongArray -> {
-//                    array.toList()
-//                }
-//                is FloatArray -> {
-//                    array.toList()
-//                }
-//                is DoubleArray -> {
-//                    array.toList()
-//                }
-//                else -> {
-//                    if (array == null) {
-//                        subFields.add(FField(null, null, arrayContentType, null, listOf()))
-//                        return
-//                    } else {
-//                        (array as Array<*>).toList()
-//                    }
-//                }
-//            }
-//        val generatorOfNeededType = field?.let { getGenerator(it, arrayType) } ?: getGenerator(arrayType)
-//        val localSubFields = mutableListOf<FField>()
-//        val indexOfLastNotNullElement = typedArray.indexOfLast { it != null }
-//        val arrayContentGenerator = getGenerator(arrayContentType)
-//        if (indexOfLastNotNullElement == -1) {
-//            localSubFields.add(FField(field, null, arrayContentType, arrayContentGenerator))
-//        } else {
-//            typedArray.filterNotNull().map { el ->
-//                val ssFFields = arrayContentType.getFFieldsForClass(el, depth + 1, null)
-//                localSubFields.add(FField(field, el, arrayContentType, arrayContentGenerator, ssFFields))
-//            }
-//        }
-//        subFields.add(FField(field, typedArray, arrayType, generatorOfNeededType, localSubFields))
-//    }
-
-//    private fun getGenerator(field: Field, fieldType: Type): Generator<*>? {
-//        return if (fieldType is ParameterizedType) {
-//            generatorRepository.getOrProduceGenerator(field.buildParameterContext(fieldType), 0)
-//        } else {
-//            generatorRepository.getOrProduceGenerator(field)
-//        }.let { gen ->
-//            if (gen is ComponentizedGenerator && gen.getComponents().any { it is ZilchGenerator }) null
-//            else gen
-//        }
-//    }
-//    private fun getGenerator(resolvedType: Type): Generator<*>? =
-//        generatorRepository.getOrProduceGenerator(resolvedType).let { gen ->
-//            if (gen is ComponentizedGenerator && gen.getComponents().any { it is ZilchGenerator }) null
-//            else gen
-//        }
-
 }

utbot-framework/src/main/kotlin/org/utbot/engine/greyboxfuzzer/generator/DataGenerator.kt

@@ -50,16 +50,6 @@ data class FParameter(
         return res
     }
 
-    fun regenerateFields() {
-        regenerateFields(classId.jClass.getAllDeclaredFields())
-    }
-
-    fun regenerateFields(fieldsToRegenerate: List<Field>) {
-        if (utModel is UtAssembleModel) {
-            utModel = Mutator.regenerateFields(classId.jClass, utModel as UtAssembleModel, fieldsToRegenerate)
-        }
-    }
-
     fun copy(): FParameter = FParameter(
         parameter,
         value,

utbot-framework/src/main/kotlin/org/utbot/engine/greyboxfuzzer/generator/FParameter.kt

@@ -26,7 +26,7 @@ object GeneratorConfigurator {
     private val minDouble: Double = -100.0
     private val maxDouble: Double = 100.0
     private val minStringLength: Int = 1
-    private val maxStringLength: Int = 10
+    private val maxStringLength: Int = 5
     val minCollectionSize: Int = 1
     val maxCollectionSize: Int = 5
 
@@ -92,10 +92,14 @@ object GeneratorConfigurator {
             is MapGenerator -> generator.configure(sizeAnnotationInstance)
             is StringGenerator -> generator.configure(
                 if (Random.getTrue(50)) {
-                    setOf('a'.code..'z'.code)
+                    if (Random.getTrue(50)) {
+                        setOf('a'.code..'z'.code)
+                    } else {
+                        setOf('A'.code..'Z'.code)
+                    }
                 } else {
                     setOf(' '.code..'~'.code)
-                }
+                }, minStringLength..maxStringLength
             )
             else -> Unit
         }

utbot-framework/src/main/kotlin/org/utbot/engine/greyboxfuzzer/generator/GeneratorConfigurator.kt

@@ -14,7 +14,7 @@ import java.util.*
 object GreyBoxFuzzerGenerators {
 
     const val seed = 42L
-    const val maxDepthOfGeneration = 4
+    const val maxDepthOfGeneration = 7
     val sourceOfRandomness = SourceOfRandomness(Random(seed))
     val genStatus = NonTrackingGenerationStatus(sourceOfRandomness)
 
@@ -41,7 +41,6 @@ object GreyBoxFuzzerGenerators {
             it.register(ShortGenerator())
             it.register(BooleanGenerator())
             it.register(IntegerGenerator())
-            it.register(Encoded())
             it.register(ByteGenerator())
             it.register(StringGenerator())
             it.register(LongGenerator())