provide fuzzing ratio property

reformat serialization
UnitTestBot · SBOne-Kenobi · Nov 9, 2022 · Oct 24, 2022 · Nov 2, 2022 · Nov 2, 2022
commit a494d22a26ca36c77c39d6b0f8c86f07c53b14ed
diff --git a/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt b/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt
@@ -128,6 +128,9 @@ abstract class AbstractSettings(
     protected fun getStringProperty(defaultValue: String) = getProperty(defaultValue) { it }
     protected inline fun <reified T : Enum<T>> getEnumProperty(defaultValue: T) =
         getProperty(defaultValue) { enumValueOf(it) }
-
+    protected fun getListProperty(defaultValue: List<String>) =
+        getProperty(defaultValue) { it.split(';') }
+    protected inline fun <reified T> getListProperty(defaultValue: List<T>, crossinline elementTransform: (String) -> T) =
+        getProperty(defaultValue) { it.split(';').map(elementTransform) }
     override fun toString(): String = container.toString()
 }
diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle
@@ -1,3 +1,6 @@
+plugins {
+    id 'org.jetbrains.kotlin.plugin.serialization' version '1.7.20'
+}
 apply plugin: 'jacoco'
 
 configurations {
@@ -58,6 +61,7 @@ dependencies {
     implementation group: 'org.apache.commons', name: 'commons-exec', version: '1.2'
     implementation group: 'io.github.microutils', name: 'kotlin-logging', version: kotlinLoggingVersion
     implementation group: 'org.jsoup', name: 'jsoup', version: '1.6.2'
+    implementation 'org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1'
     // need for tests
     implementation group: 'org.mockito', name: 'mockito-core', version: '4.2.0'
     implementation group: 'org.mockito', name: 'mockito-inline', version: '4.2.0'

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
@@ -138,7 +138,15 @@ fun main(args: Array<String>) {
             val timeBudgetSec = cmd[2].toLong()
             val cut = ClassUnderTest(classLoader.loadClass(classUnderTestName).id, outputDir, classfileDir.toFile())
 
-            runGeneration(cut, timeBudgetSec, classpathString, runFromEstimator = false, methodNameFilter = null)
+            runGeneration(
+                project = "Contest",
+                cut,
+                timeBudgetSec,
+                fuzzingRatio = 0.1,
+                classpathString,
+                runFromEstimator = false,
+                methodNameFilter = null
+            )
             println("${ContestMessage.READY}")
         }
     }
@@ -162,8 +170,10 @@ fun setOptions() {
 @ObsoleteCoroutinesApi
 @SuppressWarnings
 fun runGeneration(
+    project: String,
     cut: ClassUnderTest,
     timeLimitSec: Long,
+    fuzzingRatio: Double,
     classpathString: String,
     runFromEstimator: Boolean,
     methodNameFilter: String? = null // For debug purposes you can specify method name
@@ -205,7 +215,7 @@ fun runGeneration(
         logger.error("Seems like classloader for cut not valid (maybe it was backported to system): ${cut.classLoader}")
     }
 
-    val statsForClass = StatsForClass()
+    val statsForClass = StatsForClass(project, cut.fqn)
 
     val codeGenerator = CodeGenerator(
             cut.classId,
@@ -276,6 +286,8 @@ fun runGeneration(
 
                     val budgetForSymbolicExecution = max(0, budgetForMethod - budgetForLastSolverRequestAndConcreteExecutionRemainingStates)
 
+                    UtSettings.utBotGenerationTimeoutInMillis = budgetForMethod
+                    UtSettings.fuzzingTimeoutInMillis = (budgetForMethod * fuzzingRatio).toLong()
 
                     //start controller that will activate symbolic execution
                     GlobalScope.launch {

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
@@ -83,6 +83,7 @@ enum class Tool {
             project: ProjectToEstimate,
             cut: ClassUnderTest,
             timeLimit: Long,
+            fuzzingRatio: Double,
             methodNameFilter: String?,
             globalStats: GlobalStats,
             compiledTestDir: File,
@@ -91,8 +92,10 @@ enum class Tool {
             val classStats: StatsForClass = try {
                 withUtContext(UtContext(project.classloader)) {
                     runGeneration(
+                        project.name,
                         cut,
                         timeLimit,
+                        fuzzingRatio,
                         project.sootClasspathString,
                         runFromEstimator = true,
                         methodNameFilter
@@ -167,6 +170,7 @@ enum class Tool {
             project: ProjectToEstimate,
             cut: ClassUnderTest,
             timeLimit: Long,
+            fuzzingRatio: Double,
             methodNameFilter: String?,
             globalStats: GlobalStats,
             compiledTestDir: File,
@@ -237,6 +241,7 @@ enum class Tool {
         project: ProjectToEstimate,
         cut: ClassUnderTest,
         timeLimit: Long,
+        fuzzingRatio: Double, // maybe create some specific settings
         methodNameFilter: String?,
         globalStats: GlobalStats,
         compiledTestDir: File,
@@ -257,6 +262,7 @@ fun main(args: Array<String>) {
     if (args.isEmpty() && System.getProperty("os.name")?.run { contains("win", ignoreCase = true) } == true) {
         processedClassesThreshold = 9999 //change to change number of classes to run
         val timeLimit = 20 // increase if you want to debug something
+        val fuzzingRatio = 0.1 // sets fuzzing ratio to total test generation
 
         // Uncomment it for debug purposes:
         // you can specify method for test generation in format `classFqn.methodName`
@@ -283,12 +289,13 @@ fun main(args: Array<String>) {
             classesLists,
             jarsDir,
             "$timeLimit",
+            "$fuzzingRatio",
             outputDir,
             moduleTestDir
         )
     } else {
-        require(args.size == 6) {
-            "Wrong arguments: <classes dir> <classpath_dir> <time limit (s)> <output dir> <test dir> <junit jar path> expected, but got: ${args.toText()}"
+        require(args.size == 7) {
+            "Wrong arguments: <classes dir> <classpath_dir> <time limit (s)> <fuzzing ratio> <output dir> <test dir> <junit jar path> expected, but got: ${args.toText()}"
         }
         logger.info { "Command line: [${args.joinToString(" ")}]" }
 
@@ -315,8 +322,9 @@ fun runEstimator(
     val classesLists = File(args[0])
     val classpathDir = File(args[1])
     val timeLimit = args[2].toLong()
-    val outputDir = File(args[3])
-    // we don't use it: val moduleTestDir = File(args[4])
+    val fuzzingRatio = args[3].toDouble()
+    val outputDir = File(args[4])
+    // we don't use it: val moduleTestDir = File(args[5])
 
     val testCandidatesDir = File(outputDir, "test_candidates")
     val compiledTestDir = File(outputDir, "compiled")
@@ -411,7 +419,7 @@ fun runEstimator(
 
                     logger.info { "------------- [${project.name}] ---->--- [$classIndex:$classFqn] ---------------------" }
 
-                    tool.run(project, cut, timeLimit, methodNameFilter, globalStats, compiledTestDir, classFqn)
+                    tool.run(project, cut, timeLimit, fuzzingRatio, methodNameFilter, globalStats, compiledTestDir, classFqn)
                 }
             }
         }
@@ -423,10 +431,6 @@ fun runEstimator(
     logger.info { globalStats }
     ConcreteExecutor.defaultPool.close()
 
-//    For what?
-//    if (globalStats.statsForClasses.isNotEmpty())
-//        exitProcess(1)
-
     return globalStats
 }
 

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
@@ -109,7 +109,7 @@ class GlobalStats {
             avgCoverage.format(PRECISION) + " %"
 }
 
-class StatsForClass {
+class StatsForClass(val project: String, val className: String) {
     val testedClassNames: MutableSet<String> = ConcurrentSkipListSet()
 
     var methodsCount: Int = -1
@@ -126,16 +126,16 @@ class StatsForClass {
     var fuzzedCoverage = CoverageInstructionsSet()
     var concolicCoverage = CoverageInstructionsSet()
 
-    /**
-     * Add class [className] to respect coverage from this class.
-     */
-    fun addTestedClass(className: String) {
-        testedClassNames.add(className)
-    }
-
     private fun CoverageInstructionsSet.prettyInfo(): String =
         getCoverageInfo(testedClassNames).run { "$covered/$total" }
 
+    fun getCoverageInfo(): CoverageStatistic =
+        coverage.getCoverageInfo(testedClassNames)
+    fun getFuzzedCoverageInfo(): CoverageStatistic =
+        fuzzedCoverage.getCoverageInfo(testedClassNames)
+    fun getConcolicCoverageInfo(): CoverageStatistic =
+        concolicCoverage.getCoverageInfo(testedClassNames)
+
     override fun toString(): String = "\n<StatsForClass> :" +
             "\n\tcanceled by timeout = $canceledByTimeout" +
             "\n\t#methods = $methodsCount, " +

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt
@@ -0,0 +1,118 @@
+package org.utbot.monitoring
+
+import kotlinx.serialization.Serializable
+import org.utbot.contest.GlobalStats
+import org.utbot.contest.StatsForClass
+
+@Serializable
+data class MonitoringReport(
+    val parameters: MonitoringParameters,
+    val summarised_metrics: SummarisedMetricsReport,
+    val targets: List<TargetReport>,
+) {
+    constructor(parameters: MonitoringParameters, targets: List<TargetReport>): this(
+        parameters,
+        SummarisedMetricsReport(targets),
+        targets
+    )
+
+    constructor(parameters: MonitoringParameters, stats: GlobalStats) : this(
+        parameters,
+        stats.statsForClasses.map {
+            TargetReport(
+                TargetDescription(it.project, it.className),
+                ClassMetricsReport(it)
+            )
+        }
+    )
+}
+
+@Serializable
+data class MonitoringParameters(
+    val fuzzing_ratio: Double,
+    val class_timeout_sec: Int,
+    val run_timeout_min: Int,
+)
+
+@Serializable
+data class TargetReport(
+    val target: TargetDescription,
+    val metrics: ClassMetricsReport
+)
+
+@Serializable
+data class TargetDescription(
+    val project: String,
+    val classname: String
+)
+
+private fun Int.cover(total: Int): Double =
+    if (total == 0) 0.0 else this.toDouble() / total
+
+@Serializable
+data class ClassMetricsReport(
+    val testcases_generated: Int,
+    val failed_to_compile: Boolean,
+    val canceled_by_timeout: Boolean,
+    val total_methods_in_class: Int,
+    val methods_with_at_least_one_testcase_generated: Int,
+    val methods_with_at_least_one_exception: Int,
+    val methods_without_any_tests_and_exceptions: Int,
+    val covered_bytecode_instructions_in_class: Int,
+    val covered_bytecode_instructions_in_class_by_fuzzing: Int,
+    val covered_bytecode_instructions_in_class_by_concolic: Int,
+    val total_bytecode_instructions_in_class: Int,
+    val bytecode_instructions_coverage_in_class: Double = covered_bytecode_instructions_in_class.cover(total_bytecode_instructions_in_class),
+    val bytecode_instructions_coverage_in_class_by_fuzzing: Double = covered_bytecode_instructions_in_class_by_fuzzing.cover(total_bytecode_instructions_in_class),
+    val bytecode_instructions_coverage_in_class_by_concolic: Double = covered_bytecode_instructions_in_class_by_concolic.cover(total_bytecode_instructions_in_class)
+) {
+    constructor(statsForClass: StatsForClass) : this(
+        testcases_generated = statsForClass.testcasesGenerated,
+        failed_to_compile = statsForClass.failedToCompile,
+        canceled_by_timeout = statsForClass.canceledByTimeout,
+        total_methods_in_class = statsForClass.methodsCount,
+        methods_with_at_least_one_testcase_generated = statsForClass.statsForMethods.count { it.testsGeneratedCount > 0 },
+        methods_with_at_least_one_exception = statsForClass.methodsWithAtLeastOneException,
+        methods_without_any_tests_and_exceptions = statsForClass.statsForMethods.count { it.isSuspicious },
+        covered_bytecode_instructions_in_class = statsForClass.getCoverageInfo().covered,
+        covered_bytecode_instructions_in_class_by_fuzzing = statsForClass.getFuzzedCoverageInfo().covered,
+        covered_bytecode_instructions_in_class_by_concolic = statsForClass.getConcolicCoverageInfo().covered,
+        total_bytecode_instructions_in_class = statsForClass.coverage.totalInstructions.toInt()
+    )
+}
+
+@Serializable
+data class SummarisedMetricsReport(
+    val total_classes: Int,
+    val testcases_generated: Int,
+    val classes_failed_to_compile: Int,
+    val classes_canceled_by_timeout: Int,
+    val total_methods: Int,
+    val methods_with_at_least_one_testcase_generated: Int,
+    val methods_with_at_least_one_exception: Int,
+    val methods_without_any_tests_and_exceptions: Int,
+    val covered_bytecode_instructions: Int,
+    val covered_bytecode_instructions_by_fuzzing: Int,
+    val covered_bytecode_instructions_by_concolic: Int,
+    val total_bytecode_instructions: Int,
+    val bytecode_instructions_coverage: Double = covered_bytecode_instructions.cover(total_bytecode_instructions),
+    val bytecode_instructions_coverage_by_fuzzing: Double = covered_bytecode_instructions_by_fuzzing.cover(total_bytecode_instructions),
+    val bytecode_instructions_coverage_by_concolic: Double = covered_bytecode_instructions_by_concolic.cover(total_bytecode_instructions),
+    val averaged_bytecode_instruction_coverage_by_classes: Double
+) {
+    constructor(targets: Collection<TargetReport>) : this(
+        total_classes = targets.size,
+        testcases_generated = targets.sumOf { it.metrics.testcases_generated },
+        classes_failed_to_compile = targets.count { it.metrics.failed_to_compile },
+        classes_canceled_by_timeout = targets.count { it.metrics.canceled_by_timeout },
+        total_methods = targets.sumOf { it.metrics.total_methods_in_class },
+        methods_with_at_least_one_testcase_generated = targets.sumOf { it.metrics.methods_with_at_least_one_testcase_generated },
+        methods_with_at_least_one_exception = targets.sumOf { it.metrics.methods_with_at_least_one_exception },
+        methods_without_any_tests_and_exceptions = targets.sumOf { it.metrics.methods_without_any_tests_and_exceptions },
+        covered_bytecode_instructions = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class },
+        covered_bytecode_instructions_by_fuzzing = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_fuzzing },
+        covered_bytecode_instructions_by_concolic = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_concolic },
+        total_bytecode_instructions = targets.sumOf { it.metrics.total_bytecode_instructions_in_class },
+        averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average()
+    )
+}
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt
@@ -22,19 +22,17 @@ object MonitoringSettings : AbstractSettings(
     val processedClassesThreshold by getIntProperty(9999)
 
     /**
-     * Number of running test generation.
+     * One running generation for all projects timeout in minutes.
      */
-    val runTries by getIntProperty(1)
+    val runTimeoutMinutes by getIntProperty(180)
 
     /**
-     * One running generation for all projects timeout in minutes.
+     * Target projects to generate tests.
      */
-    val runTimeoutMinutes by getIntProperty(180)
+    val projects by getListProperty(listOf("guava"))
 
     /**
-     * Target project to generate tests.
-     *
-     * TODO: change it into list and group GlobalStats by projects.
+     * Set up fuzzing timeout relatively total generation time.
      */
-    val project by getStringProperty("guava")
+    val fuzzingRatios by getListProperty(listOf(0.1), String::toDouble)
 }