diff --git a/core/src/main/kotlin/by/jprof/telegram/bot/core/UpdateProcessingPipeline.kt b/core/src/main/kotlin/by/jprof/telegram/bot/core/UpdateProcessingPipeline.kt index 25747ce1..c965b08b 100644 --- a/core/src/main/kotlin/by/jprof/telegram/bot/core/UpdateProcessingPipeline.kt +++ b/core/src/main/kotlin/by/jprof/telegram/bot/core/UpdateProcessingPipeline.kt @@ -2,8 +2,9 @@ package by.jprof.telegram.bot.core import dev.inmo.tgbotapi.types.update.abstracts.Update import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.joinAll -import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.supervisorScope import org.apache.logging.log4j.LogManager diff --git a/eval/build.gradle.kts b/eval/build.gradle.kts index f707b713..21702c9b 100644 --- a/eval/build.gradle.kts +++ b/eval/build.gradle.kts @@ -4,6 +4,7 @@ plugins { dependencies { api(project.projects.core) + api(project.projects.eval.dto) api(libs.tgbotapi.core) implementation(project.projects.votes.tgbotapiExtensions) implementation(libs.tgbotapi.extensions.api) diff --git a/eval/dto/build.gradle.kts b/eval/dto/build.gradle.kts new file mode 100644 index 00000000..d6aa50b9 --- /dev/null +++ b/eval/dto/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + implementation(libs.kotlinx.serialization.core) +} diff --git a/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalEvent.kt b/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalEvent.kt new file mode 100644 index 00000000..fcfeb443 --- /dev/null +++ b/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalEvent.kt @@ -0,0 +1,8 @@ +package by.jprof.telegram.bot.eval.dto + +import kotlinx.serialization.Serializable + +@Serializable +data class EvalEvent( + val code: String, +) diff --git a/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalResponse.kt b/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalResponse.kt new file mode 100644 index 00000000..1e13be6c --- /dev/null +++ b/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalResponse.kt @@ -0,0 +1,16 @@ +package by.jprof.telegram.bot.eval.dto + +import kotlinx.serialization.Serializable + +@Serializable +sealed class EvalResponse { + @Serializable + data class Successful( + val language: Language, + val stdout: String? = null, + val stderr: String? = null, + ) : EvalResponse() + + @Serializable + object Unsuccessful : EvalResponse() +} diff --git a/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/Language.kt b/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/Language.kt new file mode 100644 index 00000000..067f6013 --- /dev/null +++ b/eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/Language.kt @@ -0,0 +1,10 @@ +package by.jprof.telegram.bot.eval.dto + +enum class Language { + KOTLIN, + JAVA, + JAVASCRIPT, + TYPESCRIPT, + PYTHON, + GO, +} diff --git a/eval/evaluator/Dockerfile b/eval/evaluator/Dockerfile new file mode 100644 index 00000000..5ac05b82 --- /dev/null +++ b/eval/evaluator/Dockerfile @@ -0,0 +1,36 @@ +FROM amazon/aws-lambda-java:latest + +RUN yum -y install unzip tar gzip xz && \ + yum -y clean all && \ + rm -rf /var/cache + +RUN curl -L https://github.com/JetBrains/kotlin/releases/download/v1.5.31/kotlin-compiler-1.5.31.zip --output /tmp/kotlin.zip --silent && \ + unzip /tmp/kotlin.zip -d /tmp && \ + mv /tmp/kotlinc /opt/kotlin && \ + rm /tmp/kotlin.zip + +RUN curl -L https://download.java.net/java/GA/jdk17/0d483333a00540d886896bac774ff48b/35/GPL/openjdk-17_linux-x64_bin.tar.gz --output /tmp/java.tar.gz --silent && \ + mkdir /opt/java && \ + tar -xf /tmp/java.tar.gz -C /opt/java --strip-components 1 && \ + rm /tmp/java.tar.gz + +RUN curl -L https://nodejs.org/dist/v16.11.0/node-v16.11.0-linux-x64.tar.xz --output /tmp/node.tar.xz --silent && \ + mkdir /opt/node && \ + tar -xf /tmp/node.tar.xz -C /opt/node --strip-components 1 && \ + rm /tmp/node.tar.xz + +ENV PATH="/opt/node/bin:${PATH}" +RUN npm install -g typescript@4.4.3 + +RUN curl -L https://www.python.org/ftp/python/3.10.0/Python-3.10.0.tar.xz --output /tmp/python.tar.xz --silent && \ + mkdir /opt/python && \ + tar -xf /tmp/python.tar.xz -C /opt/python --strip-components 1 && \ + rm /tmp/python.tar.xz + +RUN curl -L https://golang.org/dl/go1.17.2.linux-amd64.tar.gz --output /tmp/go.tar.gz --silent && \ + mkdir /opt/go && \ + tar -xf /tmp/go.tar.gz -C /opt/go --strip-components 1 && \ + rm /tmp/go.tar.gz + +COPY build/libs/jprof_by_bot-eval-evaluator-all.jar ${LAMBDA_TASK_ROOT}/lib/ +CMD [ "by.jprof.telegram.bot.eval.evaluator.Evaluator::handleRequest" ] diff --git a/eval/evaluator/build.gradle.kts b/eval/evaluator/build.gradle.kts new file mode 100644 index 00000000..f4e0317c --- /dev/null +++ b/eval/evaluator/build.gradle.kts @@ -0,0 +1,18 @@ +plugins { + kotlin("jvm") + id("com.github.johnrengelman.shadow") +} + +dependencies { + api(project.projects.eval.dto) + implementation(libs.bundles.aws.lambda) + implementation(libs.koin.core) + implementation(libs.kotlinx.coroutines.core) + implementation(libs.kotlinx.serialization.json) + implementation(libs.bundles.log4j) + + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testRuntimeOnly(libs.junit.jupiter.engine) + testRuntimeOnly(libs.log4j.core) +} diff --git a/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/Evaluator.kt b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/Evaluator.kt new file mode 100644 index 00000000..d7810594 --- /dev/null +++ b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/Evaluator.kt @@ -0,0 +1,53 @@ +package by.jprof.telegram.bot.eval.evaluator + +import by.jprof.telegram.bot.eval.dto.EvalEvent +import by.jprof.telegram.bot.eval.evaluator.config.jsonModule +import by.jprof.telegram.bot.eval.evaluator.config.pipelineModule +import by.jprof.telegram.bot.eval.evaluator.middleware.EvalPipeline +import com.amazonaws.services.lambda.runtime.Context +import com.amazonaws.services.lambda.runtime.RequestStreamHandler +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.encodeToStream +import org.apache.logging.log4j.LogManager +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject +import org.koin.core.context.startKoin +import java.io.InputStream +import java.io.OutputStream + +@ExperimentalSerializationApi +@Suppress("unused") +class Evaluator : RequestStreamHandler, KoinComponent { + companion object { + private val logger = LogManager.getLogger(Evaluator::class.java) + } + + init { + startKoin { + modules( + jsonModule, + pipelineModule, + ) + } + } + + private val json: Json by inject() + private val pipeline: EvalPipeline by inject() + + override fun handleRequest(input: InputStream, output: OutputStream, context: Context) = runBlocking { + val payload = input.bufferedReader().use { it.readText() } + + logger.debug("Payload: {}", payload) + + val evalEvent = json.decodeFromString(payload) + + logger.debug("Parsed event: {}", evalEvent) + + val evalResponse = pipeline.process(evalEvent) + + output.buffered().use { json.encodeToStream(evalResponse, it) } + } +} diff --git a/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/json.kt b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/json.kt new file mode 100644 index 00000000..924a7809 --- /dev/null +++ b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/json.kt @@ -0,0 +1,13 @@ +package by.jprof.telegram.bot.eval.evaluator.config + +import kotlinx.serialization.json.Json +import org.koin.dsl.module + +val jsonModule = module { + single { + Json { + encodeDefaults = false + ignoreUnknownKeys = true + } + } +} diff --git a/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/pipeline.kt b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/pipeline.kt new file mode 100644 index 00000000..310b5a7c --- /dev/null +++ b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/pipeline.kt @@ -0,0 +1,17 @@ +package by.jprof.telegram.bot.eval.evaluator.config + +import by.jprof.telegram.bot.eval.evaluator.middleware.Eval +import by.jprof.telegram.bot.eval.evaluator.middleware.EvalPipeline +import by.jprof.telegram.bot.eval.evaluator.middleware.JavaScriptEval +import org.koin.core.qualifier.named +import org.koin.dsl.module + +val pipelineModule = module { + single { + EvalPipeline(getAll()) + } + + single(named("JavaScriptEval")) { + JavaScriptEval() + } +} diff --git a/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/Eval.kt b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/Eval.kt new file mode 100644 index 00000000..56aa33ce --- /dev/null +++ b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/Eval.kt @@ -0,0 +1,8 @@ +package by.jprof.telegram.bot.eval.evaluator.middleware + +import by.jprof.telegram.bot.eval.dto.EvalEvent +import by.jprof.telegram.bot.eval.dto.EvalResponse + +interface Eval { + suspend fun eval(payload: EvalEvent): EvalResponse? +} diff --git a/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipeline.kt b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipeline.kt new file mode 100644 index 00000000..c6e70153 --- /dev/null +++ b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipeline.kt @@ -0,0 +1,32 @@ +package by.jprof.telegram.bot.eval.evaluator.middleware + +import by.jprof.telegram.bot.eval.dto.EvalEvent +import by.jprof.telegram.bot.eval.dto.EvalResponse +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.supervisorScope +import org.apache.logging.log4j.LogManager + +class EvalPipeline( + private val evals: List +) { + companion object { + private val logger = LogManager.getLogger(EvalPipeline::class.java)!! + } + + fun process(evalEvent: EvalEvent): EvalResponse = runBlocking { + supervisorScope { + evals + .map { async(exceptionHandler(it)) { it.eval(evalEvent) } } + .awaitAll() + .filterNotNull() + .firstOrNull() ?: EvalResponse.Unsuccessful + } + } + + private fun exceptionHandler(eval: Eval) = CoroutineExceptionHandler { _, exception -> + logger.error("{} failed!", eval::class.simpleName, exception) + } +} diff --git a/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/JavaScriptEval.kt b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/JavaScriptEval.kt new file mode 100644 index 00000000..1ef78298 --- /dev/null +++ b/eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/JavaScriptEval.kt @@ -0,0 +1,45 @@ +package by.jprof.telegram.bot.eval.evaluator.middleware + +import by.jprof.telegram.bot.eval.dto.EvalEvent +import by.jprof.telegram.bot.eval.dto.EvalResponse +import by.jprof.telegram.bot.eval.dto.Language +import org.apache.logging.log4j.LogManager +import java.io.IOException +import java.util.concurrent.TimeUnit +import kotlin.io.path.absolutePathString +import kotlin.io.path.writeText + +class JavaScriptEval : Eval { + companion object { + private val logger = LogManager.getLogger(JavaScriptEval::class.java) + } + + override suspend fun eval(payload: EvalEvent): EvalResponse? { + val file = kotlin.io.path.createTempFile(prefix = "JavaScriptEval", suffix = ".js") + + logger.info("Created temp file: {}", file) + + file.writeText(payload.code) + + try { + val proc = ProcessBuilder("node", file.absolutePathString()) + .directory(file.parent.toFile()) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .redirectError(ProcessBuilder.Redirect.PIPE) + .start() + + proc.waitFor(30, TimeUnit.SECONDS) + + val result = proc.exitValue() + val stdout = proc.inputStream.bufferedReader().use { it.readText() } + val stderr = proc.errorStream.bufferedReader().use { it.readText() } + + logger.info("Process finished with status: {}. stdout: {}, stderr: {}", result, stdout, stderr) + + return EvalResponse.Successful(Language.JAVASCRIPT, stdout, stderr) + } catch (e: IOException) { + e.printStackTrace() + return null + } + } +} diff --git a/eval/evaluator/src/main/resources/log4j2.xml b/eval/evaluator/src/main/resources/log4j2.xml new file mode 100644 index 00000000..1c4c0559 --- /dev/null +++ b/eval/evaluator/src/main/resources/log4j2.xml @@ -0,0 +1,20 @@ + + + + + + false + true + + + + + + + + + + + + + diff --git a/eval/evaluator/src/test/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipelineTest.kt b/eval/evaluator/src/test/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipelineTest.kt new file mode 100644 index 00000000..eb31ef06 --- /dev/null +++ b/eval/evaluator/src/test/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipelineTest.kt @@ -0,0 +1,42 @@ +package by.jprof.telegram.bot.eval.evaluator.middleware + +import by.jprof.telegram.bot.eval.dto.EvalEvent +import by.jprof.telegram.bot.eval.dto.EvalResponse +import kotlinx.coroutines.delay +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.time.Duration + +internal class EvalPipelineTest { + @Test + fun processWithBroken() { + val sut = EvalPipeline( + (1..5).map { index -> + when (index % 2) { + 0 -> Delay((index + 1) * 1000L) + else -> Fail() + } + } + ) + + Assertions.assertTimeout(Duration.ofMillis(6000)) { + sut.process(EvalEvent("")) + } + } +} + +internal class Delay( + private val delay: Long, +) : Eval { + override suspend fun eval(payload: EvalEvent): EvalResponse? { + delay(delay) + + return EvalResponse.Unsuccessful + } +} + +internal class Fail : Eval { + override suspend fun eval(payload: EvalEvent): EvalResponse? { + throw IllegalArgumentException() + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b450b237..ed356cf9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -26,6 +26,7 @@ aws-junit5 = "6.0.1" mockk = "1.12.0" [libraries] +kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines" } kotlinx-coroutines-jdk8 = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-jdk8", version.ref = "coroutines" } aws-lambda-java-events = { group = "com.amazonaws", name = "aws-lambda-java-events", version.ref = "aws-lambda-java-events" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 3b5fe134..7cfbbfb7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,4 +28,6 @@ include(":pins:dto") include(":pins:unpin") include(":pins:dynamodb") include(":pins:sfn") +include(":eval:dto") +include(":eval:evaluator") include(":runners:lambda")