diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e203fef --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +out +gen + +/build +.gradle + +# IDEs and editors +/.idea +*iml +.project +.classpath +*.launch +.settings/ + +# System Files +.DS_Store + +# Envs +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..d6b43c4 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# Ktor kafka example + +An example repo for the ktor kafka plugin: +https://github.com/karlazzampersonal/ktor-kafka + +In this example, I spin up 2 producers and a consumer for each one + +- One producer that writes a string "Producer A value" to topic `topica`. It's invoked by hitting `localhost:8080/A` +- Another producer that writes a random int from 1-10 to topic `topicb`. It's invoked by hitting `localhost:8080/B` + +Example logs: +``` +2022-02-06 20:19:33.368 [DefaultDispatcher-worker-1] INFO kafka-consumer-logger-b-group - Consumer A reading message: Producer A value +2022-02-06 20:19:33.526 [eventLoopGroupProxy-4-1] INFO ktor.application - 200 OK: GET - /A +2022-02-06 20:19:43.927 [eventLoopGroupProxy-4-1] INFO ktor.application - 200 OK: GET - /B +2022-02-06 20:19:43.937 [DefaultDispatcher-worker-2] INFO kafka-consumer-logger-b-group - Consumer B reading message: 7 +2022-02-06 20:19:50.806 [eventLoopGroupProxy-4-1] INFO ktor.application - 200 OK: GET - /A +2022-02-06 20:19:50.810 [DefaultDispatcher-worker-1] INFO kafka-consumer-logger-b-group - Consumer A reading message: Producer A value +``` diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..6081a1b --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,55 @@ +import java.util.Properties + +val ktor_version: String by project +val kotlin_version: String by project +val logback_version: String by project +val ktor_kafka_version: String by project +val kafka_version: String by project + +plugins { + application + kotlin("jvm") version "1.6.10" + id("org.jlleitschuh.gradle.ktlint") version "10.2.1" +} + +group = "com.levels" +version = "0.0.1" + +application { + mainClass.set("com.levels.ApplicationKt") +} + +val props = Properties() +props.load(file(".env").inputStream()) +repositories { + mavenCentral() + // Need a GH access token with read package scope + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/karlazzampersonal/ktor-kafka") + credentials { + username = props.getProperty("USERNAME") + password = props.getProperty("TOKEN") + } + } +} + +ktlint { + disabledRules.set(mutableListOf("no-wildcard-imports")) + filter { + exclude("**/generated/**") + exclude { element -> element.file.path.contains("generated/") } + include("**/kotlin/**") + } +} + +dependencies { + // 2 dependencies added for kafka: kafka-clients and ktor-kafka + api("org.apache.kafka:kafka-clients:$kafka_version") + implementation("com.levels:ktor-kafka:$ktor_kafka_version") + implementation("io.ktor:ktor-server-core:$ktor_version") + implementation("io.ktor:ktor-server-netty:$ktor_version") + implementation("ch.qos.logback:logback-classic:$logback_version") + testImplementation("io.ktor:ktor-server-tests:$ktor_version") + testImplementation("org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version") +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..6ed1031 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,7 @@ +kotlin.code.style=official + +ktor_version=1.6.7 +kotlin_version=1.6.10 +logback_version=1.2.3 +ktor_kafka_version=0.0.1 +kafka_version=2.7.0 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..7454180 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..69a9715 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..744e882 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..ac1b06f --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..c313a5f --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "ktor-kafka-example" diff --git a/src/main/kotlin/com/levels/Application.kt b/src/main/kotlin/com/levels/Application.kt new file mode 100644 index 0000000..d8e884b --- /dev/null +++ b/src/main/kotlin/com/levels/Application.kt @@ -0,0 +1,45 @@ +package com.levels + +import com.levels.kafka.* +import com.levels.kafka.Kafka.Feature.getJob +import com.levels.plugins.* +import io.ktor.application.* +import io.ktor.server.engine.* +import io.ktor.server.netty.* +import kotlinx.coroutines.launch +import org.apache.kafka.clients.producer.KafkaProducer + +fun main() { + embeddedServer(Netty, port = 8080, host = "0.0.0.0") { + // Logging for HTTP requests + configureCallLogging() + + // Installation of kafka below + configureKafka() + + // Create 2 producers and pass them down to the REST routes + val producerA: KafkaProducer = Producer().createProducer(producerAProps()) + val producerB: KafkaProducer = Producer().createProducer(producerBProps()) + configureRouting(producerA, producerB) + + // On startup setup consumer jobs + environment.monitor.subscribe(ApplicationStarted) { + // Fetch consumer jobs by group id + val a = ConsumerA(getJob(groupId = "a-group")) + val b = ConsumerB(getJob(groupId = "b-group")) + + // Launch consumers in diff coroutine scopes + launch { + a.start() + } + launch { + b.start() + } + } + + environment.monitor.subscribe(ApplicationStopping) { + // Shut down each consumer job when application is about to stop + Kafka.consumerList.forEach { it.shutdown() } + } + }.start(wait = true) +} diff --git a/src/main/kotlin/com/levels/kafka/ConsumerA.kt b/src/main/kotlin/com/levels/kafka/ConsumerA.kt new file mode 100644 index 0000000..9b8a545 --- /dev/null +++ b/src/main/kotlin/com/levels/kafka/ConsumerA.kt @@ -0,0 +1,15 @@ +package com.levels.kafka + +import java.time.Duration + +class ConsumerA(private val job: ConsumerJob) { + + suspend fun start() = job.start { + val records = job.consumer.poll(Duration.ofMillis(1000L)) + + for (record in records) { + val value = record.value() as String + log.info("Consumer A reading message: $value") + } + } +} diff --git a/src/main/kotlin/com/levels/kafka/ConsumerB.kt b/src/main/kotlin/com/levels/kafka/ConsumerB.kt new file mode 100644 index 0000000..b4bd706 --- /dev/null +++ b/src/main/kotlin/com/levels/kafka/ConsumerB.kt @@ -0,0 +1,15 @@ +package com.levels.kafka + +import java.time.Duration + +class ConsumerB(private val job: ConsumerJob) { + + suspend fun start() = job.start { + val records = job.consumer.poll(Duration.ofMillis(1000L)) + + for (record in records) { + val value = record.value() as Int + log.info("Consumer B reading message: $value") + } + } +} diff --git a/src/main/kotlin/com/levels/kafka/ProducerConfig.kt b/src/main/kotlin/com/levels/kafka/ProducerConfig.kt new file mode 100644 index 0000000..d55ac64 --- /dev/null +++ b/src/main/kotlin/com/levels/kafka/ProducerConfig.kt @@ -0,0 +1,26 @@ +package com.levels.kafka + +import com.levels.kafka.Kafka.Feature.kafkaConfig +import java.util.* + +fun producerAProps(): Properties { + val props = Properties() + props.setProperty("bootstrap.servers", kafkaConfig.bootstrapServer) + props.setProperty("acks", "all") + props.setProperty("retries", "0") + props.setProperty("linger.ms", "1") + props.setProperty("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + props.setProperty("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + return props +} + +fun producerBProps(): Properties { + val props = Properties() + props.setProperty("bootstrap.servers", kafkaConfig.bootstrapServer) + props.setProperty("acks", "all") + props.setProperty("retries", "0") + props.setProperty("linger.ms", "1") + props.setProperty("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + props.setProperty("value.serializer", "org.apache.kafka.common.serialization.IntegerSerializer") + return props +} diff --git a/src/main/kotlin/com/levels/plugins/Kafka.kt b/src/main/kotlin/com/levels/plugins/Kafka.kt new file mode 100644 index 0000000..34f36cd --- /dev/null +++ b/src/main/kotlin/com/levels/plugins/Kafka.kt @@ -0,0 +1,24 @@ +package com.levels.plugins + +import com.levels.kafka.Kafka +import com.levels.kafka.consumer +import com.levels.kafka.kafka +import io.ktor.application.* + +fun Application.configureKafka() { + install(Kafka) { + kafka { bootstrapServer = "localhost:9092" } + consumer { + groupId = "a-group" + topics = listOf("topica") + keyDeserializer = "org.apache.kafka.common.serialization.StringDeserializer" + valueDeserializer = "org.apache.kafka.common.serialization.StringDeserializer" + } + consumer { + groupId = "b-group" + topics = listOf("topicb") + keyDeserializer = "org.apache.kafka.common.serialization.StringDeserializer" + valueDeserializer = "org.apache.kafka.common.serialization.IntegerDeserializer" + } + } +} diff --git a/src/main/kotlin/com/levels/plugins/Monitoring.kt b/src/main/kotlin/com/levels/plugins/Monitoring.kt new file mode 100644 index 0000000..d3ca8bb --- /dev/null +++ b/src/main/kotlin/com/levels/plugins/Monitoring.kt @@ -0,0 +1,13 @@ +package com.levels.plugins + +import io.ktor.application.* +import io.ktor.features.* +import io.ktor.request.* +import org.slf4j.event.* + +fun Application.configureCallLogging() { + install(CallLogging) { + level = Level.INFO + filter { call -> call.request.path().startsWith("/") } + } +} diff --git a/src/main/kotlin/com/levels/plugins/Routing.kt b/src/main/kotlin/com/levels/plugins/Routing.kt new file mode 100644 index 0000000..06ab8db --- /dev/null +++ b/src/main/kotlin/com/levels/plugins/Routing.kt @@ -0,0 +1,38 @@ +package com.levels.plugins + +import io.ktor.application.* +import io.ktor.response.* +import io.ktor.routing.* +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import java.util.* + +fun Application.configureRouting( + producerA: KafkaProducer, + producerB: KafkaProducer +) { + routing { + + // Send producer A record (string) + get("/A") { + val message = ProducerRecord( + "topica", + UUID.randomUUID().toString(), + "Producer A value" + ) + producerA.send(message) + call.respondText("Sent request to producer A") + } + + // Send producer B record (int) + get("/B") { + val message = ProducerRecord( + "topicb", + UUID.randomUUID().toString(), + (0..10).random() + ) + producerB.send(message) + call.respondText("Sent request to producer B") + } + } +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..e573f7d --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,12 @@ + + + + %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + +