Merge pull request #25 from jeff303/demo-bugfixes

Demo bugfixes

README.md

@@ -83,6 +83,8 @@ record handling.
  ::kcr/value-type       "json"       ; the format the value should be parsed in (optional)
  ::kcr/value-parse-fn   (fn [b] nil) ; a custom function to parse the value (passed as a byte array)
                                      ; if both are omitted, the value won't be parsed
+ ::kcr/key->clj?          true       ; if true, the key will be converted to a Clojure map after parsing the bytes
+ ::kcr/val->clj?          true       ; if true, the value will be converted to a Clojure map after parsing the bytes 
  ::kcr/include-headers?   true       ; whether to include message headers in extra-metadata
  ::kcr/include-offset?    true       ; whether to include the message offset in extra-metadata
  ::kcr/include-partition? true       ; whether to include the PartitionInfo in extra-metadata
@@ -159,7 +161,16 @@ can simply be ignored/not passed along from the `map-record-fn`
 There is an uberjar build of `kc-repl` that is capable of running basic operations using `java`.  To use this mode,
 download (or build) the uberjar, then run:
 
-`java -Dclojure.core.async.pool-size=1 -jar kc-repl-*.jar /path/to/consumer.properties`
+#### Without Custom Type Handlers
+
+```shell
+java -Dclojure.core.async.pool-size=1 -jar target/kc-repl-<version>-standalone.jar /path/to/client.properties
+```
+
+#### With Custom Type Handlers
+```shell
+rlwrap java -Dclojure.core.async.pool-size=1 -cp type-handlers/avro/target/kc-repl-type-handler-avro-<version>-avro-handler.jar:target/kc-repl-<version>-standalone.jar us.jeffevans.kc_repl.java_main /path/to/client.properties
+```
 
 The syntax for the Java runtime is more similar to the original Kafka command line tools.  You can type `help` for a
 more complete list.
@@ -186,11 +197,13 @@ advancing, such as `seek-while` and `seek-until`).
 
 #### Extension
 
-The current key and message types that can be parsed are limited (only text and JSON).  There is a multimethod,
-`parse-type`, that provides an extension point through which parsing of new types can be supported.  In the future,
-support for some of these will be added to this project (as submodules that can be built separately and loaded as
-needed).  In theory, any Clojure code that defines a new method for this multimethod can take advantage of this
-mechanism to read additional types (provided the bytecode is available on the classpath).
+The base implementation supports type handling for text and json data.  Additional type handlers can be added by:
+* extending the `us.jeffevans.kc-repl.type-handlers/type-handler` protocol
+* defining a `us.jeffevans.kc-repl.type-handlers/create-type-handler` multimethod implementation
+* adding a `kc-repl-handler-ns.txt` file to your classpath (ex: in `resources`) that indicates the namespace that
+  should be loaded to allow for this type handler to be picked up at runtime
+
+Refer to the `avro` and `protobuf` implementations under `type-handlers` for reference implementations.
 
 ### Building the Uberjar
 `clojure -J-Dclojure.core.async.pool-size=1 -T:build ci`
@@ -202,6 +215,12 @@ mechanism to read additional types (provided the bytecode is available on the cl
 
 Bugs and enhancement requests will be tracked in the GitHub project (via issues).
 
+## Demo Videos
+
+* [Java Main demo](https://www.loom.com/share/15844e1d06454adfb43ba0652788505f?sid=8ced88db-9867-4034-ac0b-75c7191760f4)
+* [Clojure nREPL demo - mapping and reduction with Avro data](https://www.loom.com/share/27f35ee3788d48f08aef3fb5f69e888a?sid=7df588c1-1c15-40f7-8118-4ae6d7bee7a1)
+* [Clojure nREPL demo - conditional seeking with Avro data](https://www.loom.com/share/10855fbf73eb44f09333dea2a6095553?sid=532c2667-8d58-4f2e-9317-c3c48baf611b)
+
 ## License
 
 Copyright © 2022 Jeff Evans

src/us/jeffevans/kc_repl.clj

@@ -46,9 +46,13 @@
             {}
             (.toArray headers))))
 
-(defn- parse-type-fn-with-handlers [^String type-name type-handlers]
+(defn- parse-type-fn-with-handlers [^String type-name type-handlers v->clj?]
   (if-let [handler (get type-handlers type-name)]
-    (partial th/parse-bytes handler)
+    (if v->clj?
+      (fn [topic v]
+        (->> (th/parse-bytes handler topic v)
+             (th/->clj handler)))
+      (partial th/parse-bytes handler))
     (throw (ex-info (format "no type handler registered for %s" type-name) {::type-handler-keys (keys type-handlers)}))))
 
 (defn- make-default-record-handler-fn [^String value-type type-handlers]
@@ -60,12 +64,12 @@
 
 (defn- make-record-handler-fn [{:keys [::key-type ::key-parse-fn ::value-type ::value-parse-fn ::map-record-fn
                                        ::include-headers? ::include-topic? ::include-partition? ::include-offset?
-                                       ::include-timestamp?] :as record-handling} type-handlers]
+                                       ::include-timestamp? ::key->clj? ::val->clj?] :as record-handling} type-handlers]
   (let [k-parse-fn (cond (fn? key-parse-fn)
                          key-parse-fn
 
                          (some? key-type)
-                         (parse-type-fn-with-handlers key-type type-handlers)
+                         (parse-type-fn-with-handlers key-type type-handlers key->clj?)
 
                          ;; by default, don't try to parse the key
                          true
@@ -74,7 +78,7 @@
                          value-parse-fn
 
                          (some? value-type)
-                         (parse-type-fn-with-handlers value-type type-handlers)
+                         (parse-type-fn-with-handlers value-type type-handlers val->clj?)
 
                          ;; by default, don't try to parse the value
                          true
@@ -247,9 +251,24 @@
           (print beg))))
     (idle-poll! consumer)))
 
+(defn- remove-consumer-assignment!
+  "Removes the given `assignment` (a TopicPartition) from the given `consumer`. If the consumer did not already
+  have it assigned, this is a no-op."
+  [active-assignments-atom ^KafkaConsumer consumer assignments]
+  (doseq [topic (keys assignments)]
+    (let [[part]          (get assignments topic)
+          tp              (TopicPartition. topic part)
+          cur-assignments (set (.assignment consumer))]
+      (swap! active-assignments-atom disj tp)
+      (when (contains? cur-assignments tp)
+        (let [new-assignments (disj cur-assignments tp)]
+          (.assign consumer new-assignments)))))
+  (idle-poll! consumer)
+  ::ok)
+
 (defn- assign-and-seek! [^KafkaConsumer consumer assignments active-assignments-atom]
   (doseq [topic (keys assignments)]
-    (let [[part offset?] (get assignments topic)
+    (let [[part ^long offset?] (get assignments topic)
           tp             (TopicPartition. topic part)]
       (add-consumer-assignment! active-assignments-atom consumer tp true)
       (if (some? offset?)
@@ -260,6 +279,10 @@
                                                                                  ::new-assignment]}]
   (assign-and-seek! consumer new-assignment active-assignments-atom))
 
+(defmethod handle-repl->client-message ::unassign [^KafkaConsumer consumer {:keys [::active-assignments-atom
+                                                                                   ::remove-assignments]}]
+  (remove-consumer-assignment! active-assignments-atom consumer remove-assignments))
+
 (defmethod handle-repl->client-message ::seek [^KafkaConsumer consumer
                                                {:keys [::active-assignments-atom
                                                        ::to
@@ -428,6 +451,13 @@
                                                            ::active-assignments-atom active-assignments-atom
                                                            ::new-assignment {topic [partition offset?]}}))
 
+(defn- unassign!
+  ""
+  [to-consumer-chan from-consumer-chan active-assignments-atom topic partition & [offset?]]
+  (consumer-roundtrip to-consumer-chan from-consumer-chan {::message-type ::unassign
+                                                           ::active-assignments-atom active-assignments-atom
+                                                           ::remove-assignments {topic [partition]}}))
+
 
 (s/def ::consumer-record-map-fn fn?)
 (s/def ::poll-xf-reducing-fn fn?)
@@ -491,11 +521,20 @@
 (s/def ::value-type string?)
 (s/def ::key-parse-fn fn?)
 (s/def ::value-parse-fn fn?)
+
+(s/def ::key->clj? boolean?)
+(s/def ::val->clj? boolean?)
 (s/def ::map-record-fn fn?)
+
+(s/def ::include-headers? boolean?)
+(s/def ::include-topic? boolean?)
+(s/def ::include-partition? boolean?)
+(s/def ::include-offset? boolean?)
+(s/def ::include-timestamp? boolean?)
 (s/def ::reducing-fn fn?)
 
 (s/def ::record-handling-opts (s/keys :opt [::key-type ::value-type ::key-parse-fn ::value-parse-fn ::map-record-fn
-                                            ::reducing-fn]))
+                                            ::reducing-fn ::key->clj? ::val->clj?]))
 
 (defn record-handling-opts->poll-xf-args
   "For the given record-handling-opts, and type-handlers, produce a poll-xf-args map, which can then be passed into a
@@ -563,6 +602,7 @@
   (read-from [this topic part offset num-msg record-handling-opts])
   (last-read [this record-handling-opts])
   (assign [this topic part offset])
+  (unassign [this topic part])
   (seek [this offset])
   (seek+ [this offset+])
   (seek- [this offset-])
@@ -594,6 +634,8 @@
     (last-read* last-read-records-atom record-handling-opts type-handlers))
   (assign [_ topic part offset]
     (assign! to-consumer-chan from-consumer-chan active-assignments-atom topic part offset))
+  (unassign [_ topic part]
+    (unassign! to-consumer-chan from-consumer-chan active-assignments-atom topic part))
   (seek [_ offset]
     (seek! to-consumer-chan from-consumer-chan {::active-assignments-atom active-assignments-atom
                                                 ::to offset}))
@@ -804,14 +846,17 @@
              ^long ^{:doc "The offset to start reading from", :default 0} offset
              ^long ^{:doc "The number of messages to read", :default 10} num-msg
              ;; TODO: figure out how to get by with something like msg-format instead of record-handling-opts in Java
-             ^{:doc "Record handling options (see documentation)"} record-handling-opts)
+             ^{:doc "Record handling options (see documentation)", ::required? true} record-handling-opts)
       (defop ^{:doc "Read from the current active assignments"} poll kcr-client true true
              ^long ^{:doc "The number of messages to read", :default 10} num-msg
              ^{:doc "Record handling options (see documentation)", ::required? true} record-handling-opts)
       (defop ^{:doc "Add a topic/partition to the active assignments"} assign kcr-client true true
              ^{:doc "The topic name to read from", ::required? true} topic
              ^long ^{:doc "The topic partition to read from", ::required? true} part
              ^long ^{:doc "The offset to start reading from", ::required? true} offset)
+      (defop ^{:doc "Remove a topic/partition from the active assignments"} unassign kcr-client true true
+             ^{:doc "The topic name to read from", ::required? true} topic
+             ^long ^{:doc "The topic partition to read from", ::required? true} part)
       (defop ^{:doc "Change the offset to poll from next for the given topic/partition"} seek kcr-client true true
              ^long ^{:doc "The offset to seek to", ::required? true} offset)
       (defop ^{:doc "Increment the offset for the current assignment by the given amount"} seek+ kcr-client true true
@@ -820,16 +865,17 @@
              ^long ^{:doc "The number of offsets to recede by", ::required? true} by)
       (defop ^{:doc "Seek forward in the message stream while some condition is met"} seek-while kcr-client false false
              ^{:doc "Conditional (while) function; operates on the mapped record"} while-fn
-             ^{:doc "Record handling options (see documentation)"} record-handling-opts)
+             ^{:doc "Record handling options (see documentation)", ::required? true} record-handling-opts)
       (defop ^{:doc "Seek forward in the message stream while some condition is met"} seek-until kcr-client false false
              ^{:doc "Conditional (until) function; operates on the mapped record"} while-fn
-             ^{:doc "Record handling options (see documentation)"} record-handling-opts)
+             ^{:doc "Record handling options (see documentation)", ::required? true} record-handling-opts)
       ;; will revisit this once we have a better Java args parser
       (defop ^{:doc "Set a config option for a type handler arr", ::print-offsets? false} set-type-handler-config! kcr-client true true
              ^{:doc "The type on which to set the config option", ::required? true} type-name
              ^{:doc "The config option's key", ::required? true} k
              ^{:doc "The config option's arguments", ::required? true, ::varargs? true} args)
-      (defop ^{:doc "Print the last read results"} last-read kcr-client true true)
+      (defop ^{:doc "Print the last read results"} last-read kcr-client true true
+             ^{:doc "Record handling options (see documentation)", ::required? true} record-handling-opts)
       (defop ^{:doc "Stop the client and disconnect the session"} stop kcr-client true false)
       (intern (the-ns 'user) 'help print-clj-help)
       (when (ifn? run-body)

src/us/jeffevans/kc_repl/java_main.clj

@@ -5,7 +5,7 @@
             [us.jeffevans.kc-repl :as kcr]
             [us.jeffevans.kc-repl.type-handler-load :as thl]
             [instaparse.core :as insta])
-  (:gen-class :name us.jeffevans.KcRepl))
+  (:gen-class))
 
 (defn- get-input-line []
   (read-line))
@@ -81,7 +81,7 @@
       (let [continue? (atom true)]
         (while @continue?
           (let [input       (get-input-line)
-                [op & args] (if (nil? input) ["stop"] (parse-line-as-cli-args input))
+                [op & args] (if (str/blank? input) ["stop"] (parse-line-as-cli-args input))
                 {:keys [::kcr/invoke-fn ::kcr/opts-spec ::kcr/print-offsets?]} (get @#'kcr/java-cmds op)]
             (log/tracef "got input %s, op %s, args %s, opts-spec %s" input op (pr-str args) (pr-str opts-spec))
             (println)

src/us/jeffevans/kc_repl/type_handler_load.clj

@@ -16,8 +16,9 @@
     (require (symbol type-handler-ns))))
 
 (defn load-type-handlers []
-  (log/debug "trying to load type handlers from jars")
+  (log/debugf "trying to load type handlers from jars")
   (doseq [^JarFile jar-file (cp/classpath-jarfiles)]
+    (log/debugf "checking jar: %s" (.getName jar-file))
     (doseq [entry (-> (.entries jar-file)
                       enumeration-seq)]
       (when (= marker-file-nm (.getName entry))

test-common/src/us/jeffevans/kc_repl/test_containers.clj

@@ -8,7 +8,7 @@
 
 (def tc-cfg (TestcontainersConfiguration/getInstance))
 
-(def ^:const cp-version "The Confluent Platform base version, to use for container images" "7.1.0")
+(def ^:const cp-version "The Confluent Platform base version, to use for container images" "7.2.0")
 
 (def zk-network-alias "zookeeper")
 

test/us/jeffevans/kc_repl_test.clj

@@ -136,5 +136,10 @@
                               ::kcr/reducing-fn      reducing-fn
                               ::kcr/map-record-fn    map-rec-fn})))
       (testing " and the new offset is correct (we should now be at offset 10 for test-topic:0)"
-        (is (= {(TopicPartition. "test-topic" 0) [10 true]} (kcr/current-assignments tc/*kcr-client*)))))))
+        (is (= {(TopicPartition. "test-topic" 0) [10 true]} (kcr/current-assignments tc/*kcr-client*))))
+      (testing " and we can unassign the test-topic:0 assignment and switch to test-topic:1"
+        ;; read from partition 1 to add that to active assignments
+        (kcr/read-from tc/*kcr-client* "test-topic" 1 0 1 "json")
+        (kcr/unassign tc/*kcr-client* "test-topic" 0)
+        (is (= {(TopicPartition. "test-topic" 1) [0 true]} (kcr/current-assignments tc/*kcr-client*)))))))
 

type-handlers/src/us/jeffevans/kc_repl/type_handlers.clj

@@ -1,5 +1,4 @@
-(ns us.jeffevans.kc-repl.type-handlers
-  (:gen-class :name us.jeffevans.KcRepl))
+(ns us.jeffevans.kc-repl.type-handlers)
 
 (defprotocol type-handler
              (parse-bytes [this ^String topic ^bytes b])