From e21997c01536dd4340a257066ff822fd339b62db Mon Sep 17 00:00:00 2001 From: "R.I.Pienaar" Date: Sat, 22 Feb 2020 22:08:26 +0000 Subject: [PATCH] unified nats command --- .gitignore | 1 + .goreleaser.yml | 24 ++ .travis.yml | 1 + Dockerfile | 4 +- README.md | 193 +++++----- go.mod | 5 +- go.sum | 10 +- internal/jsch/backup.go | 35 +- internal/jsch/consumers.go | 47 ++- internal/jsch/jsch.go | 69 +++- internal/jsch/jsch_test.go | 46 --- internal/jsch/streams.go | 16 +- internal/jsch/templates.go | 6 +- jsm/restore_command.go | 4 +- jsm/util.go | 2 +- nats/account_command.go | 61 +++ nats/backup_command.go | 27 ++ nats/bench_command.go | 148 +++++++ nats/consumer_command.go | 591 ++++++++++++++++++++++++++++ nats/events_command.go | 324 ++++++++++++++++ nats/main.go | 53 +++ nats/nats_test.go | 558 +++++++++++++++++++++++++++ nats/pub_command.go | 88 +++++ nats/reply_command.go | 79 ++++ nats/restore_command.go | 58 +++ nats/server_command.go | 24 ++ nats/server_list_command.go | 132 +++++++ nats/server_ping_command.go | 166 ++++++++ nats/stream_command.go | 742 ++++++++++++++++++++++++++++++++++++ nats/sub_command.go | 67 ++++ nats/util.go | 330 ++++++++++++++++ nats/util_test.go | 82 ++++ 32 files changed, 3794 insertions(+), 199 deletions(-) create mode 100644 nats/account_command.go create mode 100644 nats/backup_command.go create mode 100644 nats/bench_command.go create mode 100644 nats/consumer_command.go create mode 100644 nats/events_command.go create mode 100644 nats/main.go create mode 100644 nats/nats_test.go create mode 100644 nats/pub_command.go create mode 100644 nats/reply_command.go create mode 100644 nats/restore_command.go create mode 100644 nats/server_command.go create mode 100644 nats/server_list_command.go create mode 100644 nats/server_ping_command.go create mode 100644 nats/stream_command.go create mode 100644 nats/sub_command.go create mode 100644 nats/util.go create mode 100644 nats/util_test.go diff --git a/.gitignore b/.gitignore index 7189e8e..920fff3 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ pkg # bin jsm/jsm dist +nats/nats diff --git a/.goreleaser.yml b/.goreleaser.yml index 61db05f..871b6c0 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -11,7 +11,30 @@ changelog: skip: true builds: + - main: ./nats + id: nats + binary: nats + env: + - GO111MODULE=on + - CGO_ENABLED=0 + goos: + - darwin + - linux + - windows + goarch: + - amd64 + - arm + - arm64 + - 386 + goarm: + - 6 + - 7 + ignore: + - goos: darwin + goarch: 386 + - main: ./jsm + id: jsm binary: jsm env: - GO111MODULE=on @@ -46,6 +69,7 @@ dockers: skip_push: true binaries: - jsm + - nats image_templates: - "synadia/jsm:latest" - "synadia/jsm:{{.Version}}" diff --git a/.travis.yml b/.travis.yml index 385f68e..6a38cc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ install: before_script: - GO_LIST=$(go list ./...) - cd jsm;go build;cd - + - cd nats;go build ;cd - - $(exit $(go fmt $GO_LIST | wc -l)) - go vet $GO_LIST - find . -type f -name "*.go" | grep -v "/vendor/" | xargs misspell -error -locale US diff --git a/Dockerfile b/Dockerfile index ae3d17a..7405930 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ COPY --from=JS /nats-server /nats-server # goreleaser does the build COPY jsm /usr/local/bin/ +COPY nats /usr/local/bin/ COPY README.md / COPY ngs-server.conf / COPY entrypoint.sh / @@ -16,4 +17,5 @@ ENV NATS_URL=jetstream:4222 RUN apk add --update ca-certificates man bash && \ mkdir -p /usr/share/man/man1 && \ - jsm --help-man > /usr/share/man/man1/jsm.1 + jsm --help-man > /usr/share/man/man1/jsm.1 && \ + nats --help-man > /usr/share/man/man1/nats.1 diff --git a/README.md b/README.md index 771297f..fd5c825 100644 --- a/README.md +++ b/README.md @@ -145,13 +145,13 @@ When defining Consumers the items below make up the entire configuration of the ### Configuration -The rest of this document introduces the `jsm` utility, but for completeness and reference this is how you'd create the ORDERS scenario. We'll configure a 1 year retention for order related messages: +The rest of this document introduces the `nats` utility, but for completeness and reference this is how you'd create the ORDERS scenario. We'll configure a 1 year retention for order related messages: ```bash -$ jsm str add ORDERS --subjects "ORDERS.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size=-1 -$ jsm con add ORDERS NEW --filter ORDERS.received --ack explicit --pull --deliver all --max-deliver=-1 --sample 100 -$ jsm con add ORDERS DISPATCH --filter ORDERS.processed --ack explicit --pull --deliver all --max-deliver=-1 --sample 100 -$ jsm con add ORDERS MONITOR --filter '' --ack none --target monitor.ORDERS --deliver last --replay instant +$ nats str add ORDERS --subjects "ORDERS.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size=-1 +$ nats con add ORDERS NEW --filter ORDERS.received --ack explicit --pull --deliver all --max-deliver=-1 --sample 100 +$ nats con add ORDERS DISPATCH --filter ORDERS.processed --ack explicit --pull --deliver all --max-deliver=-1 --sample 100 +$ nats con add ORDERS MONITOR --filter '' --ack none --target monitor.ORDERS --deliver last --replay instant ``` ## Getting Started @@ -160,7 +160,7 @@ This tech preview is limited to a single server and defaults to the global accou ### Using Docker -The `synadia/jsm:latest` docker image contains both the JetStream enabled NATS Server and the `jsm` utility this guide covers. +The `synadia/jsm:latest` docker image contains both the JetStream enabled NATS Server and the `nats` utility this guide covers. In one window start JetStream: @@ -186,7 +186,7 @@ And in another log into the utilities: $ docker run -ti --link jetstream synadia/jsm:latest ``` -This shell has the `jsm` utility and all other NATS cli tools used in the rest of this guide. +This shell has the `nats` utility and all other NATS cli tools used in the rest of this guide. Now skip to the `Administer JetStream` section. @@ -280,9 +280,9 @@ jetstream { Once the server is running it's time to use the management tool. This can be downloaded from the [GitHub Release Page](https://github.com/nats-io/jetstream/releases/) or you can use the `synadia/jsm:latest` docker image. ``` -$ jsm --help -usage: jsm [] [ ...] -JetStream Management Tool +$ nats --help +usage: nats [] [ ...] +NATS Management Utility Flags: --help Show context-sensitive help (also try --help-long and --help-man). @@ -302,7 +302,7 @@ Commands: We'll walk through the above scenario and introduce features of the CLI and of JetStream as we recreate the setup above. -Throughout this example, we'll show other commands like `nats-pub` and `nats-sub` to interact with the system. These are normal existing core NATS commands and JetStream is fully usable by only using core NATS. +Throughout this example, we'll show other commands like `nats pub` and `nats sub` to interact with the system. These are normal existing core NATS commands and JetStream is fully usable by only using core NATS. We'll touch on some additional features but please review the section on the design model to understand all possible permutations. @@ -311,7 +311,7 @@ We'll touch on some additional features but please review the section on the des JetStream is multi-tenant so you will need to check that your account is enabled for JetStream and is not limited. You can view your limits as follows: ```nohighlight -$ jsm account info +$ nats account info Memory: 0 B of 6.4 GB Storage: 0 B of 1.1 TB @@ -325,7 +325,7 @@ The first step is to set up storage for our `ORDERS` related messages, these arr #### Creating ```nohighlight -$ jsm str add ORDERS +$ nats str add ORDERS ? Subjects to consume ORDERS.* ? Storage backend file ? Retention Policy Limits @@ -361,7 +361,7 @@ Statistics: You can get prompted interactively for missing information as above, or do it all on one command. Pressing `?` in the CLI will help you map prompts to CLI options: ``` -$ jsm str add ORDERS --subjects "ORDERS.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size=-1 +$ nats str add ORDERS --subjects "ORDERS.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size=-1 ``` #### Listing @@ -369,7 +369,7 @@ $ jsm str add ORDERS --subjects "ORDERS.*" --ack --max-msgs=-1 --max-bytes=-1 -- We can confirm our Stream was created: ```nohighlight -$ jsm str ls +$ nats str ls Streams: ORDERS @@ -380,7 +380,7 @@ Streams: Information about the configuration of the Stream can be seen, and if you did not specify the Stream like below, it will prompt you based on all known ones: ```nohighlight -$ jsm str info ORDERS +$ nats str info ORDERS Information for Stream ORDERS Configuration: @@ -406,7 +406,7 @@ Statistics: Most commands that show data as above support `-j` to show the results as JSON: ```nohighlight -$ jsm str info ORDERS -j +$ nats str info ORDERS -j { "config": { "name": "ORDERS", @@ -431,14 +431,14 @@ $ jsm str info ORDERS -j } ``` -This is the general pattern for the entire `jsm` utility - prompting for needed information but every action can be run non-interactively making it usable as a cli api. All information output like seen above can be turned into JSON using `-j`. +This is the general pattern for the entire `nats` utility as it relates to JetStream - prompting for needed information but every action can be run non-interactively making it usable as a cli api. All information output like seen above can be turned into JSON using `-j`. #### Copying A stream can be copied into another, which also allows the configuration of the new one to be adjusted via CLI flags: ```nohighlight -$ jsm str cp ORDERS ARCHIVE --subjects "ORDERS_ARCVHIVE.*" --max-age 2y +$ nats str cp ORDERS ARCHIVE --subjects "ORDERS_ARCVHIVE.*" --max-age 2y Stream ORDERS was created Information for Stream ARCHIVE @@ -456,12 +456,12 @@ Configuration: A stream configuration can be edited, which allows the configuration to be adjusted via CLI flags. Here I have a incorrectly created ORDERS stream that I fix: ```nohighlight -$ jsm str info ORDERS -j | jq .config.subjects +$ nats str info ORDERS -j | jq .config.subjects [ "ORDERS.new" ] -$ jsm str edit ORDERS --subjects "ORDERS.*" +$ nats str edit ORDERS --subjects "ORDERS.*" Stream ORDERS was updated Information for Stream ORDERS @@ -474,26 +474,27 @@ Configuration: #### Publishing Into a Stream -Now let's add in some messages to our Stream. You can use `nats-pub` or `nats-bench`. Or even `nats-req` to see the publish ack being returned (these are included in the `synadia/jsm:latest` docker image). +Now let's add in some messages to our Stream. You can use `nats pub` to add messages, pass the `--wait` flag to see the publish ack being returned. You can publish without waiting for acknowledgement: ```nohighlight -$ nats-pub ORDERS.scratch hello +$ nats pub ORDERS.scratch hello Published [sub1] : 'hello' ``` But if you want to be sure your messages got to JetStream and were persisted you can make a request: ```nohighlight -$ nats-req ORDERS.scratch hello -+OK +$ nats req ORDERS.scratch hello +13:45:03 Sending request on [ORDERS.scratch] +13:45:03 Received on [_INBOX.M8drJkd8O5otORAo0sMNkg.scHnSafY]: '+OK' ``` Keep checking the status of the Stream while doing this and you'll see it's stored messages increase. ```nohighlight -$ jsm str info ORDERS +$ nats str info ORDERS Information for Stream ORDERS ... Statistics: @@ -512,7 +513,7 @@ After putting some throw away data into the Stream, we can purge all the data ou To delete all data in a stream use `purge`: ```nohighlight -$ jsm str purge ORDERS -f +$ nats str purge ORDERS -f ... Statistics: @@ -528,7 +529,7 @@ Statistics: A single message can be securely removed from the stream: ```nohighlight -$ jsm str rmm ORDERS 1 -f +$ nats str rmm ORDERS 1 -f ``` #### Deleting Sets @@ -536,8 +537,8 @@ $ jsm str rmm ORDERS 1 -f Finally for demonstration purposes, you can also delete the whole Stream and recreate it so then we're ready for creating the Consumers: ``` -$ jsm str rm ORDERS -f -$ jsm str add ORDERS --subjects "ORDERS.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size=-1 +$ nats str rm ORDERS -f +$ nats str add ORDERS --subjects "ORDERS.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size=-1 ``` ### Consumers @@ -551,7 +552,7 @@ The `NEW` and `DISPATCH` Consumers are pull-based, meaning the services consumin Pull-based Consumers are created the same as push-based Consumers, just don't specify a delivery target. ``` -$ jsm con ls ORDERS +$ nats con ls ORDERS No Consumers defined ``` @@ -560,7 +561,7 @@ We have no Consumers, lets add the `NEW` one: I supply the `--sample` options on the CLI as this is not prompted for at present, everything else is prompted. The help in the CLI explains each: ``` -$ jsm con add --sample 100 +$ nats con add --sample 100 ? Select a Stream ORDERS ? Consumer name NEW ? Delivery target @@ -599,7 +600,7 @@ A Maximum Delivery limit of 20 is set, this means if the message is not acknowle Again this can all be done in a single CLI call, lets make the `DISPATCH` Consumer: ``` -$ jsm con add ORDERS DISPATCH --filter ORDERS.processed --ack explicit --pull --deliver all --sample 100 --max-deliver 20 +$ nats con add ORDERS DISPATCH --filter ORDERS.processed --ack explicit --pull --deliver all --sample 100 --max-deliver 20 ``` #### Creating Push-Based Consumers @@ -607,7 +608,7 @@ $ jsm con add ORDERS DISPATCH --filter ORDERS.processed --ack explicit --pull -- Our `MONITOR` Consumer is push-based, has no ack and will only get new messages and is not sampled: ``` -$ jsm con add +$ nats con add ? Select a Stream ORDERS ? Consumer name MONITOR ? Delivery target monitor.ORDERS @@ -638,7 +639,7 @@ State: Again you can do this with a single non interactive command: ``` -$ jsm con add ORDERS MONITOR --ack none --target monitor.ORDERS --deliver last --replay instant --filter '' +$ nats con add ORDERS MONITOR --ack none --target monitor.ORDERS --deliver last --replay instant --filter '' ``` #### Listing @@ -646,7 +647,7 @@ $ jsm con add ORDERS MONITOR --ack none --target monitor.ORDERS --deliver last - You can get a quick list of all the Consumers for a specific Stream: ``` -$ jsm con ls ORDERS +$ nats con ls ORDERS Consumers for Stream ORDERS: DISPATCH @@ -659,7 +660,7 @@ Consumers for Stream ORDERS: All details for an Consumer can be queried, lets first look at a pull-based Consumer: ``` -$ jsm con info ORDERS DISPATCH +$ nats con info ORDERS DISPATCH Information for Consumer ORDERS > DISPATCH Configuration: @@ -691,21 +692,21 @@ Pull-based Consumers require you to specifically ask for messages and ack them, First we ensure we have a message: ``` -$ nats-pub ORDERS.processed "order 1" -$ nats-pub ORDERS.processed "order 2" -$ nats-pub ORDERS.processed "order 3" +$ nats pub ORDERS.processed "order 1" +$ nats pub ORDERS.processed "order 2" +$ nats pub ORDERS.processed "order 3" ``` -We can now read them using `jsm`: +We can now read them using `nats`: ``` -$ jsm con next ORDERS DISPATCH +$ nats con next ORDERS DISPATCH --- received on ORDERS.processed order 1 Acknowledged message -$ jsm con next ORDERS DISPATCH +$ nats con next ORDERS DISPATCH --- received on ORDERS.processed order 2 @@ -717,19 +718,19 @@ You can prevent ACKs by supplying `--no-ack`. To do this from code you'd send a `Request()` to `$JS.NEXT.ORDERS.DISPATCH`: ``` -$ nats-req '$JS.NEXT.ORDERS.DISPATCH' '' +$ nats req '$JS.NEXT.ORDERS.DISPATCH' '' Published [$JS.NEXT.ORDERS.DISPATCH] : '' -Received [ORDERS.processed] : 'order 3' +Received [ORDERS.processed] : 'order 3' ``` -Here `nats-req` cannot ack, but in your code you'd respond to the received message with a nil payload as an Ack to JetStream. +Here `nats req` cannot ack, but in your code you'd respond to the received message with a nil payload as an Ack to JetStream. #### Consuming Push-Based Consumers Push-based Consumers will publish messages to a subject and anyone who subscribes to the subject will get them, they support different Acknowledgement models covered later, but here on the `MONITOR` Consumer we have no Acknowledgement. ``` -$ jsm con info ORDERS MONITOR +$ nats con info ORDERS MONITOR ... Delivery Subject: monitor.ORDERS ... @@ -738,7 +739,7 @@ $ jsm con info ORDERS MONITOR The Consumer is publishing to that subject, so lets listen there: ``` -$ nats-sub monitor.ORDERS +$ nats sub monitor.ORDERS Listening on [monitor.ORDERS] [#3] Received on [ORDERS.processed]: 'order 3' [#4] Received on [ORDERS.processed]: 'order 4' @@ -789,7 +790,7 @@ Consumers have 3 acknowledgement modes: To understand how Consumers track messages we will start with a clean `ORDERS` Stream and `DISPATCH` Consumer. ``` -$ jsm str info ORDERS +$ nats str info ORDERS ... Statistics: @@ -803,7 +804,7 @@ Statistics: The Set is entirely empty ``` -$ jsm con info ORDERS DISPATCH +$ nats con info ORDERS DISPATCH ... State: @@ -818,9 +819,9 @@ The Consumer has no messages oustanding and has never had any (Consumer sequence We publish one message to the Stream and see that the Stream received it: ``` -$ nats-pub ORDERS.processed "order 4" -Published [ORDERS.processed] : 'order 4' -$ jsm str info ORDERS +$ nats pub ORDERS.processed "order 4" +Published 7 bytes to ORDERS.processed +$ nats str info ORDERS ... Statistics: @@ -834,13 +835,13 @@ Statistics: As the Consumer is pull-based, we can fetch the message, ack it, and check the Consumer state: ``` -$ jsm con next ORDERS DISPATCH +$ nats con next ORDERS DISPATCH --- received on ORDERS.processed order 4 Acknowledged message -$ jsm con info ORDERS DISPATCH +$ nats con info ORDERS DISPATCH ... State: @@ -855,14 +856,14 @@ The message got delivered and acknowledged - `Acknowledgement floor` is `1` and We'll publish another message, fetch it but not Ack it this time and see the status: ``` -$ nats-pub ORDERS.processed "order 5" -Published [ORDERS.processed] : 'order 5' +$ nats pub ORDERS.processed "order 5" +Published 7 bytes to ORDERS.processed -$ jsm con next ORDERS DISPATCH --no-ack +$ nats con next ORDERS DISPATCH --no-ack --- received on ORDERS.processed order 5 -$ jsm con info ORDERS DISPATCH +$ nats con info ORDERS DISPATCH State: Last Delivered Message: Consumer sequence: 3 Stream sequence: 3 @@ -876,11 +877,11 @@ Now we can see the Consumer have processed 2 messages (obs sequence is 3, next m If I fetch it again and again do not ack it: ``` -$ jsm con next ORDERS DISPATCH --no-ack +$ nats con next ORDERS DISPATCH --no-ack --- received on ORDERS.processed order 5 -$ jsm con info ORDERS DISPATCH +$ nats con info ORDERS DISPATCH State: Last Delivered Message: Consumer sequence: 4 Stream sequence: 3 @@ -894,12 +895,12 @@ The Consumer sequence increases - each delivery attempt increase the sequence - Finally if I then fetch it again and ack it this time: ``` -$ jsm con next ORDERS DISPATCH +$ nats con next ORDERS DISPATCH --- received on ORDERS.processed order 5 Acknowledged message -$ jsm con info ORDERS DISPATCH +$ nats con info ORDERS DISPATCH State: Last Delivered Message: Consumer sequence: 5 Stream sequence: 3 @@ -939,8 +940,8 @@ Lets look at each of these, first we make a new Stream `ORDERS` and add 100 mess Now create a `DeliverAll` pull-based Consumer: ``` -$ jsm con add ORDERS ALL --pull --filter ORDERS.processed --ack none --replay instant --deliver all -$ jsm con next ORDERS ALL +$ nats con add ORDERS ALL --pull --filter ORDERS.processed --ack none --replay instant --deliver all +$ nats con next ORDERS ALL --- received on ORDERS.processed order 1 @@ -950,8 +951,8 @@ Acknowledged message Now create a `DeliverLast` pull-based Consumer: ``` -$ jsm con add ORDERS LAST --pull --filter ORDERS.processed --ack none --replay instant --deliver last -$ jsm con next ORDERS LAST +$ nats con add ORDERS LAST --pull --filter ORDERS.processed --ack none --replay instant --deliver last +$ nats con next ORDERS LAST --- received on ORDERS.processed order 100 @@ -961,8 +962,8 @@ Acknowledged message Now create a `MsgSetSeq` pull-based Consumer: ``` -$ jsm con add ORDERS TEN --pull --filter ORDERS.processed --ack none --replay instant --deliver 10 -$ jsm con next ORDERS TEN +$ nats con add ORDERS TEN --pull --filter ORDERS.processed --ack none --replay instant --deliver 10 +$ nats con next ORDERS TEN --- received on ORDERS.processed order 10 @@ -972,10 +973,10 @@ Acknowledged message And finally a time-based Consumer. Let's add some messages a minute apart: ``` -$ jsm str purge ORDERS +$ nats str purge ORDERS $ for i in 1 2 3 do - nats-pub ORDERS.processed "order ${i}" + nats pub ORDERS.processed "order ${i}" sleep 60 done ``` @@ -983,8 +984,8 @@ done Then create an Consumer that starts 2 minutes ago: ``` -$ jsm con add ORDERS 2MIN --pull --filter ORDERS.processed --ack none --replay instant --deliver 2m -$ jsm con next ORDERS 2MIN +$ nats con add ORDERS 2MIN --pull --filter ORDERS.processed --ack none --replay instant --deliver 2m +$ nats con next ORDERS 2MIN --- received on ORDERS.processed order 2 @@ -1002,13 +1003,13 @@ Ephemeral Consumers can only be push-based. Terminal 1: ``` -$ nats-sub my.monitor +$ nats sub my.monitor ``` Terminal 2: ``` -$ jsm con add ORDERS --filter '' --ack none --target 'my.monitor' --deliver last --replay instant --ephemeral +$ nats con add ORDERS --filter '' --ack none --target 'my.monitor' --deliver last --replay instant --ephemeral ``` The `--ephemeral` switch tells the system to make an Ephemeral Consumer. @@ -1022,7 +1023,7 @@ This is useful in load testing scenarios etc. This is called the `ReplayPolicy` You can only set `ReplayPolicy` on push-based Consumers. ``` -$ jsm con add ORDERS REPLAY --target out.original --filter ORDERS.processed --ack none --deliver all --sample 100 --replay original +$ nats con add ORDERS REPLAY --target out.original --filter ORDERS.processed --ack none --deliver all --sample 100 --replay original ... Replay Policy: original ... @@ -1033,7 +1034,7 @@ Now lets publish messages into the Set 10 seconds apart: ``` $ for i in 1 2 3 <15:15:35 do - nats-pub ORDERS.processed "order ${i}" + nats pub ORDERS.processed "order ${i}" sleep 10 done Published [ORDERS.processed] : 'order 1' @@ -1044,7 +1045,7 @@ Published [ORDERS.processed] : 'order 3' And when we consume them they will come to us 10 seconds apart: ``` -$ nats-sub -t out.original +$ nats sub -t out.original Listening on [out.original] 2020/01/03 15:17:26 [#1] Received on [ORDERS.processed]: 'order 1' 2020/01/03 15:17:36 [#2] Received on [ORDERS.processed]: 'order 2' @@ -1057,7 +1058,7 @@ Listening on [out.original] When you have many similar streams it can be helpful to auto create them, lets say you have a service by client and they are on subjects `CLIENT.*`, you can construct a template that will auto generate streams for any matching traffic. ``` -$ jsm str template add CLIENTS --subjects "CLIENT.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size 2048 --max-streams 1024 +$ nats str template add CLIENTS --subjects "CLIENT.*" --ack --max-msgs=-1 --max-bytes=-1 --max-age=1y --storage file --retention limits --max-msg-size 2048 --max-streams 1024 Stream Template CLIENTS was created Information for Stream Template CLIENTS @@ -1083,13 +1084,13 @@ Managed Streams: You can see now streams exist now for it, let's publish some data: ``` -$ nats-pub CLIENT.acme hello +$ nats pub CLIENT.acme hello ``` And we'll have 1 new Stream: ``` -$ jsm str ls +$ nats str ls Streams: CLIENTS_acme @@ -1107,12 +1108,12 @@ Consumers can sample Ack'ed messages for you and publish samples so your monitor #### Configuration -You can configure an Consumer for sampling by passing the `--sample 80` option to `jsm obs add`, this tells the system to sample 80% of Acknowledgements. +You can configure an Consumer for sampling by passing the `--sample 80` option to `nats consumer add`, this tells the system to sample 80% of Acknowledgements. When viewing info of an Consumer you can tell if it's sampled or not: ``` -$ jsm con info ORDERS NEW +$ nats con info ORDERS NEW ... Sampling Rate: 100 ... @@ -1120,10 +1121,10 @@ $ jsm con info ORDERS NEW #### Consuming -Samples are published to `$JS.EVENT.METRIC.CONSUMER_ACK..` in JSON format containing `server.server.JetStreamMetricConsumerAckPre`. Use the `jsm con events` command to view samples: +Samples are published to `$JS.EVENT.METRIC.CONSUMER_ACK..` in JSON format containing `server.server.JetStreamMetricConsumerAckPre`. Use the `nats con events` command to view samples: ```nohighlight -$ jsm con events ORDERS NEW +$ nats con events ORDERS NEW Listening for Advisories on $JS.EVENT.ADVISORY.*.ORDERS.NEW Listening for Metrics on $JS.EVENT.METRIC.*.ORDERS.NEW @@ -1136,7 +1137,7 @@ Listening for Metrics on $JS.EVENT.METRIC.*.ORDERS.NEW ``` ```nohighlight -$ jsm con events ORDERS NEW --json +$ nats con events ORDERS NEW --json { "stream": "ORDERS", "consumer": "NEW", @@ -1168,14 +1169,14 @@ All of these subjects are found as constants in the NATS Server source, so for e Many API calls that do not have specific structured data as response will reply with either `+OK` or `-ERR `, in the reference below this is what is known as `Standard OK/ERR`. Even those that reply with JSON structures can reply with `-ERR ` instead of the expected JSON, so you need to check for that first before attempting to parse the result. ```nohighlight -$ nats-req '$JS.STREAM.INFO' nonexisting -Published [$JS.STREAM.INFO] : 'nonexisting' +$ nats req '$JS.STREAM.INFO' nonexisting +Published 11 bytes to $JS.STREAM.INFO Received [_INBOX.lcWgjX2WgJLxqepU0K9pNf.mpBW9tHK] : '-ERR stream not found' ``` ```nohighlight -$ nats-req '$JS.STREAM.INFO' ORDERS -Published [$JS.STREAM.INFO] : 'ORDERS' +$ nats req '$JS.STREAM.INFO' ORDERS +Published 6 bytes to $JS.STREAM.INFO Received [_INBOX.fwqdpoWtG8XFXHKfqhQDVA.vBecyWmF] : '{ "config": { "name": "ORDERS", @@ -1185,11 +1186,11 @@ Received [_INBOX.fwqdpoWtG8XFXHKfqhQDVA.vBecyWmF] : '{ ### Admin API -All the admin actions the `jsm` CLI can do falls in the sections below. +All the admin actions the `nats` CLI can do falls in the sections below. Subjects that and in `T` like `server.JetStreamCreateConsumerT` are formats and would need to have the Stream Name and in some cases also the Consumer name interpolated into them. In this case `t := fmt.Sprintf(server.JetStreamCreateConsumerT, streamName)` to get the final subject. -The command `jsm events` will show you an audit log of all API access events which includes the full content of each admin request, use this to view the structure of messages the `jsm` command sends. +The command `nats events` will show you an audit log of all API access events which includes the full content of each admin request, use this to view the structure of messages the `nats` command sends. #### General Info @@ -1288,8 +1289,8 @@ In all the Synadia maintained API's you can simply do `msg.Respond(nil)` (or lan If you have a pull-based Consumer you can send a standard NATS Request to `$JS.STREAM..CONSUMER..NEXT`, here the format is defined in `server.JetStreamRequestNextT` and requires populating using `fmt.Sprintf()`. ```nohighlight -$ nats-req '$JS.STREAM.ORDERS.CONSUMER.test.NEXT' '1' -Published [$JS.STREAM.ORDERS.CONSUMER.test.NEXT] : '1' +$ nats req '$JS.STREAM.ORDERS.CONSUMER.test.NEXT' '1' +Published 1 bytes to $JS.STREAM.ORDERS.CONSUMER.test.NEXT Received [js.1] : 'message 1' ``` @@ -1300,8 +1301,8 @@ Here we ask for just 1 message - `nats-req` only shows 1 - but you can fetch a b If you know the Stream sequence of a message you can fetch it directly, this does not support acks. Do a Request() to `$JS.STREAM.ORDERS.MSG.BYSEQ` sending it the message sequence as payload. Here the prefix is defined in `server.JetStreamMsgBySeqT` which also requires populating using `fmt.Sprintf()`. ```nohighlight -$ nats-req '$JS.STREAM.ORDERS.MSG.BYSEQ' '1' -Published [$JS.STREAM.ORDERS.MSG.BYSEQ] : '1' +$ nats req '$JS.STREAM.ORDERS.MSG.BYSEQ' '1' +Published 1 bytes to $JS.STREAM.ORDERS.MSG.BYSEQ Received [_INBOX.cJrbzPJfZrq8NrFm1DsZuH.k91Gb4xM] : '{ "Subject": "js.1", "Data": "MQ==", diff --git a/go.mod b/go.mod index 44e7f51..f7d9f24 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,10 @@ require ( github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/dustin/go-humanize v1.0.0 github.com/google/go-cmp v0.4.0 - github.com/nats-io/nats-server/v2 v2.1.5-0.20200218040645-a9bebc145396 + github.com/guptarohit/asciigraph v0.4.1 + github.com/nats-io/nats-server/v2 v2.1.5-0.20200224190004-582c33995fa5 github.com/nats-io/nats.go v1.9.1 github.com/xlab/tablewriter v0.0.0-20160610135559-80b567a11ad5 - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) diff --git a/go.sum b/go.sum index 8f5783d..0c1054f 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/guptarohit/asciigraph v0.4.1 h1:YHmCMN8VH81BIUIgTg2Fs3B52QDxNZw2RQ6j5pGoSxo= +github.com/guptarohit/asciigraph v0.4.1/go.mod h1:9fYEfE5IGJGxlP1B+w8wHFy7sNZMhPtn59f0RLtpRFM= github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ= github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -34,12 +36,8 @@ github.com/nats-io/jwt v0.3.0 h1:xdnzwFETV++jNc4W1mw//qFyJGb2ABOombmZJQS4+Qo= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.5-0.20200211194714-a27d1068905e h1:Oc2qVue0d8CiGzm8ZI66J4BeotA5PbtM18RRIr7Dvu0= -github.com/nats-io/nats-server/v2 v2.1.5-0.20200211194714-a27d1068905e/go.mod h1:FBtPk+0dj8JdsLJ1eiXh7OLXnJiwjO6VpJkzjYo7lro= -github.com/nats-io/nats-server/v2 v2.1.5-0.20200217135231-ab456c13f897 h1:0fD4rgxs5FrQdbsbaUiJ3JmkRnQqFKnwtra1RbXZZ0I= -github.com/nats-io/nats-server/v2 v2.1.5-0.20200217135231-ab456c13f897/go.mod h1:FBtPk+0dj8JdsLJ1eiXh7OLXnJiwjO6VpJkzjYo7lro= -github.com/nats-io/nats-server/v2 v2.1.5-0.20200218040645-a9bebc145396 h1:/6bZkgGacqNEU5jkDTn1ZCFj/awvYyRPUfXSDmLBW2U= -github.com/nats-io/nats-server/v2 v2.1.5-0.20200218040645-a9bebc145396/go.mod h1:FBtPk+0dj8JdsLJ1eiXh7OLXnJiwjO6VpJkzjYo7lro= +github.com/nats-io/nats-server/v2 v2.1.5-0.20200224190004-582c33995fa5 h1:VtTGWNFc/b7u+uQDGCZ6MiFIK7TdiaJWd0yqsNWnwKE= +github.com/nats-io/nats-server/v2 v2.1.5-0.20200224190004-582c33995fa5/go.mod h1:FBtPk+0dj8JdsLJ1eiXh7OLXnJiwjO6VpJkzjYo7lro= github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0 h1:qMd4+pRHgdr1nAClu+2h/2a5F2TmKcCzjCDazVgRoX4= diff --git a/internal/jsch/backup.go b/internal/jsch/backup.go index f21fcbb..161e3b2 100644 --- a/internal/jsch/backup.go +++ b/internal/jsch/backup.go @@ -63,7 +63,7 @@ func BackupJetStreamConfiguration(backupDir string) error { } // RestoreJetStreamConfiguration restores the configuration from a backup made by BackupJetStreamConfiguration -func RestoreJetStreamConfiguration(backupDir string) error { +func RestoreJetStreamConfiguration(backupDir string, update bool) error { backups := []*BackupData{} // load all backups files since we have to do them in a specific order @@ -117,7 +117,7 @@ func RestoreJetStreamConfiguration(backupDir string) error { return nil } - err = eachOfType("stream", restoreStream) + err = eachOfType("stream", func(d *BackupData) error { return restoreStream(d, update) }) if err != nil { return err } @@ -136,7 +136,7 @@ func RestoreJetStreamConfiguration(backupDir string) error { } // RestoreJetStreamConfigurationFile restores a single file from a backup made by BackupJetStreamConfiguration -func RestoreJetStreamConfigurationFile(path string) error { +func RestoreJetStreamConfigurationFile(path string, update bool) error { log.Printf("Reading file %s", path) b, err := ioutil.ReadFile(path) if err != nil { @@ -155,7 +155,7 @@ func RestoreJetStreamConfigurationFile(path string) error { switch bd.Type { case "stream": - err = restoreStream(bd) + err = restoreStream(bd, update) case "consumer": err = restoreConsumer(bd) case "stream_template": @@ -167,7 +167,7 @@ func RestoreJetStreamConfigurationFile(path string) error { return err } -func restoreStream(backup *BackupData) error { +func restoreStream(backup *BackupData, update bool) error { if backup.Type != "stream" { return fmt.Errorf("cannot restore backup of type %q as Stream", backup.Type) } @@ -183,8 +183,29 @@ func restoreStream(backup *BackupData) error { return nil } - log.Printf("Restoring Stream %s", sc.Name) - _, err = NewStreamFromDefault(sc.Name, sc) + known, err := IsKnownStream(sc.Name) + if err != nil { + return err + } + + switch { + case known && !update: + err = fmt.Errorf("stream %s exists and update was not specified", sc.Name) + case known && update: + var stream *Stream + stream, err = LoadStream(sc.Name) + if err != nil { + return err + } + + log.Printf("Updating Stream %s configuration", sc.Name) + err = stream.UpdateConfiguration(sc) + + default: + log.Printf("Restoring Stream %s", sc.Name) + _, err = NewStreamFromDefault(sc.Name, sc) + } + return err } diff --git a/internal/jsch/consumers.go b/internal/jsch/consumers.go index f953376..8898549 100644 --- a/internal/jsch/consumers.go +++ b/internal/jsch/consumers.go @@ -88,7 +88,7 @@ func createDurableConsumer(request server.CreateConsumerRequest) (name string, e return "", err } - response, err := nc.Request(fmt.Sprintf(server.JetStreamCreateConsumerT, request.Stream, request.Config.Durable), jreq, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamCreateConsumerT, request.Stream, request.Config.Durable), jreq, timeout) if err != nil { return "", err } @@ -106,7 +106,7 @@ func createEphemeralConsumer(request server.CreateConsumerRequest) (name string, return "", err } - response, err := nc.Request(fmt.Sprintf(server.JetStreamCreateEphemeralConsumerT, request.Stream), jreq, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamCreateEphemeralConsumerT, request.Stream), jreq, timeout) if err != nil { return "", err } @@ -182,7 +182,7 @@ func loadConfigForConsumer(consumer *Consumer) (err error) { } func loadConsumerInfo(s string, c string) (info server.ConsumerInfo, err error) { - response, err := nc.Request(fmt.Sprintf(server.JetStreamConsumerInfoT, s, c), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamConsumerInfoT, s, c), nil, timeout) if err != nil { return info, err } @@ -368,6 +368,11 @@ func (c *Consumer) Subscribe(h func(*nats.Msg)) (sub *nats.Subscription, err err return nil, fmt.Errorf("consumer %s > %s is not push-based", c.stream, c.name) } + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.Subscribe(c.DeliverySubject(), h) } @@ -377,6 +382,11 @@ func (c *Consumer) ChanSubscribe(ch chan *nats.Msg) (sub *nats.Subscription, err return nil, fmt.Errorf("consumer %s > %s is not push-based", c.stream, c.name) } + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.ChanSubscribe(c.DeliverySubject(), ch) } @@ -386,6 +396,11 @@ func (c *Consumer) ChanQueueSubscribe(group string, ch chan *nats.Msg) (sub *nat return nil, fmt.Errorf("consumer %s > %s is not push-based", c.stream, c.name) } + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.ChanQueueSubscribe(c.DeliverySubject(), group, ch) } @@ -395,6 +410,11 @@ func (c *Consumer) SubscribeSync() (sub *nats.Subscription, err error) { return nil, fmt.Errorf("consumer %s > %s is not push-based", c.stream, c.name) } + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.SubscribeSync(c.DeliverySubject()) } @@ -404,6 +424,11 @@ func (c *Consumer) QueueSubscribe(queue string, h func(*nats.Msg)) (sub *nats.Su return nil, fmt.Errorf("consumer %s > %s is not push-based", c.stream, c.name) } + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.QueueSubscribe(c.DeliverySubject(), queue, h) } @@ -413,6 +438,11 @@ func (c *Consumer) QueueSubscribeSync(queue string) (sub *nats.Subscription, err return nil, fmt.Errorf("consumer %s > %s is not push-based", c.stream, c.name) } + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.QueueSubscribeSync(c.DeliverySubject(), queue) } @@ -422,12 +452,17 @@ func (c *Consumer) QueueSubscribeSyncWithChan(queue string, ch chan *nats.Msg) ( return nil, fmt.Errorf("consumer %s > %s is not push-based", c.stream, c.name) } + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.QueueSubscribeSyncWithChan(c.DeliverySubject(), queue, ch) } // NextMsgs retrieves the next n messages func NextMsgs(stream string, consumer string, n int) (m *nats.Msg, err error) { - return nc.Request(NextSubject(stream, consumer), []byte(strconv.Itoa(n)), Timeout) + return nrequest(NextSubject(stream, consumer), []byte(strconv.Itoa(n)), timeout) } // NextMsgs retrieves the next n messages @@ -436,7 +471,7 @@ func (c *Consumer) NextMsgs(n int) (m *nats.Msg, err error) { return nil, fmt.Errorf("consumer %s > %s is not pull-based", c.stream, c.name) } - return nc.Request(c.NextSubject(), []byte(strconv.Itoa(n)), Timeout) + return nrequest(c.NextSubject(), []byte(strconv.Itoa(n)), timeout) } // NextMsg retrieves the next message @@ -461,7 +496,7 @@ func (c *Consumer) Configuration() (config server.ConsumerConfig) { // Delete deletes the Consumer, after this the Consumer object should be disposed func (c *Consumer) Delete() (err error) { - response, err := nc.Request(fmt.Sprintf(server.JetStreamDeleteConsumerT, c.StreamName(), c.Name()), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamDeleteConsumerT, c.StreamName(), c.Name()), nil, timeout) if err != nil { return err } diff --git a/internal/jsch/jsch.go b/internal/jsch/jsch.go index 45f948f..30e930e 100644 --- a/internal/jsch/jsch.go +++ b/internal/jsch/jsch.go @@ -30,25 +30,46 @@ import ( "fmt" "sort" "strings" + "sync" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) -// Timeout used when performing JetStream requests -var Timeout = 5 * time.Second - +var timeout = 5 * time.Second var nc *nats.Conn +var mu sync.Mutex + +// Connect connects to NATS and configures it to use the connection in future interaction +func Connect(servers string, opts ...nats.Option) (err error) { + mu.Lock() + defer mu.Unlock() + + nc, err = nats.Connect(servers, opts...) + + return err +} + +// SetTimeout sets the timeout for requests to JetStream +func SetTimeout(t time.Duration) { + mu.Lock() + defer mu.Unlock() + + timeout = t +} // SetConnection sets the connection used to perform requests func SetConnection(c *nats.Conn) { + mu.Lock() + defer mu.Unlock() + nc = c } // IsJetStreamEnabled determines if JetStream is enabled for the current account func IsJetStreamEnabled() bool { - _, err := nc.Request(server.JetStreamEnabled, nil, Timeout) + _, err := nrequest(server.JetStreamEnabled, nil, timeout) return err == nil } @@ -112,7 +133,7 @@ func IsKnownConsumer(stream string, consumer string) (bool, error) { // JetStreamAccountInfo retrieves information about the current account limits and more func JetStreamAccountInfo() (info server.JetStreamAccountStats, err error) { - response, err := nc.Request(server.JetStreamInfo, nil, Timeout) + response, err := nrequest(server.JetStreamInfo, nil, timeout) if err != nil { return info, err } @@ -133,7 +154,7 @@ func JetStreamAccountInfo() (info server.JetStreamAccountStats, err error) { func StreamNames() (streams []string, err error) { streams = []string{} - response, err := nc.Request(server.JetStreamListStreams, nil, Timeout) + response, err := nrequest(server.JetStreamListStreams, nil, timeout) if err != nil { return streams, err } @@ -156,7 +177,7 @@ func StreamNames() (streams []string, err error) { func StreamTemplateNames() (templates []string, err error) { templates = []string{} - response, err := nc.Request(server.JetStreamListTemplates, nil, Timeout) + response, err := nrequest(server.JetStreamListTemplates, nil, timeout) if err != nil { return templates, err } @@ -179,7 +200,7 @@ func StreamTemplateNames() (templates []string, err error) { func ConsumerNames(stream string) (consumers []string, err error) { consumers = []string{} - response, err := nc.Request(fmt.Sprintf(server.JetStreamConsumersT, stream), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamConsumersT, stream), nil, timeout) if err != nil { return consumers, err } @@ -236,23 +257,29 @@ func EachStreamTemplate(cb func(*StreamTemplate)) (err error) { return nil } -// Subscribe subscribes a consumer, creating it if not present from a template configuration modified by opts. Stream should exist. See nats.Subscribe -// -// TODO: I dont really think this kind of thing is a good idea, but its awfully verbose without it so I suspect we will need to cater for this -func Subscribe(stream string, consumer string, cb func(*nats.Msg), deflt server.ConsumerConfig, opts ...ConsumerOption) (*nats.Subscription, error) { - c, err := LoadOrNewConsumerFromDefault(stream, consumer, deflt, opts...) - if err != nil { - return nil, err - } - - return c.Subscribe(cb) -} - // Flush flushes the underlying NATS connection func Flush() error { + nc := nconn() + + if nc == nil { + return fmt.Errorf("nats connection is not set, use SetConnection()") + } + return nc.Flush() } -func NATSConn() *nats.Conn { +func nconn() *nats.Conn { + mu.Lock() + defer mu.Unlock() + return nc } + +func nrequest(subj string, data []byte, timeout time.Duration) (*nats.Msg, error) { + nc := nconn() + if nc == nil { + return nil, fmt.Errorf("nats connection is not set, use SetConnection()") + } + + return nc.Request(subj, data, timeout) +} diff --git a/internal/jsch/jsch_test.go b/internal/jsch/jsch_test.go index a74cf4c..4d86fca 100644 --- a/internal/jsch/jsch_test.go +++ b/internal/jsch/jsch_test.go @@ -14,8 +14,6 @@ package jsch_test import ( - "context" - "fmt" "io/ioutil" "testing" "time" @@ -242,50 +240,6 @@ func TestEachStream(t *testing.T) { } } -func TestSubscribeAndCreate(t *testing.T) { - srv, nc := startJSServer(t) - defer srv.Shutdown() - defer nc.Flush() - - s, err := jsch.LoadOrNewStream("ORDERS", jsch.Subjects("ORDERS.*"), jsch.MaxAge(24*365*time.Hour), jsch.MemoryStorage()) - checkErr(t, err, "ORDERS create failed") - s.Purge() - - for i := 0; i < 10; i++ { - nc.Request("ORDERS.new", []byte(fmt.Sprintf("order %d", i)), 5*time.Second) - } - - ctr := 0 - done := make(chan struct{}) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - h := func(m *nats.Msg) { - m.Respond(nil) - - ctr++ - if ctr == 10 { - done <- struct{}{} - } - } - - _, err = jsch.Subscribe("ORDERS", "NEW", h, jsch.DefaultConsumer, jsch.DurableName("NEW"), jsch.DeliverySubject("out.new"), jsch.MaxDeliveryAttempts(5), jsch.FilterStreamBySubject("ORDERS.new")) - checkErr(t, err, "subscribe failed") - - names, err := s.ConsumerNames() - checkErr(t, err, "could not get consumer names") - - if len(names) != 1 || names[0] != "NEW" { - t.Fatalf("incorrect consumer created, expected [NEW] got %v", names) - } - - select { - case <-done: - case <-ctx.Done(): - t.Fatal("timeout") - } -} - func TestIsKnownStreamTemplate(t *testing.T) { srv, _ := startJSServer(t) defer srv.Shutdown() diff --git a/internal/jsch/streams.go b/internal/jsch/streams.go index 1f53312..8f875a6 100644 --- a/internal/jsch/streams.go +++ b/internal/jsch/streams.go @@ -72,7 +72,7 @@ func NewStreamFromDefault(name string, dflt server.StreamConfig, opts ...StreamO return nil, err } - response, err := nc.Request(fmt.Sprintf(server.JetStreamCreateStreamT, name), jreq, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamCreateStreamT, name), jreq, timeout) if err != nil { return nil, err } @@ -144,7 +144,7 @@ func loadConfigForStream(stream *Stream) (err error) { } func loadStreamInfo(stream string) (info *server.StreamInfo, err error) { - response, err := nc.Request(fmt.Sprintf(server.JetStreamStreamInfoT, stream), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamStreamInfoT, stream), nil, timeout) if err != nil { return nil, err } @@ -265,7 +265,7 @@ func (s *Stream) UpdateConfiguration(cfg server.StreamConfig, opts ...StreamOpti return err } - response, err := nc.Request(fmt.Sprintf(server.JetStreamUpdateStreamT, s.Name()), jcfg, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamUpdateStreamT, s.Name()), jcfg, timeout) if err != nil { return err } @@ -309,7 +309,7 @@ func (s *Stream) LoadOrNewConsumerFromDefault(name string, deflt server.Consumer // ConsumerNames is a list of all known consumers for this Stream func (s *Stream) ConsumerNames() (names []string, err error) { - response, err := nc.Request(fmt.Sprintf(server.JetStreamConsumersT, s.Name()), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamConsumersT, s.Name()), nil, timeout) if err != nil { return names, err } @@ -363,7 +363,7 @@ func (s *Stream) State() (stats server.StreamState, err error) { // Delete deletes the Stream, after this the Stream object should be disposed func (s *Stream) Delete() error { - response, err := nc.Request(fmt.Sprintf(server.JetStreamDeleteStreamT, s.Name()), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamDeleteStreamT, s.Name()), nil, timeout) if err != nil { return err } @@ -377,7 +377,7 @@ func (s *Stream) Delete() error { // Purge deletes all messages from the Stream func (s *Stream) Purge() error { - response, err := nc.Request(fmt.Sprintf(server.JetStreamPurgeStreamT, s.Name()), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamPurgeStreamT, s.Name()), nil, timeout) if err != nil { return err } @@ -391,7 +391,7 @@ func (s *Stream) Purge() error { // LoadMessage loads a message from the message set by its sequence number func (s *Stream) LoadMessage(seq int) (msg server.StoredMsg, err error) { - response, err := nc.Request(fmt.Sprintf(server.JetStreamMsgBySeqT, s.Name()), []byte(strconv.Itoa(seq)), Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamMsgBySeqT, s.Name()), []byte(strconv.Itoa(seq)), timeout) if err != nil { return server.StoredMsg{}, err } @@ -411,7 +411,7 @@ func (s *Stream) LoadMessage(seq int) (msg server.StoredMsg, err error) { // DeleteMessage deletes a specific message from the Stream by overwriting it with random data func (s *Stream) DeleteMessage(seq int) (err error) { - response, err := nc.Request(fmt.Sprintf(server.JetStreamDeleteMsgT, s.Name()), []byte(strconv.Itoa(seq)), Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamDeleteMsgT, s.Name()), []byte(strconv.Itoa(seq)), timeout) if err != nil { return err } diff --git a/internal/jsch/templates.go b/internal/jsch/templates.go index 9dbff6b..5de0f1f 100644 --- a/internal/jsch/templates.go +++ b/internal/jsch/templates.go @@ -30,7 +30,7 @@ func NewStreamTemplate(name string, maxStreams uint32, config server.StreamConfi return nil, err } - response, err := nc.Request(fmt.Sprintf(server.JetStreamCreateTemplateT, name), jreq, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamCreateTemplateT, name), jreq, timeout) if err != nil { return nil, err } @@ -64,7 +64,7 @@ func LoadStreamTemplate(name string) (template *StreamTemplate, err error) { } func loadConfigForStreamTemplate(template *StreamTemplate) (err error) { - response, err := nc.Request(fmt.Sprintf(server.JetStreamTemplateInfoT, template.Name()), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamTemplateInfoT, template.Name()), nil, timeout) if err != nil { return err } @@ -87,7 +87,7 @@ func loadConfigForStreamTemplate(template *StreamTemplate) (err error) { // Delete deletes the StreamTemplate, after this the StreamTemplate object should be disposed func (t *StreamTemplate) Delete() error { - response, err := nc.Request(fmt.Sprintf(server.JetStreamDeleteTemplateT, t.Name()), nil, Timeout) + response, err := nrequest(fmt.Sprintf(server.JetStreamDeleteTemplateT, t.Name()), nil, timeout) if err != nil { return err } diff --git a/jsm/restore_command.go b/jsm/restore_command.go index a64a633..af6abab 100644 --- a/jsm/restore_command.go +++ b/jsm/restore_command.go @@ -36,8 +36,8 @@ func (c *restoreCmd) restoreAction(_ *kingpin.ParseContext) error { } if c.backupDir != "" { - return jsch.RestoreJetStreamConfiguration(c.backupDir) + return jsch.RestoreJetStreamConfiguration(c.backupDir, false) } - return jsch.RestoreJetStreamConfigurationFile(c.file) + return jsch.RestoreJetStreamConfigurationFile(c.file, false) } diff --git a/jsm/util.go b/jsm/util.go index 9432b6d..43283c4 100644 --- a/jsm/util.go +++ b/jsm/util.go @@ -269,7 +269,7 @@ func prepareHelper(servers string, opts ...nats.Option) (*nats.Conn, error) { } if timeout != 0 { - jsch.Timeout = timeout + jsch.SetTimeout(timeout) } jsch.SetConnection(nc) diff --git a/nats/account_command.go b/nats/account_command.go new file mode 100644 index 0000000..ef203eb --- /dev/null +++ b/nats/account_command.go @@ -0,0 +1,61 @@ +// Copyright 2019 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + + "github.com/dustin/go-humanize" + "gopkg.in/alecthomas/kingpin.v2" + + "github.com/nats-io/jetstream/internal/jsch" +) + +type actCmd struct { + json bool +} + +func configureActCommand(app *kingpin.Application) { + c := &actCmd{} + act := app.Command("account", "JetStream account information") + act.Command("info", "Account information").Alias("nfo").Action(c.infoAction) + act.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) +} + +func (c *actCmd) infoAction(pc *kingpin.ParseContext) error { + _, err := prepareHelper(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + + info, err := jsch.JetStreamAccountInfo() + kingpin.FatalIfError(err, "could not request account info") + + if c.json { + printJSON(info) + return nil + } + + fmt.Println("JetStream Account Information:") + fmt.Println() + fmt.Printf(" Memory: %s of %s\n", humanize.IBytes(info.Memory), humanize.IBytes(uint64(info.Limits.MaxMemory))) + fmt.Printf(" Storage: %s of %s\n", humanize.IBytes(info.Store), humanize.IBytes(uint64(info.Limits.MaxStore))) + + if info.Limits.MaxStreams == -1 { + fmt.Printf(" Streams: %d of Unlimited\n", info.Streams) + } else { + fmt.Printf("Message Sets: %d of %d\n", info.Streams, info.Limits.MaxStreams) + } + fmt.Println() + + return nil +} diff --git a/nats/backup_command.go b/nats/backup_command.go new file mode 100644 index 0000000..a97f111 --- /dev/null +++ b/nats/backup_command.go @@ -0,0 +1,27 @@ +package main + +import ( + "gopkg.in/alecthomas/kingpin.v2" + + "github.com/nats-io/jetstream/internal/jsch" +) + +type backupCmd struct { + outDir string +} + +func configureBackupCommand(app *kingpin.Application) { + c := &backupCmd{} + + backup := app.Command("backup", "JetStream configuration backup utility").Action(c.backupAction) + backup.Arg("output", "Directory to write backup to").Required().StringVar(&c.outDir) +} + +func (c *backupCmd) backupAction(_ *kingpin.ParseContext) error { + _, err := prepareHelper(servers, natsOpts()...) + if err != nil { + return err + } + + return jsch.BackupJetStreamConfiguration(c.outDir) +} diff --git a/nats/bench_command.go b/nats/bench_command.go new file mode 100644 index 0000000..b08f7bd --- /dev/null +++ b/nats/bench_command.go @@ -0,0 +1,148 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io/ioutil" + "log" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/bench" + "gopkg.in/alecthomas/kingpin.v2" +) + +type benchCmd struct { + subject string + numPubs int + numSubs int + numMsg int + msgSize int + csvFile string +} + +func configureBenchCommand(app *kingpin.Application) { + c := &benchCmd{} + bench := app.Command("bench", "Benchmark Utility").Action(c.bench) + bench.Arg("subject", "Subject to use for testing").Required().StringVar(&c.subject) + bench.Flag("pub", "Number of concurrent publishers").Default("1").IntVar(&c.numPubs) + bench.Flag("sub", "Number of concurrent subscribers").Default("0").IntVar(&c.numSubs) + bench.Flag("msgs", "Number of messages to publish").Default("100000").IntVar(&c.numMsg) + bench.Flag("size", "Size of the test messages").Default("128").IntVar(&c.msgSize) + bench.Flag("csv", "Save benchmark data to CSV file").StringVar(&c.csvFile) +} + +func (c *benchCmd) bench(_ *kingpin.ParseContext) error { + if c.numMsg <= 0 { + return fmt.Errorf("number of messages should be greater than 0") + } + + bm := bench.NewBenchmark("NATS", c.numSubs, c.numPubs) + + startwg := &sync.WaitGroup{} + donewg := &sync.WaitGroup{} + + for i := 0; i < c.numSubs; i++ { + nc, err := nats.Connect(servers, natsOpts()...) + if err != nil { + return fmt.Errorf("nats connection %d failed: %s", i, err) + } + defer nc.Close() + + startwg.Add(1) + donewg.Add(1) + + go c.runSubscriber(bm, nc, startwg, donewg) + } + startwg.Wait() + + pubCounts := bench.MsgsPerClient(c.numMsg, c.numPubs) + for i := 0; i < c.numPubs; i++ { + nc, err := nats.Connect(servers, natsOpts()...) + if err != nil { + return fmt.Errorf("nats connection %d failed: %s", i, err) + } + defer nc.Close() + + startwg.Add(1) + donewg.Add(1) + + go c.runPublisher(bm, nc, startwg, donewg, pubCounts[i]) + } + + log.Printf("Starting benchmark [msgs=%d, msgsize=%d, pubs=%d, subs=%d]\n", c.numMsg, c.msgSize, c.numPubs, c.numSubs) + + startwg.Wait() + donewg.Wait() + + bm.Close() + + fmt.Println(bm.Report()) + + if c.csvFile != "" { + csv := bm.CSV() + ioutil.WriteFile(c.csvFile, []byte(csv), 0644) + fmt.Printf("Saved metric data in csv file %s\n", c.csvFile) + } + + return nil +} + +func (c *benchCmd) runPublisher(bm *bench.Benchmark, nc *nats.Conn, startwg *sync.WaitGroup, donewg *sync.WaitGroup, numMsg int) { + startwg.Done() + + var msg []byte + if c.msgSize > 0 { + msg = make([]byte, c.msgSize) + } + + start := time.Now() + + for i := 0; i < c.numMsg; i++ { + nc.Publish(c.subject, msg) + } + nc.Flush() + bm.AddPubSample(bench.NewSample(numMsg, c.msgSize, start, time.Now(), nc)) + + donewg.Done() +} + +func (c *benchCmd) runSubscriber(bm *bench.Benchmark, nc *nats.Conn, startwg *sync.WaitGroup, donewg *sync.WaitGroup) { + received := 0 + ch := make(chan time.Time, 2) + + sub, _ := nc.Subscribe(c.subject, func(msg *nats.Msg) { + received++ + if received == 1 { + ch <- time.Now() + } + if received >= c.numMsg { + ch <- time.Now() + } + }) + + sub.SetPendingLimits(-1, -1) + nc.Flush() + startwg.Done() + + start := <-ch + end := <-ch + + bm.AddSubSample(bench.NewSample(c.numMsg, c.msgSize, start, end, nc)) + + nc.Close() + donewg.Done() +} diff --git a/nats/consumer_command.go b/nats/consumer_command.go new file mode 100644 index 0000000..b2b558c --- /dev/null +++ b/nats/consumer_command.go @@ -0,0 +1,591 @@ +// Copyright 2019 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/AlecAivazis/survey/v2" + api "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "gopkg.in/alecthomas/kingpin.v2" + + "github.com/nats-io/jetstream/internal/jsch" +) + +type consumerCmd struct { + consumer string + stream string + json bool + force bool + ack bool + raw bool + destination string + + maxDeliver int + pull bool + replayPolicy string + startPolicy string + ackPolicy string + ackWait time.Duration + samplePct int + filterSubject string + delivery string + ephemeral bool +} + +func configureConsumerCommand(app *kingpin.Application) { + c := &consumerCmd{} + + cons := app.Command("consumer", "JetStream Consumer management").Alias("con").Alias("obs").Alias("c") + + consInfo := cons.Command("info", "Consumer information").Alias("nfo").Action(c.infoAction) + consInfo.Arg("stream", "Stream name").StringVar(&c.stream) + consInfo.Arg("consumer", "Consumer name").StringVar(&c.consumer) + consInfo.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + + consLs := cons.Command("ls", "List known Consumers").Alias("list").Action(c.lsAction) + consLs.Arg("stream", "Stream name").StringVar(&c.stream) + consLs.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + + consRm := cons.Command("rm", "Removes a Consumer").Alias("delete").Alias("del").Action(c.rmAction) + consRm.Arg("stream", "Stream name").StringVar(&c.stream) + consRm.Arg("consumer", "Consumer name").StringVar(&c.consumer) + consRm.Flag("force", "Force removal without prompting").Short('f').BoolVar(&c.force) + + addCreateFlags := func(f *kingpin.CmdClause) { + f.Flag("target", "Push based delivery target subject").StringVar(&c.delivery) + f.Flag("filter", "Filter Stream by subjects").Default("_unset_").StringVar(&c.filterSubject) + f.Flag("replay", "Replay Policy (instant, original)").EnumVar(&c.replayPolicy, "instant", "original") + f.Flag("deliver", "Start policy (all, last, 1h, msg sequence)").StringVar(&c.startPolicy) + f.Flag("ack", "Acknowledgement policy (none, all, explicit)").StringVar(&c.ackPolicy) + f.Flag("wait", "Acknowledgement waiting time").Default("-1s").DurationVar(&c.ackWait) + f.Flag("sample", "Percentage of requests to sample for monitoring purposes").Default("-1").IntVar(&c.samplePct) + f.Flag("ephemeral", "Create an ephemeral Consumer").Default("false").BoolVar(&c.ephemeral) + f.Flag("pull", "Deliver messages in 'pull' mode").BoolVar(&c.pull) + f.Flag("max-deliver", "Maximum amount of times a message will be delivered").IntVar(&c.maxDeliver) + } + + consAdd := cons.Command("add", "Creates a new Consumer").Alias("create").Alias("new").Action(c.createAction) + consAdd.Arg("stream", "Stream name").StringVar(&c.stream) + consAdd.Arg("consumer", "Consumer name").StringVar(&c.consumer) + addCreateFlags(consAdd) + + consCp := cons.Command("copy", "Creates a new Consumer based on the configuration of another").Alias("cp").Action(c.cpAction) + consCp.Arg("stream", "Stream name").Required().StringVar(&c.stream) + consCp.Arg("source", "Source Consumer name").Required().StringVar(&c.consumer) + consCp.Arg("destination", "Destination Consumer name").Required().StringVar(&c.destination) + addCreateFlags(consCp) + + consNext := cons.Command("next", "Retrieves messages from Pull Consumers without interactive prompts").Action(c.nextAction) + consNext.Arg("stream", "Stream name").StringVar(&c.stream) + consNext.Arg("consumer", "Consumer name").StringVar(&c.consumer) + consNext.Flag("ack", "Acknowledge received message").Default("true").BoolVar(&c.ack) + consNext.Flag("raw", "Show only the message").Short('r').BoolVar(&c.raw) + + consSub := cons.Command("sub", "Retrieves messages from Consumers").Action(c.subAction) + consSub.Arg("stream", "Stream name").StringVar(&c.stream) + consSub.Arg("consumer", "Consumer name").StringVar(&c.consumer) + consSub.Flag("ack", "Acknowledge received message").Default("true").BoolVar(&c.ack) + consSub.Flag("raw", "Show only the message").Short('r').BoolVar(&c.raw) +} + +func (c *consumerCmd) rmAction(_ *kingpin.ParseContext) error { + c.connectAndSetup(true, true) + + if !c.force { + ok, err := askConfirmation(fmt.Sprintf("Really delete Consumer %s > %s", c.stream, c.consumer), false) + kingpin.FatalIfError(err, "could not obtain confirmation") + + if !ok { + return nil + } + } + + consumer, err := jsch.LoadConsumer(c.stream, c.consumer) + kingpin.FatalIfError(err, "could not load Consumer") + + return consumer.Delete() +} + +func (c *consumerCmd) lsAction(pc *kingpin.ParseContext) error { + c.connectAndSetup(true, false) + + stream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not load Consumers") + + consumers, err := stream.ConsumerNames() + kingpin.FatalIfError(err, "could not load Consumers") + + if c.json { + err = printJSON(consumers) + kingpin.FatalIfError(err, "could not display Consumers") + return nil + } + + if len(consumers) == 0 { + fmt.Println("No Consumers defined") + return nil + } + + fmt.Printf("Consumers for Stream %s:\n", c.stream) + fmt.Println() + for _, sc := range consumers { + fmt.Printf("\t%s\n", sc) + } + fmt.Println() + + return nil +} + +func (c *consumerCmd) infoAction(pc *kingpin.ParseContext) error { + c.connectAndSetup(true, true) + + consumer, err := jsch.LoadConsumer(c.stream, c.consumer) + kingpin.FatalIfError(err, "could not load Consumer %s > %s", c.stream, c.consumer) + + config := consumer.Configuration() + state, err := consumer.State() + kingpin.FatalIfError(err, "could not load Consumer %s > %s", c.stream, c.consumer) + + if c.json { + printJSON(api.ConsumerInfo{ + Name: consumer.Name(), + Config: config, + State: state, + }) + return nil + } + + fmt.Printf("Information for Consumer %s > %s\n", c.stream, c.consumer) + fmt.Println() + fmt.Println("Configuration:") + fmt.Println() + if config.Durable != "" { + fmt.Printf(" Durable Name: %s\n", config.Durable) + } + if config.Delivery != "" { + fmt.Printf(" Delivery Subject: %s\n", config.Delivery) + } else { + fmt.Printf(" Pull Mode: true\n") + } + if config.FilterSubject != "" { + fmt.Printf(" Filter Subject: %s\n", config.FilterSubject) + } + if config.StreamSeq != 0 { + fmt.Printf(" Start Sequence: %d\n", config.StreamSeq) + } + if !config.StartTime.IsZero() { + fmt.Printf(" Start Time: %v\n", config.StartTime) + } + if config.DeliverAll { + fmt.Printf(" Deliver All: %v\n", config.DeliverAll) + } + if config.DeliverLast { + fmt.Printf(" Deliver Last: %v\n", config.DeliverLast) + } + fmt.Printf(" Ack Policy: %s\n", config.AckPolicy.String()) + if config.AckPolicy != api.AckNone { + fmt.Printf(" Ack Wait: %v\n", config.AckWait) + } + fmt.Printf(" Replay Policy: %s\n", config.ReplayPolicy.String()) + if config.MaxDeliver != -1 { + fmt.Printf(" Maximum Deliveries: %d\n", config.MaxDeliver) + } + if config.SampleFrequency != "" { + fmt.Printf(" Sampling Rate: %s\n", config.SampleFrequency) + } + + fmt.Println() + + fmt.Println("State:") + fmt.Println() + fmt.Printf(" Last Delivered Message: Consumer sequence: %d Stream sequence: %d\n", state.Delivered.ConsumerSeq, state.Delivered.StreamSeq) + fmt.Printf(" Acknowledgment floor: Consumer sequence: %d Stream sequence: %d\n", state.AckFloor.ConsumerSeq, state.AckFloor.StreamSeq) + fmt.Printf(" Pending Messages: %d\n", len(state.Pending)) + fmt.Printf(" Redelivered Messages: %d\n", len(state.Redelivered)) + fmt.Println() + + return nil +} + +func (c *consumerCmd) replayPolicyFromString(p string) api.ReplayPolicy { + switch strings.ToLower(p) { + case "instant": + return api.ReplayInstant + case "original": + return api.ReplayOriginal + default: + kingpin.Fatalf("invalid replay policy '%s'", p) + return 0 // unreachable + } +} + +func (c *consumerCmd) ackPolicyFromString(p string) api.AckPolicy { + switch strings.ToLower(p) { + case "none": + return api.AckNone + case "all": + return api.AckAll + case "explicit": + return api.AckExplicit + default: + kingpin.Fatalf("invalid ack policy '%s'", p) + // unreachable + return 0 + } +} + +func (c *consumerCmd) sampleFreqFromString(s int) string { + if s > 100 || s < 0 { + kingpin.Fatalf("sample percent is not between 0 and 100") + } + + if s > 0 { + return strconv.Itoa(c.samplePct) + } + + return "" +} + +func (c *consumerCmd) defaultConsumer() *api.ConsumerConfig { + return &api.ConsumerConfig{ + AckPolicy: api.AckExplicit, + } +} + +func (c *consumerCmd) setStartPolicy(cfg *api.ConsumerConfig, policy string) { + if policy == "" { + return + } + + if policy == "all" { + cfg.DeliverAll = true + } else if policy == "last" { + cfg.DeliverLast = true + } else if ok, _ := regexp.MatchString("^\\d+$", policy); ok { + seq, _ := strconv.Atoi(policy) + cfg.StreamSeq = uint64(seq) + } else { + d, err := parseDurationString(policy) + kingpin.FatalIfError(err, "could not parse starting delta") + cfg.StartTime = time.Now().Add(-d) + } +} + +func (c *consumerCmd) cpAction(pc *kingpin.ParseContext) (err error) { + c.connectAndSetup(true, false) + + source, err := jsch.LoadConsumer(c.stream, c.consumer) + kingpin.FatalIfError(err, "could not load source Consumer") + + cfg := source.Configuration() + + if c.ackWait > 0 { + cfg.AckWait = c.ackWait + } + + if c.samplePct != -1 { + cfg.SampleFrequency = c.sampleFreqFromString(c.samplePct) + } + + if c.startPolicy != "" { + c.setStartPolicy(&cfg, c.startPolicy) + } + + if c.ephemeral { + cfg.Durable = "" + } else { + cfg.Durable = c.destination + } + + if c.delivery != "" { + cfg.Delivery = c.delivery + } + + if c.pull { + cfg.Delivery = "" + c.ackPolicy = "explicit" + } + + if c.ackPolicy != "" { + cfg.AckPolicy = c.ackPolicyFromString(c.ackPolicy) + } + + if c.filterSubject != "_unset_" { + cfg.FilterSubject = c.filterSubject + } + + if c.replayPolicy != "" { + cfg.ReplayPolicy = c.replayPolicyFromString(c.replayPolicy) + } + + if c.maxDeliver != 0 { + cfg.MaxDeliver = c.maxDeliver + } + + _, err = jsch.NewConsumerFromDefault(c.stream, cfg) + kingpin.FatalIfError(err, "Consumer creation failed") + + if cfg.Durable == "" { + return nil + } + + c.consumer = cfg.Durable + return c.infoAction(pc) +} + +func (c *consumerCmd) createAction(pc *kingpin.ParseContext) (err error) { + c.connectAndSetup(true, false) + cfg := c.defaultConsumer() + + if c.consumer == "" && !c.ephemeral { + err = survey.AskOne(&survey.Input{ + Message: "Consumer name", + Help: "This will be used for the name of the durable subscription to be used when referencing this Consumer later. Settable using 'name' CLI argument", + }, &c.consumer, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "could not request durable name") + } + cfg.Durable = c.consumer + + if ok, _ := regexp.MatchString(`\.|\*|>`, cfg.Durable); ok { + kingpin.Fatalf("durable name can not contain '.', '*', '>'") + } + + if !c.pull && c.delivery == "" { + err = survey.AskOne(&survey.Input{ + Message: "Delivery target", + Help: "Consumers can be in 'push' or 'pull' mode, in 'push' mode messages are dispatched in real time to a target NATS subject, this is that subject. Leaving this blank creates a 'pull' mode Consumer. Settable using --target and --pull", + }, &c.delivery) + kingpin.FatalIfError(err, "could not request delivery target") + } + + if c.ephemeral && c.delivery == "" { + kingpin.Fatalf("ephemeral Consumers has to be push-based.") + } + + cfg.Delivery = c.delivery + + // pull is always explicit + if c.delivery == "" { + c.ackPolicy = "explicit" + } + + if c.startPolicy == "" { + err = survey.AskOne(&survey.Input{ + Message: "Start policy (all, last, 1h, msg sequence)", + Help: "This controls how the Consumer starts out, does it make all messages available, only the latest, ones after a certain time or time sequence. Settable using --deliver", + }, &c.startPolicy, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "could not request start policy") + } + + c.setStartPolicy(cfg, c.startPolicy) + + if c.ackPolicy == "" { + err = survey.AskOne(&survey.Select{ + Message: "Acknowledgement policy", + Options: []string{"none", "all", "explicit"}, + Default: "none", + Help: "Messages that are not acknowledged will be redelivered at a later time. 'none' means no acknowledgement is needed only 1 delivery ever, 'all' means acknowledging message 10 will also acknowledge 0-9 and 'explicit' means each has to be acknowledged specifically. Settable using --ack", + }, &c.ackPolicy) + kingpin.FatalIfError(err, "could not ask acknowledgement policy") + } + + cfg.AckPolicy = c.ackPolicyFromString(c.ackPolicy) + if cfg.AckPolicy == api.AckNone { + cfg.MaxDeliver = -1 + } + + if c.ackWait > 0 { + cfg.AckWait = c.ackWait + } + + if c.samplePct > 0 { + if c.samplePct > 100 { + kingpin.Fatalf("sample percent is not between 0 and 100") + } + + cfg.SampleFrequency = strconv.Itoa(c.samplePct) + } + + if cfg.Delivery != "" { + if c.replayPolicy == "" { + mode := "" + err = survey.AskOne(&survey.Select{ + Message: "Replay policy", + Options: []string{"instant", "original"}, + Default: "instant", + Help: "Replay policy is the time interval at which messages are delivered to interested parties. 'instant' means deliver all as soon as possible while 'original' will match the time intervals in which messages were received, useful for replaying production traffic in development. Settable using --replay", + }, &mode) + kingpin.FatalIfError(err, "could not ask replay policy") + c.replayPolicy = mode + } + } + + if c.replayPolicy != "" { + cfg.ReplayPolicy = c.replayPolicyFromString(c.replayPolicy) + } + + if c.filterSubject == "_unset_" { + err = survey.AskOne(&survey.Input{ + Message: "Filter Stream by subject (blank for all)", + Default: "", + Help: "Stream can consume more than one subject - or a wildcard - this allows you to filter out just a single subject from all the ones entering the Stream for delivery to the Consumer. Settable using --filter", + }, &c.filterSubject) + kingpin.FatalIfError(err, "could not ask for filtering subject") + } + cfg.FilterSubject = c.filterSubject + + if c.maxDeliver == 0 && cfg.AckPolicy != api.AckNone { + err = survey.AskOne(&survey.Input{ + Message: "Maximum Allowed Deliveries", + Default: "-1", + Help: "When this is -1 unlimited attempts to deliver an un acknowledged message is made, when this is >0 it will be maximum amount of times a message is delivered after which it is ignored. Settable using --max-deliver.", + }, &c.maxDeliver) + kingpin.FatalIfError(err, "could not ask for maximum allowed deliveries") + } + + if c.maxDeliver != 0 && cfg.AckPolicy != api.AckNone { + cfg.MaxDeliver = c.maxDeliver + } + + created, err := jsch.NewConsumerFromDefault(c.stream, *cfg) + kingpin.FatalIfError(err, "Consumer creation failed: ") + + c.consumer = created.Name() + + return c.infoAction(pc) +} + +func (c *consumerCmd) getNextMsgDirect(stream string, consumer string) error { + msg, err := jsch.NextMsgs(stream, consumer, 1) + kingpin.FatalIfError(err, "could not load next message") + + if !c.raw { + info, err := jsch.ParseJSMsgMetadata(msg) + if err != nil { + fmt.Printf("--- subject: %s\n", msg.Subject) + } else { + fmt.Printf("--- subject: %s / delivered: %d / stream seq: %d / consumer seq: %d\n", msg.Subject, info.Delivered(), info.StreamSequence(), info.ConsumerSequence()) + } + + fmt.Println(string(msg.Data)) + } else { + fmt.Println(string(msg.Data)) + } + + if c.ack { + err = msg.Respond(api.AckAck) + kingpin.FatalIfError(err, "could not Acknowledge message") + jsch.Flush() + if !c.raw { + fmt.Println("\nAcknowledged message") + } + } + + return nil +} + +func (c *consumerCmd) getNextMsg(consumer *jsch.Consumer) error { + return c.getNextMsgDirect(consumer.StreamName(), consumer.Name()) +} + +func (c *consumerCmd) subscribeConsumer(consumer *jsch.Consumer) (err error) { + if !c.raw { + fmt.Printf("Subscribing to topic %s auto acknowlegement: %v\n\n", consumer.DeliverySubject(), c.ack) + fmt.Println("Consumer Info:") + fmt.Printf(" Ack Policy: %s\n", consumer.AckPolicy().String()) + if consumer.AckPolicy() != api.AckNone { + fmt.Printf(" Ack Wait: %v\n", consumer.AckWait()) + } + fmt.Println() + } + + _, err = consumer.Subscribe(func(m *nats.Msg) { + var msginfo *jsch.MsgInfo + var err error + + msginfo, err = jsch.ParseJSMsgMetadata(m) + kingpin.FatalIfError(err, "could not parse JetStream metadata") + + if !c.raw { + if msginfo != nil { + fmt.Printf("[%s] subject: %s / delivered: %d / consumer seq: %d / stream seq: %d\n", time.Now().Format("15:04:05"), m.Subject, msginfo.Delivered(), msginfo.ConsumerSequence(), msginfo.StreamSequence()) + } else { + fmt.Printf("[%s] %s reply: %s\n", time.Now().Format("15:04:05"), m.Subject, m.Reply) + } + + fmt.Printf("%s\n", string(m.Data)) + if !strings.HasSuffix(string(m.Data), "\n") { + fmt.Println() + } + } else { + fmt.Println(string(m.Data)) + } + + if c.ack { + err = m.Respond(nil) + if err != nil { + fmt.Printf("Acknowledging message via subject %s failed: %s\n", m.Reply, err) + } + } + }) + kingpin.FatalIfError(err, "could not subscribe") + + <-context.Background().Done() + + return nil +} + +func (c *consumerCmd) subAction(_ *kingpin.ParseContext) error { + c.connectAndSetup(true, true) + + consumer, err := jsch.LoadConsumer(c.stream, c.consumer) + kingpin.FatalIfError(err, "could not get Consumer info") + + switch { + case consumer.IsPullMode(): + return c.getNextMsg(consumer) + case consumer.IsPushMode(): + return c.subscribeConsumer(consumer) + default: + return fmt.Errorf("consumer %s > %s is in an unknown state", c.stream, c.consumer) + } +} + +func (c *consumerCmd) nextAction(_ *kingpin.ParseContext) error { + c.connectAndSetup(false, false) + + return c.getNextMsgDirect(c.stream, c.consumer) +} + +func (c *consumerCmd) connectAndSetup(askStream bool, askConsumer bool) { + _, err := prepareHelper(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + + if askStream { + c.stream, err = selectStream(c.stream) + kingpin.FatalIfError(err, "could not select Stream") + + if askConsumer { + c.consumer, err = selectConsumer(c.stream, c.consumer) + kingpin.FatalIfError(err, "could not select Consumer") + } + } +} diff --git a/nats/events_command.go b/nats/events_command.go new file mode 100644 index 0000000..0751af3 --- /dev/null +++ b/nats/events_command.go @@ -0,0 +1,324 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net" + "regexp" + "strconv" + "strings" + "time" + + "github.com/dustin/go-humanize" + api "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "gopkg.in/alecthomas/kingpin.v2" + + "github.com/nats-io/jetstream/internal/jsch" +) + +type eventsCmd struct { + json bool + + bodyF string + bodyFRe *regexp.Regexp + subjectF string + subjectFRe *regexp.Regexp + + metricsF bool + advisoriesF bool + allF bool + connsF bool +} + +func configureEventsCommand(app *kingpin.Application) { + c := &eventsCmd{} + events := app.Command("events", "Show Advisories and Events").Alias("event").Alias("e").Action(c.eventsAction) + events.Flag("all", "Show all events").Default("false").Short('a').BoolVar(&c.allF) + events.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + events.Flag("subject", "Filter the messages by the subject using regular expressions").Default(".").StringVar(&c.subjectF) + events.Flag("filter", "Filter across the entire event using regular expressions").Default(".").StringVar(&c.bodyF) + events.Flag("metrics", "Shows metric events (false)").Default("false").BoolVar(&c.metricsF) + events.Flag("advisories", "Shows advisory events (true)").Default("true").BoolVar(&c.advisoriesF) + events.Flag("connections", "Shows connections being opened and closed (false)").Default("false").BoolVar(&c.connsF) + +} + +func (c *eventsCmd) Printf(f string, arg ...interface{}) { + if !c.json { + fmt.Printf(f, arg...) + } +} + +func (c *eventsCmd) eventsAction(_ *kingpin.ParseContext) error { + nc, err := prepareHelper(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + + c.subjectFRe, err = regexp.Compile(strings.ToUpper(c.subjectF)) + kingpin.FatalIfError(err, "invalid subjects regular expression") + c.bodyFRe, err = regexp.Compile(strings.ToUpper(c.bodyF)) + kingpin.FatalIfError(err, "invalid body regular expression") + + if c.advisoriesF || c.allF { + c.Printf("Listening for Advisories on %s.>\n", api.JetStreamAdvisoryPrefix) + nc.Subscribe(fmt.Sprintf("%s.>", api.JetStreamAdvisoryPrefix), func(m *nats.Msg) { + c.renderAdvisory(m) + }) + } + + if c.metricsF || c.allF { + c.Printf("Listening for Metrics on %s.>\n", api.JetStreamMetricPrefix) + nc.Subscribe(fmt.Sprintf("%s.>", api.JetStreamMetricPrefix), func(m *nats.Msg) { + c.renderMetric(m) + }) + } + + if c.connsF || c.allF { + c.Printf("Listening for Client Connection events on $SYS.ACCOUNT.*.CONNECT\n") + nc.Subscribe("$SYS.ACCOUNT.*.CONNECT", func(m *nats.Msg) { + c.renderConnection(m) + }) + + c.Printf("Listening for Client Disconnection events on $SYS.ACCOUNT.*.DISCONNECT\n") + nc.Subscribe("$SYS.ACCOUNT.*.DISCONNECT", func(m *nats.Msg) { + c.renderDisconnection(m) + }) + } + + <-context.Background().Done() + + return nil +} + +func parseTime(t string) time.Time { + tstamp, err := time.Parse(time.RFC3339, t) + if err != nil { + tstamp = time.Now().UTC() + } + + return tstamp +} + +func leftPad(s string, indent int) string { + out := []string{} + format := fmt.Sprintf("%%%ds", indent) + + for _, l := range strings.Split(s, "\n") { + out = append(out, fmt.Sprintf(format, " ")+l) + } + + return strings.Join(out, "\n") +} + +func (c *eventsCmd) renderDisconnection(m *nats.Msg) { + if !c.subjectFRe.MatchString(strings.ToUpper(m.Subject)) { + return + } + + if !c.bodyFRe.MatchString(strings.ToUpper(string(m.Data))) { + return + } + + if c.json { + fmt.Println(string(m.Data)) + return + } + + event := api.DisconnectEventMsg{} + err := json.Unmarshal(m.Data, &event) + if err != nil { + fmt.Printf("Event parsing failed: %s\n\n", err) + fmt.Println(leftPad(string(m.Data), 10)) + return + } + + fmt.Printf("[%s] [%d] Client Disconnection\n", event.Server.Time.Format("15:04:05"), event.Server.Seq) + fmt.Printf(" Reason: %s\n", event.Reason) + fmt.Printf(" Server: %s\n", event.Server.Name) + fmt.Println() + fmt.Println(" Client:") + fmt.Printf(" ID: %d\n", event.Client.ID) + if event.Client.User != "" { + fmt.Printf(" User: %s\n", event.Client.User) + } + if event.Client.Name != "" { + fmt.Printf(" Name: %s\n", event.Client.Name) + } + fmt.Printf(" Account: %s\n", event.Client.Account) + fmt.Println() + fmt.Println(" Stats:") + fmt.Printf(" Received: %d messages (%s)\n", event.Received.Msgs, humanize.IBytes(uint64(event.Received.Bytes))) + fmt.Printf(" Published: %d messages (%s)\n", event.Sent.Msgs, humanize.IBytes(uint64(event.Sent.Bytes))) + fmt.Println() +} + +func (c *eventsCmd) renderConnection(m *nats.Msg) { + if !c.subjectFRe.MatchString(strings.ToUpper(m.Subject)) { + return + } + if !c.bodyFRe.MatchString(strings.ToUpper(string(m.Data))) { + return + } + + if c.json { + fmt.Println(string(m.Data)) + return + } + + event := api.ConnectEventMsg{} + err := json.Unmarshal(m.Data, &event) + if err != nil { + fmt.Printf("Event parsing failed: %s\n\n", err) + fmt.Println(leftPad(string(m.Data), 10)) + return + } + + fmt.Printf("[%s] [%d] Client Connection\n", event.Server.Time.Format("15:04:05"), event.Server.Seq) + fmt.Println(" Server:") + fmt.Printf(" Name: %s\n", event.Server.Name) + if event.Server.Cluster != "" { + fmt.Printf(" Cluster: %s\n", event.Server.Cluster) + } + fmt.Println() + fmt.Println(" Client:") + fmt.Printf(" ID: %d\n", event.Client.ID) + if event.Client.User != "" { + fmt.Printf(" User: %s\n", event.Client.User) + } + if event.Client.Name != "" { + fmt.Printf(" Name: %s\n", event.Client.Name) + } + fmt.Printf(" Account: %s\n", event.Client.Account) + fmt.Printf(" Language: %s\n", event.Client.Lang) + fmt.Printf(" Version: %s\n", event.Client.Version) + fmt.Printf(" Host: %s\n", event.Client.Host) + fmt.Println() +} + +func (c *eventsCmd) renderDeliveryExceeded(event *api.ConsumerDeliveryExceededAdvisory, m *nats.Msg) { + if !c.subjectFRe.MatchString(strings.ToUpper(m.Subject)) { + return + } + + if c.json { + fmt.Println(string(m.Data)) + return + } + + fmt.Printf("[%s] [%s] Delivery Attempts Exceeded\n", parseTime(event.Time).Format("15:04:05"), event.ID) + fmt.Printf(" Consumer: %s > %s\n", event.Stream, event.Consumer) + fmt.Printf(" Stream Sequence: %d\n", event.StreamSeq) + fmt.Printf(" Deliveries: %d\n", event.Deliveries) + fmt.Println() +} + +func (c *eventsCmd) renderAPIAudit(event *api.JetStreamAPIAudit, m *nats.Msg) { + if !c.subjectFRe.MatchString(strings.ToUpper(event.Subject)) { + return + } + + if c.json { + fmt.Println(string(m.Data)) + return + } + + fmt.Printf("[%s] [%s] API Access\n", parseTime(event.Time).Format("15:04:05"), event.ID) + fmt.Printf(" Server: %s\n", event.Server) + fmt.Printf(" Subject: %s\n", event.Subject) + fmt.Printf(" Client:\n") + if event.Client.User != "" { + fmt.Printf(" User: %s Account: %s\n", event.Client.User, event.Client.Account) + } else { + fmt.Printf(" Account: %s\n", event.Client.Account) + } + fmt.Printf(" Host: %s\n", net.JoinHostPort(event.Client.Host, strconv.Itoa(event.Client.Port))) + fmt.Printf(" ID: %d\n", event.Client.CID) + if event.Client.Name != "" { + fmt.Printf(" Name: %s\n", event.Client.Name) + } + if event.Client.Language != "" && event.Client.Version != "" { + fmt.Printf(" Lanuage: %s %s\n", event.Client.Language, event.Client.Version) + } + fmt.Println() + + fmt.Println(" Request:") + fmt.Println() + if event.Request != "" { + fmt.Println(leftPad(event.Request, 10)) + } else { + fmt.Println(" Empty Request") + } + fmt.Println() + + if event.Response != "" { + fmt.Println(" Response:") + fmt.Println() + fmt.Println(leftPad(event.Response, 10)) + } + fmt.Println() +} + +func (c *eventsCmd) renderAckSample(event *api.ConsumerAckMetric, m *nats.Msg) { + if !c.subjectFRe.MatchString(strings.ToUpper(m.Subject)) { + return + } + + if c.json { + fmt.Println(string(m.Data)) + return + } + + fmt.Printf("[%s] [%s] Acknowledgement Sample\n", parseTime(event.Time).Format("15:04:05"), event.ID) + fmt.Printf(" Consumer: %s > %s\n", event.Stream, event.Consumer) + fmt.Printf(" Stream Sequence: %d\n", event.StreamSeq) + fmt.Printf(" Consumer Sequence: %d\n", event.ConsumerSeq) + fmt.Printf(" Deliveries: %d\n", event.Deliveries) + fmt.Printf(" Delay: %v\n", time.Duration(event.Delay)) + fmt.Println() +} + +func (c *eventsCmd) renderAdvisory(m *nats.Msg) { + if !c.bodyFRe.MatchString(strings.ToUpper(string(m.Data))) { + return + } + + _, event, err := jsch.ParseEvent(m.Data) + if err != nil { + fmt.Printf("Event parsing failed: %s\n\n", err) + fmt.Println(leftPad(string(m.Data), 10)) + return + } + + switch event := event.(type) { + case *api.ConsumerDeliveryExceededAdvisory: + c.renderDeliveryExceeded(event, m) + + case *api.JetStreamAPIAudit: + c.renderAPIAudit(event, m) + + default: + fmt.Println(string(m.Data)) + } +} + +func (c *eventsCmd) renderMetric(m *nats.Msg) { + if !c.bodyFRe.MatchString(strings.ToUpper(string(m.Data))) { + return + } + + _, event, err := jsch.ParseEvent(m.Data) + if err != nil { + fmt.Printf("Event parsing failed: %s\n\n", err) + fmt.Println(leftPad(string(m.Data), 10)) + return + } + + switch event := event.(type) { + case *api.ConsumerAckMetric: + c.renderAckSample(event, m) + + default: + fmt.Println(string(m.Data)) + } +} diff --git a/nats/main.go b/nats/main.go new file mode 100644 index 0000000..4ceabbc --- /dev/null +++ b/nats/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "log" + "os" + "time" + + "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + servers string + creds string + tlsCert string + tlsKey string + tlsCA string + timeout time.Duration + version string +) + +func main() { + if version == "" { + version = "development" + } + + ncli := kingpin.New("nats", "NATS Management Utility") + ncli.Author("NATS Authors ") + ncli.Version(version) + ncli.HelpFlag.Short('h') + + ncli.Flag("server", "NATS servers").Short('s').Default("localhost:4222").Envar("NATS_URL").StringVar(&servers) + ncli.Flag("creds", "User credentials").Envar("NATS_CREDS").StringVar(&creds) + ncli.Flag("tlscert", "TLS public certificate").ExistingFileVar(&tlsCert) + ncli.Flag("tlskey", "TLS private key").ExistingFileVar(&tlsCert) + ncli.Flag("tlsca", "TLS certificate authority chain").ExistingFileVar(&tlsCA) + ncli.Flag("timeout", "Time to wait on responses from NATS").Default("2s").Envar("NATS_TIMEOUT").DurationVar(&timeout) + + log.SetFlags(log.Ltime) + + configurePubCommand(ncli) + configureSubCommand(ncli) + configureReplyCommand(ncli) + configureBenchCommand(ncli) + configureServerCommand(ncli) + configureActCommand(ncli) + configureEventsCommand(ncli) + configureStreamCommand(ncli) + configureConsumerCommand(ncli) + configureBackupCommand(ncli) + configureRestoreCommand(ncli) + + kingpin.MustParse(ncli.Parse(os.Args[1:])) +} diff --git a/nats/nats_test.go b/nats/nats_test.go new file mode 100644 index 0000000..140f689 --- /dev/null +++ b/nats/nats_test.go @@ -0,0 +1,558 @@ +// Copyright 2019 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + + "github.com/nats-io/jetstream/internal/jsch" +) + +func runNatsCli(t *testing.T, args ...string) (output []byte) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + var cmd string + if os.Getenv("CI") == "true" { + cmd = fmt.Sprintf("./nats %s", strings.Join(args, " ")) + } else { + cmd = fmt.Sprintf("go run $(ls *.go | grep -v _test.go) %s", strings.Join(args, " ")) + } + execution := exec.CommandContext(ctx, "bash", "-c", cmd) + out, err := execution.CombinedOutput() + if err != nil { + t.Fatalf("nats utility failed: %v\n%v", err, string(out)) + } + + return out +} + +func setupJStreamTest(t *testing.T) (srv *server.Server, nc *nats.Conn) { + dir, err := ioutil.TempDir("", "") + checkErr(t, err, "could not create temporary js store: %v", err) + + srv, err = server.NewServer(&server.Options{ + Port: -1, + StoreDir: dir, + JetStream: true, + }) + checkErr(t, err, "could not start js server: %v", err) + + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Errorf("nats server did not start") + } + + nc, err = prepareHelper(srv.ClientURL()) + checkErr(t, err, "could not connect client to server @ %s: %v", srv.ClientURL(), err) + + streams, err := jsch.StreamNames() + checkErr(t, err, "could not load streams: %v", err) + if len(streams) != 0 { + t.Fatalf("found %v message streams but it should be empty", streams) + } + + return srv, nc +} + +func setupConsTest(t *testing.T) (srv *server.Server, nc *nats.Conn) { + srv, nc = setupJStreamTest(t) + + _, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create stream: %v", err) + streamShouldExist(t, "mem1") + + return srv, nc +} + +func streamShouldExist(t *testing.T, stream string) { + t.Helper() + known, err := jsch.IsKnownStream(stream) + checkErr(t, err, "stream lookup failed: %v", err) + if !known { + t.Fatalf("%s does not exist", stream) + } +} + +func streamShouldNotExist(t *testing.T, stream string) { + t.Helper() + known, err := jsch.IsKnownStream(stream) + checkErr(t, err, "stream lookup failed: %v", err) + if known { + t.Fatalf("unexpectedly found %s already existing", stream) + } +} + +func consumerShouldExist(t *testing.T, stream string, consumer string) { + t.Helper() + known, err := jsch.IsKnownConsumer(stream, consumer) + checkErr(t, err, "consumer lookup failed: %v", err) + if !known { + t.Fatalf("%s does not exist", consumer) + } +} + +func streamInfo(t *testing.T, stream string) *server.StreamInfo { + t.Helper() + str, err := jsch.LoadStream(stream) + checkErr(t, err, "could not load stream %s", stream) + info, err := str.Information() + checkErr(t, err, "could not load stream %s", stream) + + return info +} + +func mem1Stream() server.StreamConfig { + return server.StreamConfig{ + Name: "mem1", + Subjects: []string{"js.mem.>"}, + Storage: server.MemoryStorage, + } +} + +func pull1Cons() server.ConsumerConfig { + return server.ConsumerConfig{ + Durable: "push1", + DeliverAll: true, + AckPolicy: server.AckExplicit, + } +} + +func TestCLIStreamCreate(t *testing.T) { + srv, _ := setupJStreamTest(t) + defer srv.Shutdown() + + runNatsCli(t, fmt.Sprintf("--server='%s' str create mem1 --subjects 'js.mem.>,js.other' --storage m --max-msgs=-1 --max-age=-1 --max-bytes=-1 --ack --retention limits --max-msg-size=1024", srv.ClientURL())) + streamShouldExist(t, "mem1") + info := streamInfo(t, "mem1") + + if len(info.Config.Subjects) != 2 { + t.Fatalf("expected 2 subjects in the message stream, got %v", info.Config.Subjects) + } + + if info.Config.Subjects[0] != "js.mem.>" && info.Config.Subjects[1] != "js.other" { + t.Fatalf("expects [js.mem.>, js.other] got %v", info.Config.Subjects) + } + + if info.Config.Retention != server.LimitsPolicy { + t.Fatalf("incorrect retention policy, expected limits got %s", info.Config.Retention.String()) + } + + if info.Config.Storage != server.MemoryStorage { + t.Fatalf("incorrect storage received, expected memory got %s", info.Config.Storage.String()) + } + + if info.Config.MaxMsgSize != 1024 { + t.Fatalf("incorrect max message size stream, expected 1024 got %v", info.Config.MaxMsgSize) + } +} + +func TestCLIStreamInfo(t *testing.T) { + srv, _ := setupJStreamTest(t) + defer srv.Shutdown() + + _, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create stream: %v", err) + streamShouldExist(t, "mem1") + + out := runNatsCli(t, fmt.Sprintf("--server='%s' str info mem1 -j", srv.ClientURL())) + + var info server.StreamInfo + err = json.Unmarshal(out, &info) + checkErr(t, err, "could not parse cli output: %v", err) + + if info.Config.Name != "mem1" { + t.Fatalf("expected info for mem1, got %s", info.Config.Name) + + } +} + +func TestCLIStreamDelete(t *testing.T) { + srv, _ := setupJStreamTest(t) + defer srv.Shutdown() + + _, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create message stream: %v", err) + streamShouldExist(t, "mem1") + + runNatsCli(t, fmt.Sprintf("--server='%s' str rm mem1 -f", srv.ClientURL())) + streamShouldNotExist(t, "mem1") +} + +func TestCLIStreamLs(t *testing.T) { + srv, _ := setupJStreamTest(t) + defer srv.Shutdown() + + _, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create stream: %v", err) + streamShouldExist(t, "mem1") + + out := runNatsCli(t, fmt.Sprintf("--server='%s' str ls -j", srv.ClientURL())) + + list := []string{} + err = json.Unmarshal(out, &list) + checkErr(t, err, "could not parse cli output: %v", err) + + if len(list) != 1 { + t.Fatalf("expected 1 ms got %v", list) + } + + if list[0] != "mem1" { + t.Fatalf("expected [mem1] got %v", list) + } +} + +func TestCLIStreamPurge(t *testing.T) { + srv, nc := setupJStreamTest(t) + defer srv.Shutdown() + + stream, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create stream: %v", err) + streamShouldExist(t, "mem1") + + _, err = nc.Request("js.mem.1", []byte("hello"), time.Second) + checkErr(t, err, "could not publish message: %v", err) + + i, err := stream.Information() + checkErr(t, err, "could not get message stream info: %v", err) + if i.State.Msgs != 1 { + t.Fatalf("expected 1 message but got %d", i.State.Msgs) + } + + runNatsCli(t, fmt.Sprintf("--server='%s' str purge mem1 -f", srv.ClientURL())) + i, err = stream.Information() + checkErr(t, err, "could not get message stream info: %v", err) + if i.State.Msgs != 0 { + t.Fatalf("expected 0 messages but got %d", i.State.Msgs) + } +} + +func TestCLIStreamGet(t *testing.T) { + srv, nc := setupJStreamTest(t) + defer srv.Shutdown() + + stream, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create stream: %v", err) + streamShouldExist(t, "mem1") + + _, err = nc.Request("js.mem.1", []byte("hello"), time.Second) + checkErr(t, err, "could not publish message: %v", err) + + item, err := stream.LoadMessage(1) + checkErr(t, err, "could not get message: %v", err) + if string(item.Data) != "hello" { + t.Fatalf("got incorrect data from message, expected 'hello' got: %v", string(item.Data)) + } + + out := runNatsCli(t, fmt.Sprintf("--server='%s' str get mem1 1 -j", srv.ClientURL())) + err = json.Unmarshal(out, &item) + checkErr(t, err, "could not parse output: %v", err) + if string(item.Data) != "hello" { + t.Fatalf("got incorrect data from message, expected 'hello' got: %v", string(item.Data)) + } +} + +func TestCLIConsumerInfo(t *testing.T) { + srv, _ := setupConsTest(t) + defer srv.Shutdown() + + _, err := jsch.NewConsumerFromDefault("mem1", pull1Cons()) + checkErr(t, err, "could not create consumer: %v", err) + consumerShouldExist(t, "mem1", "push1") + + out := runNatsCli(t, fmt.Sprintf("--server='%s' con info mem1 push1 -j", srv.ClientURL())) + var info server.ConsumerInfo + err = json.Unmarshal(out, &info) + checkErr(t, err, "could not parse output: %v", err) + + if info.Config.Durable != "push1" { + t.Fatalf("did not find into for push1 in cli output: %v", string(out)) + } +} + +func TestCLIConsumerLs(t *testing.T) { + srv, _ := setupConsTest(t) + defer srv.Shutdown() + + _, err := jsch.NewConsumerFromDefault("mem1", pull1Cons()) + checkErr(t, err, "could not create consumer: %v", err) + consumerShouldExist(t, "mem1", "push1") + + out := runNatsCli(t, fmt.Sprintf("--server='%s' con ls mem1 -j", srv.ClientURL())) + var info []string + err = json.Unmarshal(out, &info) + checkErr(t, err, "could not parse output: %v", err) + + if len(info) != 1 { + t.Fatalf("expected 1 item in output received %d", len(info)) + } + + if info[0] != "push1" { + t.Fatalf("did not find into for push1 in cli output: %v", string(out)) + } +} + +func TestCLIConsumerDelete(t *testing.T) { + srv, _ := setupConsTest(t) + defer srv.Shutdown() + + _, err := jsch.NewConsumerFromDefault("mem1", pull1Cons()) + checkErr(t, err, "could not create consumer: %v", err) + consumerShouldExist(t, "mem1", "push1") + + runNatsCli(t, fmt.Sprintf("--server='%s' con rm mem1 push1 -f", srv.ClientURL())) + + list, err := jsch.ConsumerNames("mem1") + checkErr(t, err, "could not check cnsumer: %v", err) + if len(list) != 0 { + t.Fatalf("Expected no consumer, got %v", list) + } +} + +func TestCLIConsumerAdd(t *testing.T) { + srv, _ := setupConsTest(t) + defer srv.Shutdown() + + runNatsCli(t, fmt.Sprintf("--server='%s' con add mem1 push1 --replay instant --deliver all --pull --filter '' --max-deliver 20", srv.ClientURL())) + consumerShouldExist(t, "mem1", "push1") +} + +func TestCLIConsumerNext(t *testing.T) { + srv, nc := setupConsTest(t) + defer srv.Shutdown() + + push1, err := jsch.NewConsumerFromDefault("mem1", pull1Cons()) + checkErr(t, err, "could not create consumer: %v", err) + consumerShouldExist(t, "mem1", "push1") + + push1.Reset() + if !push1.IsPullMode() { + t.Fatalf("push1 is not push mode: %#v", push1.Configuration()) + } + + _, err = nc.Request("js.mem.1", []byte("hello"), time.Second) + checkErr(t, err, "could not publish to mem1: %v", err) + + out := runNatsCli(t, fmt.Sprintf("--server='%s' con next mem1 push1 --raw", srv.ClientURL())) + + if strings.TrimSpace(string(out)) != "hello" { + t.Fatalf("did not receive 'hello', got: '%s'", string(out)) + } +} + +func TestCLIStreamEdit(t *testing.T) { + srv, _ := setupJStreamTest(t) + defer srv.Shutdown() + + mem1, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create stream: %v", err) + streamShouldExist(t, "mem1") + + runNatsCli(t, fmt.Sprintf("--server='%s' str edit mem1 --subjects other", srv.ClientURL())) + + err = mem1.Reset() + checkErr(t, err, "could not reset stream: %v", err) + + if len(mem1.Subjects()) != 1 { + t.Fatalf("expected [other] got %v", mem1.Subjects()) + } + + if mem1.Subjects()[0] != "other" { + t.Fatalf("expected [other] got %v", mem1.Subjects()) + } + +} + +func TestCLIStreamCopy(t *testing.T) { + srv, _ := setupJStreamTest(t) + defer srv.Shutdown() + + _, err := jsch.NewStreamFromDefault("mem1", mem1Stream()) + checkErr(t, err, "could not create stream: %v", err) + streamShouldExist(t, "mem1") + + runNatsCli(t, fmt.Sprintf("--server='%s' str cp mem1 file1 --storage file --subjects other", srv.ClientURL())) + streamShouldExist(t, "file1") + + stream, err := jsch.LoadStream("file1") + checkErr(t, err, "could not get stream: %v", err) + info, err := stream.Information() + checkErr(t, err, "could not get stream: %v", err) + if info.Config.Storage != server.FileStorage { + t.Fatalf("Expected file storage got %s", info.Config.Storage.String()) + } +} + +func TestCLIConsumerCopy(t *testing.T) { + srv, _ := setupConsTest(t) + defer srv.Shutdown() + + _, err := jsch.NewConsumerFromDefault("mem1", pull1Cons()) + checkErr(t, err, "could not create consumer: %v", err) + consumerShouldExist(t, "mem1", "push1") + + runNatsCli(t, fmt.Sprintf("--server='%s' con cp mem1 push1 pull1 --pull", srv.ClientURL())) + consumerShouldExist(t, "mem1", "pull1") + + pull1, err := jsch.LoadConsumer("mem1", "pull1") + checkErr(t, err, "could not get consumer: %v", err) + consumerShouldExist(t, "mem1", "pull1") + + ols, err := jsch.ConsumerNames("mem1") + checkErr(t, err, "could not get consumer: %v", err) + + if len(ols) != 2 { + t.Fatalf("expected 2 consumers, got %d", len(ols)) + } + + if !pull1.IsPullMode() { + t.Fatalf("Expected pull1 to be pull-based, got %v", pull1.Configuration()) + } +} + +func TestCLIBackupRestore(t *testing.T) { + srv, _ := setupConsTest(t) + defer srv.Shutdown() + + dir, err := ioutil.TempDir("", "") + checkErr(t, err, "temp dir failed") + defer os.RemoveAll(dir) + + target := filepath.Join(dir, "backup") + + mem1, err := jsch.LoadStream("mem1") + checkErr(t, err, "fetch mem1 failed") + origMem1Config := mem1.Configuration() + + c1, err := mem1.NewConsumerFromDefault(jsch.DefaultConsumer, jsch.DurableName("c1")) + checkErr(t, err, "consumer c1 failed") + origC1Config := c1.Configuration() + + t1, err := jsch.NewStreamTemplate("t1", 1, jsch.DefaultStream) + checkErr(t, err, "TEST template create failed") + origT1Config := t1.Configuration() + + runNatsCli(t, fmt.Sprintf("--server='%s' backup '%s'", srv.ClientURL(), target)) + + checkErr(t, mem1.Delete(), "mem1 delete failed") + checkErr(t, t1.Delete(), "t1 delete failed") + + runNatsCli(t, fmt.Sprintf("--server='%s' restore '%s'", srv.ClientURL(), target)) + + mem1, err = jsch.LoadStream("mem1") + checkErr(t, err, "fetch mem1 failed") + if !cmp.Equal(mem1.Configuration(), origMem1Config) { + t.Fatalf("mem1 recreate failed") + } + + c1, err = mem1.LoadConsumer("c1") + checkErr(t, err, "fetch c1 failed") + if !cmp.Equal(c1.Configuration(), origC1Config) { + t.Fatalf("mem1 recreate failed") + } + + t1, err = jsch.LoadStreamTemplate("t1") + checkErr(t, err, "template load failed") + if !cmp.Equal(t1.Configuration(), origT1Config) { + t.Fatalf("mem1 recreate failed") + } +} + +func TestCLIBackupRestore_UpdateStream(t *testing.T) { + srv, _ := setupConsTest(t) + defer srv.Shutdown() + + dir, err := ioutil.TempDir("", "") + checkErr(t, err, "temp dir failed") + defer os.RemoveAll(dir) + + target := filepath.Join(dir, "backup") + + mem1, err := jsch.LoadStream("mem1") + checkErr(t, err, "fetch mem1 failed") + + runNatsCli(t, fmt.Sprintf("--server='%s' backup '%s'", srv.ClientURL(), target)) + + runNatsCli(t, fmt.Sprintf("--server='%s' stream edit mem1 --subjects x", srv.ClientURL())) + checkErr(t, mem1.Reset(), "reset failed") + subs := mem1.Subjects() + if len(subs) != 1 || subs[0] != "x" { + t.Fatalf("expected [x] got %q", subs) + } + + runNatsCli(t, fmt.Sprintf("--server='%s' restore '%s' --update-streams", srv.ClientURL(), target)) + checkErr(t, mem1.Reset(), "reset failed") + subs = mem1.Subjects() + if len(subs) != 1 || subs[0] != "js.mem.>" { + t.Fatalf("expected [js.mem.>] got %q", subs) + } +} + +func TestCLIMessageRm(t *testing.T) { + srv, nc := setupConsTest(t) + defer srv.Shutdown() + + checkErr(t, nc.Publish("js.mem.1", []byte("msg1")), "publish failed") + checkErr(t, nc.Publish("js.mem.1", []byte("msg2")), "publish failed") + checkErr(t, nc.Publish("js.mem.1", []byte("msg3")), "publish failed") + + mem1, err := jsch.LoadStream("mem1") + checkErr(t, err, "load failed") + + state, err := mem1.State() + checkErr(t, err, "state failed") + + if state.Msgs != 3 { + t.Fatalf("no message added to stream") + } + + runNatsCli(t, fmt.Sprintf("--server='%s' str rmm mem1 2 -f", srv.ClientURL())) + state, err = mem1.State() + checkErr(t, err, "state failed") + + if state.Msgs != 2 { + t.Fatalf("message was not removed") + } + + msg, err := mem1.LoadMessage(1) + checkErr(t, err, "load failed") + if cmp.Equal(msg.Data, []byte("msg1")) { + checkErr(t, err, "load failed") + } + + msg, err = mem1.LoadMessage(3) + checkErr(t, err, "load failed") + if cmp.Equal(msg.Data, []byte("msg3")) { + checkErr(t, err, "load failed") + } + + msg, err = mem1.LoadMessage(2) + if err == nil { + t.Fatalf("loading delete message did not fail") + } +} diff --git a/nats/pub_command.go b/nats/pub_command.go new file mode 100644 index 0000000..d71090c --- /dev/null +++ b/nats/pub_command.go @@ -0,0 +1,88 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "fmt" + "log" + "os" + + "golang.org/x/crypto/ssh/terminal" + "gopkg.in/alecthomas/kingpin.v2" +) + +type pubCmd struct { + subject string + body string + req bool +} + +func configurePubCommand(app *kingpin.Application) { + c := &pubCmd{} + act := app.Command("pub", "Generic data publishing utility").Action(c.publish) + act.Arg("subject", "Subject to subscribe to").Required().StringVar(&c.subject) + act.Arg("body", "Message body").StringVar(&c.body) + act.Flag("wait", "Wait for a reply from a service").Short('w').BoolVar(&c.req) + + req := app.Command("request", "Generic data request utility").Alias("req").Action(c.publish) + req.Arg("subject", "Subject to subscribe to").Required().StringVar(&c.subject) + req.Arg("body", "Message body").StringVar(&c.body) + req.Flag("wait", "Wait for a reply from a service").Short('w').Default("true").Hidden().BoolVar(&c.req) + +} + +func (c *pubCmd) publish(pc *kingpin.ParseContext) error { + nc, err := newNatsConn(servers, natsOpts()...) + if err != nil { + return err + } + defer nc.Close() + + if c.body == "" && terminal.IsTerminal(int(os.Stdout.Fd())) { + reader := bufio.NewReader(os.Stdin) + c.body, err = reader.ReadString('\n') + if err != nil { + return err + } + } + + if c.body == "" { + return fmt.Errorf("please specify a body to publish either as command argument or on standard input") + } + + if c.req { + log.Printf("Sending request on [%s]\n", c.subject) + m, err := nc.Request(c.subject, []byte(c.body), timeout) + if err != nil { + return err + } + + log.Printf("Received on [%s]: '%s'", m.Subject, string(m.Data)) + return nil + } else { + nc.Publish(c.subject, []byte(c.body)) + } + + nc.Flush() + + err = nc.LastError() + if err != nil { + return err + } + + log.Printf("Published %d bytes to %s\n", len(c.body), c.subject) + + return nil +} diff --git a/nats/reply_command.go b/nats/reply_command.go new file mode 100644 index 0000000..78ab512 --- /dev/null +++ b/nats/reply_command.go @@ -0,0 +1,79 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "log" + "os" + "os/signal" + + "github.com/nats-io/nats.go" + "gopkg.in/alecthomas/kingpin.v2" +) + +type replyCmd struct { + subject string + body string + queue string + echo bool +} + +func configureReplyCommand(app *kingpin.Application) { + c := &replyCmd{} + act := app.Command("reply", "Generic service reply utility").Action(c.reply) + act.Arg("subject", "Subject to subscribe to").Required().StringVar(&c.subject) + act.Arg("body", "Reply body").StringVar(&c.body) + act.Flag("echo", "Echo back what is received").BoolVar(&c.echo) + act.Flag("queue", "Queue group name").Default("NATS-RPLY-22").Short('q').StringVar(&c.queue) +} + +func (c *replyCmd) reply(_ *kingpin.ParseContext) error { + nc, err := newNatsConn(servers, natsOpts()...) + if err != nil { + return err + } + + if c.body == "" && !c.echo { + log.Println("No body supplied, enabling echo mode") + c.echo = true + } + + i := 0 + nc.QueueSubscribe(c.subject, c.queue, func(m *nats.Msg) { + log.Printf("[#%d] Received on [%s]: '%s'\n", i, m.Subject, string(m.Data)) + if c.echo { + m.Respond(m.Data) + } else { + m.Respond([]byte(c.body)) + } + }) + nc.Flush() + + err = nc.LastError() + if err != nil { + return err + } + + log.Printf("Listening on [%s]", c.subject) + + ic := make(chan os.Signal, 1) + signal.Notify(ic, os.Interrupt) + <-ic + + log.Printf("\nDraining...") + nc.Drain() + log.Fatalf("Exiting") + + return nil +} diff --git a/nats/restore_command.go b/nats/restore_command.go new file mode 100644 index 0000000..27e2225 --- /dev/null +++ b/nats/restore_command.go @@ -0,0 +1,58 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + + "gopkg.in/alecthomas/kingpin.v2" + + "github.com/nats-io/jetstream/internal/jsch" +) + +type restoreCmd struct { + backupDir string + file string + updateStream bool +} + +func configureRestoreCommand(app *kingpin.Application) { + c := &restoreCmd{} + + restore := app.Command("restore", "Restores a restore of JetStream configuration").Action(c.restoreAction) + restore.Arg("directory", "Directory to read restore from").StringVar(&c.backupDir) + restore.Flag("file", "File to read restore from").StringVar(&c.file) + restore.Flag("update-streams", "Update existing stream configuration").BoolVar(&c.updateStream) +} + +func (c *restoreCmd) restoreAction(_ *kingpin.ParseContext) error { + if c.file == "" && c.backupDir == "" { + return fmt.Errorf("a file or directory is required") + } + + if c.file != "" && c.backupDir != "" { + return fmt.Errorf("both file and directory can not be supplied") + } + + _, err := prepareHelper(servers, natsOpts()...) + if err != nil { + return err + } + + if c.backupDir != "" { + return jsch.RestoreJetStreamConfiguration(c.backupDir, c.updateStream) + } + + return jsch.RestoreJetStreamConfigurationFile(c.file, c.updateStream) +} diff --git a/nats/server_command.go b/nats/server_command.go new file mode 100644 index 0000000..0128000 --- /dev/null +++ b/nats/server_command.go @@ -0,0 +1,24 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "gopkg.in/alecthomas/kingpin.v2" +) + +func configureServerCommand(app *kingpin.Application) { + srv := app.Command("server", "Server information").Alias("srv") + configureServerListCommand(srv) + configureServerPingCommand(srv) +} diff --git a/nats/server_list_command.go b/nats/server_list_command.go new file mode 100644 index 0000000..d760fdd --- /dev/null +++ b/nats/server_list_command.go @@ -0,0 +1,132 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + "regexp" + "sync" + "sync/atomic" + + "github.com/dustin/go-humanize" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/xlab/tablewriter" + "gopkg.in/alecthomas/kingpin.v2" +) + +type SrvLsCmd struct { + expect uint32 + json bool + filter string +} + +func configureServerListCommand(srv *kingpin.CmdClause) { + c := &SrvLsCmd{} + + ls := srv.Command("list", "List known servers").Alias("ls").Action(c.list) + ls.Arg("expect", "How many servers to expect").Uint32Var(&c.expect) + ls.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + ls.Flag("filter", "Regular expression filter on server name").Short('f').StringVar(&c.filter) +} + +func (c *SrvLsCmd) list(_ *kingpin.ParseContext) error { + if creds == "" { + return fmt.Errorf("listing servers requires credentials supplied with --creds") + } + + nc, err := newNatsConn(servers, natsOpts()...) + if err != nil { + return err + } + defer nc.Close() + + ec, err := nats.NewEncodedConn(nc, nats.JSON_ENCODER) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + seen := uint32(0) + var results []*server.ServerStatsMsg + mu := &sync.Mutex{} + filter, err := regexp.Compile(c.filter) + if err != nil { + return err + } + + table := tablewriter.CreateTable() + table.AddHeaders("Name", "Cluster", "IP", "Version", "Conns", "Routes", "GWs", "Mem", "CPU", "Slow", "Uptime") + + sub, err := ec.Subscribe(nc.NewRespInbox(), func(ssm *server.ServerStatsMsg) { + last := atomic.AddUint32(&seen, 1) + + if !filter.MatchString(ssm.Server.Name) { + return + } + + mu.Lock() + results = append(results, ssm) + mu.Unlock() + + table.AddRow(ssm.Server.Name, ssm.Server.Cluster, ssm.Server.Host, ssm.Server.Version, int(ssm.Stats.Connections), len(ssm.Stats.Routes), len(ssm.Stats.Gateways), humanize.IBytes(uint64(ssm.Stats.Mem)), fmt.Sprintf("%.1f", ssm.Stats.CPU), ssm.Stats.SlowConsumers, humanizeTime(ssm.Stats.Start)) + + if last == c.expect { + cancel() + } + }) + if err != nil { + return err + } + + err = nc.PublishRequest("$SYS.REQ.SERVER.PING", sub.Subject, nil) + if err != nil { + return err + } + + ic := make(chan os.Signal, 1) + signal.Notify(ic, os.Interrupt) + + select { + case <-ic: + cancel() + case <-ctx.Done(): + } + + sub.Drain() + + if c.json { + j, err := json.Marshal(results) + if err != nil { + return err + } + + fmt.Println(string(j)) + return nil + } + + fmt.Print(table.Render()) + + if c.expect != 0 && c.expect != seen { + fmt.Printf("\nMissing %d server(s)\n", c.expect-atomic.LoadUint32(&seen)) + } + + return nil +} diff --git a/nats/server_ping_command.go b/nats/server_ping_command.go new file mode 100644 index 0000000..4edf7c3 --- /dev/null +++ b/nats/server_ping_command.go @@ -0,0 +1,166 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/guptarohit/asciigraph" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "gopkg.in/alecthomas/kingpin.v2" +) + +type SrvPingCmd struct { + expect uint32 + graph bool +} + +func configureServerPingCommand(srv *kingpin.CmdClause) { + c := &SrvPingCmd{} + + ls := srv.Command("ping", "Ping all servers").Action(c.ping) + ls.Arg("expect", "How many servers to expect").Uint32Var(&c.expect) + ls.Flag("graph", "Produce a response distribution graph").BoolVar(&c.graph) +} + +func (c *SrvPingCmd) ping(_ *kingpin.ParseContext) error { + if creds == "" { + return fmt.Errorf("listing servers requires credentials supplied with --creds") + } + + nc, err := newNatsConn(servers, natsOpts()...) + if err != nil { + return err + } + defer nc.Close() + + ec, err := nats.NewEncodedConn(nc, nats.JSON_ENCODER) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + + seen := uint32(0) + mu := &sync.Mutex{} + start := time.Now() + times := []float64{} + + sub, err := ec.Subscribe(nc.NewRespInbox(), func(ssm *server.ServerStatsMsg) { + last := atomic.AddUint32(&seen, 1) + + mu.Lock() + defer mu.Unlock() + + since := time.Since(start) + rtt := since.Milliseconds() + times = append(times, float64(rtt)) + + fmt.Printf("%-60s rtt=%s\n", ssm.Server.Name, since) + + if last == c.expect { + cancel() + } + }) + if err != nil { + return err + } + + err = nc.PublishRequest("$SYS.REQ.SERVER.PING", sub.Subject, nil) + if err != nil { + return err + } + + ic := make(chan os.Signal, 1) + signal.Notify(ic, os.Interrupt) + + select { + case <-ic: + cancel() + case <-ctx.Done(): + } + + sub.Drain() + + c.summarize(times) + + if c.expect != 0 && c.expect != seen { + fmt.Printf("\nMissing %d server(s)\n", c.expect-atomic.LoadUint32(&seen)) + } + + return nil +} + +func (c *SrvPingCmd) summarize(times []float64) { + fmt.Println() + fmt.Println("---- ping statistics ----") + + if len(times) > 0 { + sum := 0.0 + min := 999999.0 + max := -1.0 + avg := 0.0 + + for _, value := range times { + sum += value + if value < min { + min = value + } + if value > max { + max = value + } + } + + avg = sum / float64(len(times)) + + fmt.Printf("%d replies max: %.2f min: %.2f avg: %.2f\n", len(times), max, min, avg) + + if c.graph { + fmt.Println() + fmt.Println(c.chart(times)) + } + return + } + + fmt.Println("no responses received") +} + +func (c *SrvPingCmd) chart(times []float64) string { + sort.Float64s(times) + + latest := times[len(times)-1] + bcount := int(latest/25) + 1 + buckets := make([]float64, bcount) + + for _, t := range times { + b := t / 25.0 + buckets[int(b)]++ + } + + return asciigraph.Plot( + buckets, + asciigraph.Height(15), + asciigraph.Width(60), + asciigraph.Offset(5), + asciigraph.Caption("Responses per 25ms"), + ) +} diff --git a/nats/stream_command.go b/nats/stream_command.go new file mode 100644 index 0000000..717429e --- /dev/null +++ b/nats/stream_command.go @@ -0,0 +1,742 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/AlecAivazis/survey/v2" + "github.com/dustin/go-humanize" + api "github.com/nats-io/nats-server/v2/server" + "github.com/xlab/tablewriter" + "gopkg.in/alecthomas/kingpin.v2" + + "github.com/nats-io/jetstream/internal/jsch" +) + +type streamCmd struct { + stream string + force bool + json bool + msgID int64 + retentionPolicyS string + + destination string + subjects []string + ack bool + storage string + maxMsgLimit int64 + maxBytesLimit int64 + maxAgeLimit string + maxMsgSize int32 + rPolicy api.RetentionPolicy + reportSortConsumers bool + reportSortMsgs bool + reportSortName bool + reportRaw bool + maxStreams int +} + +func configureStreamCommand(app *kingpin.Application) { + c := &streamCmd{msgID: -1} + + str := app.Command("stream", "JetStream Stream management").Alias("str").Alias("st").Alias("ms").Alias("s") + + strInfo := str.Command("info", "Stream information").Alias("nfo").Alias("i").Action(c.infoAction) + strInfo.Arg("stream", "Stream to retrieve information for").StringVar(&c.stream) + strInfo.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + + addCreateFlags := func(f *kingpin.CmdClause) { + f.Flag("subjects", "Subjects that are consumed by the Stream").Default().StringsVar(&c.subjects) + f.Flag("ack", "Acknowledge publishes").Default("true").BoolVar(&c.ack) + f.Flag("max-msgs", "Maximum amount of messages to keep").Default("0").Int64Var(&c.maxMsgLimit) + f.Flag("max-bytes", "Maximum bytes to keep").Default("0").Int64Var(&c.maxBytesLimit) + f.Flag("max-age", "Maximum age of messages to keep").Default("").StringVar(&c.maxAgeLimit) + f.Flag("storage", "Storage backend to use (file, memory)").EnumVar(&c.storage, "file", "f", "memory", "m") + f.Flag("retention", "Defines a retention policy (limits, interest, work)").EnumVar(&c.retentionPolicyS, "limits", "interest", "workq", "work") + f.Flag("max-msg-size", "Maximum size any 1 message may be").Int32Var(&c.maxMsgSize) + f.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + } + + strAdd := str.Command("create", "Create a new Stream").Alias("add").Alias("new").Action(c.addAction) + strAdd.Arg("stream", "Stream name").StringVar(&c.stream) + addCreateFlags(strAdd) + + strEdit := str.Command("edit", "Edits an existing stream").Action(c.editAction) + strEdit.Arg("stream", "Stream to retrieve edit").StringVar(&c.stream) + addCreateFlags(strEdit) + + strCopy := str.Command("copy", "Creates a new Stream based on the configuration of another").Alias("cp").Action(c.cpAction) + strCopy.Arg("source", "Source Stream to copy").Required().StringVar(&c.stream) + strCopy.Arg("destination", "New Stream to create").Required().StringVar(&c.destination) + addCreateFlags(strCopy) + + strRm := str.Command("rm", "Removes a Stream").Alias("delete").Alias("del").Action(c.rmAction) + strRm.Arg("stream", "Stream name").StringVar(&c.stream) + strRm.Flag("force", "Force removal without prompting").Short('f').BoolVar(&c.force) + + strLs := str.Command("ls", "List all known Streams").Alias("list").Alias("l").Action(c.lsAction) + strLs.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + + strPurge := str.Command("purge", "Purge a Stream without deleting it").Action(c.purgeAction) + strPurge.Arg("stream", "Stream name").StringVar(&c.stream) + strPurge.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + strPurge.Flag("force", "Force removal without prompting").Short('f').BoolVar(&c.force) + + strRmMsg := str.Command("rmm", "Securely removes an individual message from a Stream").Action(c.rmMsgAction) + strRmMsg.Arg("stream", "Stream name").StringVar(&c.stream) + strRmMsg.Arg("id", "Message ID to remove").Int64Var(&c.msgID) + strRmMsg.Flag("force", "Force removal without prompting").Short('f').BoolVar(&c.force) + + strGet := str.Command("get", "Retrieves a specific message from a Stream").Action(c.getAction) + strGet.Arg("stream", "Stream name").StringVar(&c.stream) + strGet.Arg("id", "Message ID to retrieve").Int64Var(&c.msgID) + strGet.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + + strReport := str.Command("report", "Reports on Stream statistics").Action(c.reportAction) + strReport.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + strReport.Flag("consumers", "Sort by number of Consumers").Short('o').BoolVar(&c.reportSortConsumers) + strReport.Flag("messages", "Sort by number of Messages").Short('m').BoolVar(&c.reportSortMsgs) + strReport.Flag("name", "Sort by Stream name").Short('n').BoolVar(&c.reportSortName) + strReport.Flag("raw", "Show un-formatted numbers").Short('r').BoolVar(&c.reportRaw) + + strTemplate := str.Command("template", "Manages Stream Templates").Alias("templ").Alias("t") + + strTAdd := strTemplate.Command("create", "Creates a new Stream Template").Alias("add").Alias("new").Action(c.streamTemplateAdd) + strTAdd.Arg("stream", "Template name").StringVar(&c.stream) + strTAdd.Flag("max-streams", "Maximum amount of streams that this template can generate").Default("-1").IntVar(&c.maxStreams) + addCreateFlags(strTAdd) + + strTLs := strTemplate.Command("ls", "List all known Stream Templates").Alias("list").Alias("l").Action(c.streamTemplateLs) + strTLs.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) + + strTRm := strTemplate.Command("rm", "Removes a Stream Template").Alias("delete").Alias("del").Action(c.streamTemplateRm) + strTRm.Arg("template", "Stream Template name").StringVar(&c.stream) + strTRm.Flag("force", "Force removal without prompting").Short('f').BoolVar(&c.force) + + strTInfo := strTemplate.Command("info", "Stream Template information").Alias("nfo").Alias("i").Action(c.streamTemplateInfo) + strTInfo.Arg("template", "Stream Template to retrieve information for").StringVar(&c.stream) + strTInfo.Flag("json", "Produce JSON output").Short('j').BoolVar(&c.json) +} + +func (c *streamCmd) streamTemplateRm(_ *kingpin.ParseContext) (err error) { + nc, err := newNatsConn(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + jsch.SetConnection(nc) + + c.stream, err = selectStreamTemplate(c.stream) + kingpin.FatalIfError(err, "could not pick a Stream Template to operate on") + + template, err := jsch.LoadStreamTemplate(c.stream) + kingpin.FatalIfError(err, "could not load Stream Template") + + if !c.force { + ok, err := askConfirmation(fmt.Sprintf("Really delete Stream Template %q, this will remove all managed Streams this template created as well", c.stream), false) + kingpin.FatalIfError(err, "could not obtain confirmation") + + if !ok { + return nil + } + } + + err = template.Delete() + kingpin.FatalIfError(err, "could not delete Stream Template") + + return nil +} + +func (c *streamCmd) streamTemplateAdd(pc *kingpin.ParseContext) (err error) { + cfg := c.prepareConfig() + + if c.maxStreams == -1 { + err = survey.AskOne(&survey.Input{ + Message: "Maximum Streams", + }, &c.maxStreams, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "invalid input") + } + + if c.maxStreams < 0 { + kingpin.Fatalf("Maximum Streams can not be negative") + } + + cfg.Name = "" + + _, err = jsch.NewStreamTemplate(c.stream, uint32(c.maxStreams), cfg) + kingpin.FatalIfError(err, "could not create Stream Template") + + fmt.Printf("Stream Template %s was created\n\n", c.stream) + + return c.streamTemplateInfo(pc) +} + +func (c *streamCmd) streamTemplateInfo(_ *kingpin.ParseContext) error { + nc, err := newNatsConn(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + jsch.SetConnection(nc) + + c.stream, err = selectStreamTemplate(c.stream) + kingpin.FatalIfError(err, "could not pick a Stream Template to operate on") + + info, err := jsch.LoadStreamTemplate(c.stream) + kingpin.FatalIfError(err, "could not load Stream Template %q", c.stream) + + if c.json { + err = printJSON(info.Configuration()) + kingpin.FatalIfError(err, "could not display info") + return nil + } + + fmt.Printf("Information for Stream Template %s\n", c.stream) + fmt.Println() + c.showStreamConfig(info.StreamConfiguration()) + fmt.Printf(" Maximum Streams: %d\n", info.MaxStreams()) + fmt.Println() + fmt.Println("Managed Streams:") + fmt.Println() + if len(info.Streams()) == 0 { + fmt.Println(" No Streams have been defined by this template") + } else { + managed := info.Streams() + sort.Strings(managed) + for _, n := range managed { + fmt.Printf(" %s\n", n) + } + } + fmt.Println() + + return nil +} + +func (c *streamCmd) streamTemplateLs(_ *kingpin.ParseContext) error { + _, err := prepareHelper(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + + names, err := jsch.StreamTemplateNames() + kingpin.FatalIfError(err, "could not list Stream Templates") + + if c.json { + err = printJSON(names) + kingpin.FatalIfError(err, "could not display Stream Templates") + return nil + } + + if len(names) == 0 { + fmt.Println("No Streams Templates defined") + return nil + } + + fmt.Println("Stream Templates:") + fmt.Println() + for _, t := range names { + fmt.Printf("\t%s\n", t) + } + fmt.Println() + + return nil +} + +func (c *streamCmd) reportAction(pc *kingpin.ParseContext) error { + _, err := prepareHelper(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + + type stat struct { + Name string + Consumers int + Msgs int64 + Bytes uint64 + } + + if !c.json { + fmt.Print("Obtaining Stream stats\n\n") + } + + stats := []stat{} + jsch.EachStream(func(stream *jsch.Stream) { + info, err := stream.Information() + kingpin.FatalIfError(err, "could not get stream info for %s", stream.Name()) + stats = append(stats, stat{info.Config.Name, info.State.Consumers, int64(info.State.Msgs), info.State.Bytes}) + }) + + if len(stats) == 0 { + if !c.json { + fmt.Println("No Streams defined") + } + return nil + } + + if c.json { + j, err := json.MarshalIndent(stats, "", " ") + kingpin.FatalIfError(err, "could not JSON marshal stats") + fmt.Println(string(j)) + return nil + } + + if c.reportSortConsumers { + sort.Slice(stats, func(i, j int) bool { return stats[i].Consumers < stats[j].Consumers }) + } else if c.reportSortMsgs { + sort.Slice(stats, func(i, j int) bool { return stats[i].Msgs < stats[j].Msgs }) + } else if c.reportSortName { + sort.Slice(stats, func(i, j int) bool { return stats[i].Name < stats[j].Name }) + } else { + sort.Slice(stats, func(i, j int) bool { return stats[i].Bytes < stats[j].Bytes }) + } + + table := tablewriter.CreateTable() + table.AddHeaders("Stream", "Consumers", "Messages", "Bytes") + + for _, s := range stats { + if c.reportRaw { + table.AddRow(s.Name, s.Consumers, s.Msgs, s.Bytes) + } else { + table.AddRow(s.Name, s.Consumers, humanize.Comma(s.Msgs), humanize.IBytes(s.Bytes)) + } + } + + fmt.Println(table.Render()) + + return nil +} + +func (c *streamCmd) copyAndEditStream(cfg api.StreamConfig) (api.StreamConfig, error) { + var err error + + cfg.NoAck = !c.ack + + if len(c.subjects) > 0 { + cfg.Subjects = c.splitCLISubjects() + } + + if c.storage != "" { + cfg.Storage = c.storeTypeFromString(c.storage) + } + + if c.retentionPolicyS != "" { + cfg.Retention = c.retentionPolicyFromString(strings.ToLower(c.storage)) + } + + if c.maxBytesLimit != 0 { + cfg.MaxBytes = c.maxBytesLimit + } + + if c.maxMsgLimit != 0 { + cfg.MaxMsgs = c.maxMsgLimit + } + + if c.maxAgeLimit != "" { + cfg.MaxAge, err = parseDurationString(c.maxAgeLimit) + if err != nil { + return api.StreamConfig{}, fmt.Errorf("invalid maximum age limit format: %v", err) + } + } + + if c.maxMsgSize != 0 { + cfg.MaxMsgSize = c.maxMsgSize + } + + return cfg, nil +} + +func (c *streamCmd) editAction(pc *kingpin.ParseContext) error { + c.connectAndAskStream() + + sourceStream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not request Stream %s configuration", c.stream) + + cfg, err := c.copyAndEditStream(sourceStream.Configuration()) + kingpin.FatalIfError(err, "could not create new configuration for Stream %s", c.stream) + + err = sourceStream.UpdateConfiguration(cfg) + kingpin.FatalIfError(err, "could not edit Stream %s", c.stream) + + if !c.json { + fmt.Printf("Stream %s was updated\n\n", c.stream) + } + + return c.infoAction(pc) +} + +func (c *streamCmd) cpAction(pc *kingpin.ParseContext) error { + if c.stream == c.destination { + kingpin.Fatalf("source and destination Stream names cannot be the same") + } + + c.connectAndAskStream() + + sourceStream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not request Stream %s configuration", c.stream) + + cfg, err := c.copyAndEditStream(sourceStream.Configuration()) + kingpin.FatalIfError(err, "could not copy Stream %s", c.stream) + + cfg.Name = c.destination + + _, err = jsch.NewStreamFromDefault(cfg.Name, cfg) + kingpin.FatalIfError(err, "could not create Stream") + + if !c.json { + fmt.Printf("Stream %s was created\n\n", c.stream) + } + + c.stream = c.destination + return c.infoAction(pc) +} + +func (c *streamCmd) showStreamConfig(cfg api.StreamConfig) { + fmt.Println("Configuration:") + fmt.Println() + fmt.Printf(" Subjects: %s\n", strings.Join(cfg.Subjects, ", ")) + fmt.Printf(" Acknowledgements: %v\n", !cfg.NoAck) + fmt.Printf(" Retention: %s - %s\n", cfg.Storage.String(), cfg.Retention.String()) + fmt.Printf(" Replicas: %d\n", cfg.Replicas) + fmt.Printf(" Maximum Messages: %d\n", cfg.MaxMsgs) + fmt.Printf(" Maximum Bytes: %d\n", cfg.MaxBytes) + fmt.Printf(" Maximum Age: %s\n", cfg.MaxAge.String()) + fmt.Printf(" Maximum Message Size: %d\n", cfg.MaxMsgSize) + fmt.Printf(" Maximum Consumers: %d\n", cfg.MaxConsumers) + if cfg.Template != "" { + fmt.Printf(" Managed by Template: %s\n", cfg.Template) + } +} + +func (c *streamCmd) infoAction(_ *kingpin.ParseContext) error { + c.connectAndAskStream() + + stream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not request Stream info") + mstats, err := stream.Information() + kingpin.FatalIfError(err, "could not request Stream info") + + if c.json { + err = printJSON(mstats) + kingpin.FatalIfError(err, "could not display info") + return nil + } + + fmt.Printf("Information for Stream %s\n", c.stream) + fmt.Println() + c.showStreamConfig(mstats.Config) + fmt.Println() + fmt.Println("State:") + fmt.Println() + fmt.Printf(" Messages: %s\n", humanize.Comma(int64(mstats.State.Msgs))) + fmt.Printf(" Bytes: %s\n", humanize.IBytes(mstats.State.Bytes)) + fmt.Printf(" FirstSeq: %s\n", humanize.Comma(int64(mstats.State.FirstSeq))) + fmt.Printf(" LastSeq: %s\n", humanize.Comma(int64(mstats.State.LastSeq))) + fmt.Printf(" Active Consumers: %d\n", mstats.State.Consumers) + + fmt.Println() + + return nil +} + +func (c *streamCmd) splitCLISubjects() []string { + new := []string{} + + re := regexp.MustCompile(`,|\t|\s`) + for _, s := range c.subjects { + if re.MatchString(s) { + new = append(new, splitString(s)...) + } else { + new = append(new, s) + } + } + + return new +} + +func (c *streamCmd) storeTypeFromString(s string) api.StorageType { + switch s { + case "file", "f": + return api.FileStorage + case "memory", "m": + return api.MemoryStorage + default: + kingpin.Fatalf("invalid storage type %s", c.storage) + return 0 // unreachable + } +} + +func (c *streamCmd) retentionPolicyFromString(s string) api.RetentionPolicy { + switch strings.ToLower(c.retentionPolicyS) { + case "limits": + return api.LimitsPolicy + case "interest": + return api.InterestPolicy + case "work queue", "workq", "work": + return api.WorkQueuePolicy + default: + kingpin.Fatalf("invalid retention policy %s", c.retentionPolicyS) + return 0 // unreachable + } +} + +func (c *streamCmd) prepareConfig() (cfg api.StreamConfig) { + var err error + + if c.stream == "" { + err = survey.AskOne(&survey.Input{ + Message: "Stream Name", + }, &c.stream, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "invalid input") + } + + if len(c.subjects) == 0 { + subjects := "" + err = survey.AskOne(&survey.Input{ + Message: "Subjects to consume", + Help: "Streams consume messages from subjects, this is a space or comma separated list that can include wildcards. Settable using --subjects", + }, &subjects, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "invalid input") + + c.subjects = splitString(subjects) + } + + c.subjects = c.splitCLISubjects() + + if c.storage == "" { + err = survey.AskOne(&survey.Select{ + Message: "Storage backend", + Options: []string{"file", "memory"}, + Help: "Streams are stored on the server, this can be one of many backends and all are usable in clustering mode. Settable using --storage", + }, &c.storage, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "invalid input") + } + + storage := c.storeTypeFromString(c.storage) + + if c.retentionPolicyS == "" { + err = survey.AskOne(&survey.Select{ + Message: "Retention Policy", + Options: []string{"Limits", "Interest", "Work Queue"}, + Help: "Messages are retained either based on limits like size and age (Limits), as long as there are Consumers (Interest) or until any worker processed them (Work Queue)", + Default: "Limits", + }, &c.retentionPolicyS, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "invalid input") + } + + c.rPolicy = c.retentionPolicyFromString(strings.ToLower(c.retentionPolicyS)) + + var maxAge time.Duration + if c.maxMsgLimit == 0 { + c.maxMsgLimit, err = askOneInt("Message count limit", "-1", "Defines the amount of messages to keep in the store for this Stream, when exceeded oldest messages are removed, -1 for unlimited. Settable using --max-msgs") + kingpin.FatalIfError(err, "invalid input") + } + + if c.maxBytesLimit == 0 { + c.maxBytesLimit, err = askOneInt("Message size limit", "-1", "Defines the combined size of all messages in a Stream, when exceeded oldest messages are removed, -1 for unlimited. Settable using --max-bytes") + kingpin.FatalIfError(err, "invalid input") + } + + if c.maxAgeLimit == "" { + err = survey.AskOne(&survey.Input{ + Message: "Maximum message age limit", + Default: "-1", + Help: "Defines the oldest messages that can be stored in the Stream, any messages older than this period will be removed, -1 for unlimited. Supports units (s)econds, (m)inutes, (h)ours, (y)ears, (M)onths, (d)ays. Settable using --max-age", + }, &c.maxAgeLimit) + kingpin.FatalIfError(err, "invalid input") + } + + if c.maxAgeLimit != "-1" { + maxAge, err = parseDurationString(c.maxAgeLimit) + kingpin.FatalIfError(err, "invalid maximum age limit format") + } + + if c.maxMsgSize == 0 { + err = survey.AskOne(&survey.Input{ + Message: "Maximum individual message size", + Default: "-1", + Help: "Defines the maximum size any single message may be to be accepted by the Stream. Settable using --max-msg-size", + }, &c.maxMsgSize) + kingpin.FatalIfError(err, "invalid input") + } + + _, err = prepareHelper(servers, natsOpts()...) + kingpin.FatalIfError(err, "could not create Stream") + + cfg = api.StreamConfig{ + Name: c.stream, + Subjects: c.subjects, + MaxMsgs: c.maxMsgLimit, + MaxBytes: c.maxBytesLimit, + MaxMsgSize: c.maxMsgSize, + MaxAge: maxAge, + Storage: storage, + NoAck: !c.ack, + Retention: c.rPolicy, + MaxConsumers: -1, + Replicas: 0, + } + + return cfg +} + +func (c *streamCmd) addAction(pc *kingpin.ParseContext) (err error) { + cfg := c.prepareConfig() + + _, err = jsch.NewStreamFromDefault(c.stream, cfg) + kingpin.FatalIfError(err, "could not create Stream") + + fmt.Printf("Stream %s was created\n\n", c.stream) + + return c.infoAction(pc) +} + +func (c *streamCmd) rmAction(_ *kingpin.ParseContext) (err error) { + c.connectAndAskStream() + + if !c.force { + ok, err := askConfirmation(fmt.Sprintf("Really delete Stream %s", c.stream), false) + kingpin.FatalIfError(err, "could not obtain confirmation") + + if !ok { + return nil + } + } + + stream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not remove Stream") + + err = stream.Delete() + kingpin.FatalIfError(err, "could not remove Stream") + + return nil +} + +func (c *streamCmd) purgeAction(pc *kingpin.ParseContext) (err error) { + c.connectAndAskStream() + + if !c.force { + ok, err := askConfirmation(fmt.Sprintf("Really purge Stream %s", c.stream), false) + kingpin.FatalIfError(err, "could not obtain confirmation") + + if !ok { + return nil + } + } + + stream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not purge Stream") + + err = stream.Purge() + kingpin.FatalIfError(err, "could not purge Stream") + + return c.infoAction(pc) +} + +func (c *streamCmd) lsAction(_ *kingpin.ParseContext) (err error) { + prepareHelper(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + + streams, err := jsch.StreamNames() + kingpin.FatalIfError(err, "could not list Streams") + + if c.json { + err = printJSON(streams) + kingpin.FatalIfError(err, "could not display Streams") + return nil + } + + if len(streams) == 0 { + fmt.Println("No Streams defined") + return nil + } + + fmt.Println("Streams:") + fmt.Println() + for _, s := range streams { + fmt.Printf("\t%s\n", s) + } + fmt.Println() + + return nil +} + +func (c *streamCmd) rmMsgAction(_ *kingpin.ParseContext) (err error) { + c.connectAndAskStream() + + if c.msgID == -1 { + id := "" + err = survey.AskOne(&survey.Input{ + Message: "Message ID to remove", + }, &id, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "invalid input") + + idint, err := strconv.Atoi(id) + kingpin.FatalIfError(err, "invalid number") + + c.msgID = int64(idint) + } + + stream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not load Stream %s", c.stream) + + if !c.force { + ok, err := askConfirmation(fmt.Sprintf("Really remove message %d from Stream %s", c.msgID, c.stream), false) + kingpin.FatalIfError(err, "could not obtain confirmation") + + if !ok { + return nil + } + } + + return stream.DeleteMessage(int(c.msgID)) +} + +func (c *streamCmd) getAction(_ *kingpin.ParseContext) (err error) { + c.connectAndAskStream() + + if c.msgID == -1 { + id := "" + err = survey.AskOne(&survey.Input{ + Message: "Message ID to retrieve", + }, &id, survey.WithValidator(survey.Required)) + kingpin.FatalIfError(err, "invalid input") + + idint, err := strconv.Atoi(id) + kingpin.FatalIfError(err, "invalid number") + + c.msgID = int64(idint) + } + + stream, err := jsch.LoadStream(c.stream) + kingpin.FatalIfError(err, "could not load Stream %s", c.stream) + + item, err := stream.LoadMessage(int(c.msgID)) + kingpin.FatalIfError(err, "could not retrieve %s#%d", c.stream, c.msgID) + + if c.json { + printJSON(item) + return nil + } + + fmt.Printf("Item: %s#%d received %v on Subject %s\n\n", c.stream, c.msgID, item.Time, item.Subject) + fmt.Println(string(item.Data)) + fmt.Println() + return nil +} + +func (c *streamCmd) connectAndAskStream() { + nc, err := newNatsConn(servers, natsOpts()...) + kingpin.FatalIfError(err, "setup failed") + jsch.SetConnection(nc) + + c.stream, err = selectStream(c.stream) + kingpin.FatalIfError(err, "could not pick a Stream to operate on") +} diff --git a/nats/sub_command.go b/nats/sub_command.go new file mode 100644 index 0000000..cddabaa --- /dev/null +++ b/nats/sub_command.go @@ -0,0 +1,67 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + + "github.com/nats-io/nats.go" + "gopkg.in/alecthomas/kingpin.v2" +) + +type subCmd struct { + subject string + queue string +} + +func configureSubCommand(app *kingpin.Application) { + c := &subCmd{} + act := app.Command("sub", "Generic subscription client").Action(c.subscribe) + act.Arg("subject", "Subject to subscribe to").Required().StringVar(&c.subject) + act.Flag("queue", "Subscribe to a named queue group").StringVar(&c.queue) +} + +func (c *subCmd) subscribe(_ *kingpin.ParseContext) error { + nc, err := newNatsConn(servers, natsOpts()...) + if err != nil { + return err + } + defer nc.Close() + + i := 0 + + handler := func(m *nats.Msg) { + i += 1 + log.Printf("[#%d] Received on [%s]: '%s'", i, m.Subject, string(m.Data)) + } + + if c.queue != "" { + nc.QueueSubscribe(c.subject, c.queue, handler) + } else { + nc.Subscribe(c.subject, handler) + } + nc.Flush() + + err = nc.LastError() + if err != nil { + return err + } + + log.Printf("Listening on [%s]", c.subject) + + <-context.Background().Done() + + return nil +} diff --git a/nats/util.go b/nats/util.go new file mode 100644 index 0000000..d7c7132 --- /dev/null +++ b/nats/util.go @@ -0,0 +1,330 @@ +// Copyright 2020 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "strconv" + "strings" + "time" + "unicode" + + "github.com/AlecAivazis/survey/v2" + "github.com/nats-io/nats.go" + + "github.com/nats-io/jetstream/internal/jsch" +) + +func selectConsumer(stream string, consumer string) (string, error) { + if consumer != "" { + known, err := jsch.IsKnownConsumer(stream, consumer) + if err != nil { + return "", err + } + + if known { + return consumer, nil + } + } + + consumers, err := jsch.ConsumerNames(stream) + if err != nil { + return "", err + } + + switch len(consumers) { + case 0: + return "", fmt.Errorf("no Consumers are defined for Stream %s", stream) + default: + c := "" + + err = survey.AskOne(&survey.Select{ + Message: "Select a Consumer", + Options: consumers, + }, &c) + if err != nil { + return "", err + } + + return c, nil + } +} + +func selectStreamTemplate(template string) (string, error) { + if template != "" { + known, err := jsch.IsKnownStreamTemplate(template) + if err != nil { + return "", err + } + + if known { + return template, nil + } + } + + templates, err := jsch.StreamTemplateNames() + if err != nil { + return "", err + } + + switch len(templates) { + case 0: + return "", errors.New("no Streams Templates are defined") + default: + s := "" + + err = survey.AskOne(&survey.Select{ + Message: "Select a Stream Template", + Options: templates, + }, &s) + if err != nil { + return "", err + } + + return s, nil + } +} + +func selectStream(stream string) (string, error) { + if stream != "" { + known, err := jsch.IsKnownStream(stream) + if err != nil { + return "", err + } + + if known { + return stream, nil + } + } + + streams, err := jsch.StreamNames() + if err != nil { + return "", err + } + + switch len(streams) { + case 0: + return "", errors.New("no Streams are defined") + default: + s := "" + + err = survey.AskOne(&survey.Select{ + Message: "Select a Stream", + Options: streams, + }, &s) + if err != nil { + return "", err + } + + return s, nil + } +} + +func printJSON(d interface{}) error { + j, err := json.MarshalIndent(d, "", " ") + if err != nil { + return err + } + + fmt.Println(string(j)) + + return nil +} +func parseDurationString(dstr string) (dur time.Duration, err error) { + dstr = strings.TrimSpace(dstr) + + if len(dstr) <= 0 { + return dur, nil + } + + ls := len(dstr) + di := ls - 1 + unit := dstr[di:] + + switch unit { + case "w", "W": + val, err := strconv.ParseFloat(dstr[:di], 32) + if err != nil { + return dur, err + } + + dur = time.Duration(val*7*24) * time.Hour + + case "d", "D": + val, err := strconv.ParseFloat(dstr[:di], 32) + if err != nil { + return dur, err + } + + dur = time.Duration(val*24) * time.Hour + case "M": + val, err := strconv.ParseFloat(dstr[:di], 32) + if err != nil { + return dur, err + } + + dur = time.Duration(val*24*30) * time.Hour + case "Y", "y": + val, err := strconv.ParseFloat(dstr[:di], 32) + if err != nil { + return dur, err + } + + dur = time.Duration(val*24*365) * time.Hour + case "s", "S", "m", "h", "H": + dur, err = time.ParseDuration(dstr) + if err != nil { + return dur, err + } + + default: + return dur, fmt.Errorf("invalid time unit %s", unit) + } + + return dur, nil +} + +func askConfirmation(prompt string, dflt bool) (bool, error) { + ans := dflt + + err := survey.AskOne(&survey.Confirm{ + Message: prompt, + Default: dflt, + }, &ans) + + return ans, err +} + +func askOneInt(prompt string, dflt string, help string) (int64, error) { + val := "" + err := survey.AskOne(&survey.Input{ + Message: prompt, + Default: dflt, + Help: help, + }, &val) + if err != nil { + return 0, err + } + + i, err := strconv.Atoi(val) + if err != nil { + return 0, err + } + + return int64(i), nil +} + +func splitString(s string) []string { + return strings.FieldsFunc(s, func(c rune) bool { + if unicode.IsSpace(c) { + return true + } + + if c == ',' { + return true + } + + return false + }) +} + +func natsOpts() []nats.Option { + totalWait := 10 * time.Minute + reconnectDelay := time.Second + + opts := []nats.Option{ + nats.Name("NATS CLI"), + nats.MaxReconnects(-1), + nats.ReconnectWait(reconnectDelay), + nats.MaxReconnects(int(totalWait / reconnectDelay)), + nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { + if err != nil { + log.Printf("Disconnected due to: %s, will attempt reconnects for %.0fm", err.Error(), totalWait.Minutes()) + } + }), + nats.ReconnectHandler(func(nc *nats.Conn) { + log.Printf("Reconnected [%s]", nc.ConnectedUrl()) + }), + nats.ClosedHandler(func(nc *nats.Conn) { + err := nc.LastError() + if err != nil { + log.Fatalf("Exiting: %v", nc.LastError()) + } + }), + } + + if creds != "" { + opts = append(opts, nats.UserCredentials(creds)) + } + + if tlsCert != "" && tlsKey != "" { + opts = append(opts, nats.ClientCert(tlsCert, tlsKey)) + } + + if tlsCA != "" { + opts = append(opts, nats.RootCAs(tlsCA)) + } + + return opts +} + +func newNatsConn(servers string, opts ...nats.Option) (*nats.Conn, error) { + return nats.Connect(servers, opts...) +} + +func prepareHelper(servers string, opts ...nats.Option) (*nats.Conn, error) { + nc, err := newNatsConn(servers, opts...) + if err != nil { + return nil, err + } + + if timeout != 0 { + jsch.SetTimeout(timeout) + } + + jsch.SetConnection(nc) + + return nc, err +} + +func humanizeTime(t time.Time) string { + d := time.Since(t) + // Just use total seconds for uptime, and display days / years + tsecs := d / time.Second + tmins := tsecs / 60 + thrs := tmins / 60 + tdays := thrs / 24 + tyrs := tdays / 365 + + if tyrs > 0 { + return fmt.Sprintf("%dy%dd%dh%dm%ds", tyrs, tdays%365, thrs%24, tmins%60, tsecs%60) + } + + if tdays > 0 { + return fmt.Sprintf("%dd%dh%dm%ds", tdays, thrs%24, tmins%60, tsecs%60) + } + + if thrs > 0 { + return fmt.Sprintf("%dh%dm%ds", thrs, tmins%60, tsecs%60) + } + + if tmins > 0 { + return fmt.Sprintf("%dm%ds", tmins, tsecs%60) + } + + return fmt.Sprintf("%ds", tsecs) +} diff --git a/nats/util_test.go b/nats/util_test.go new file mode 100644 index 0000000..8872d3e --- /dev/null +++ b/nats/util_test.go @@ -0,0 +1,82 @@ +// Copyright 2019 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" +) + +func checkErr(t *testing.T, err error, format string, a ...interface{}) { + t.Helper() + if err == nil { + return + } + + t.Fatalf(format, a...) +} + +func TestSplitString(t *testing.T) { + for _, s := range []string{"x y", "x y", "x y", "x,y", "x, y"} { + parts := splitString(s) + if parts[0] != "x" && parts[1] != "y" { + t.Fatalf("Expected x and y from %s, got %v", s, parts) + } + } + + parts := splitString("x foo.*") + if parts[0] != "x" && parts[1] != "y" { + t.Fatalf("Expected x and foo.* from 'x foo.*', got %v", parts) + } +} + +func TestParseDurationString(t *testing.T) { + d, err := parseDurationString("") + checkErr(t, err, "failed to parse empty duration: %s", err) + if d.Nanoseconds() != 0 { + t.Fatalf("expected 0 ns from empty duration, got %v", d) + } + + _, err = parseDurationString("1f") + if err.Error() != "invalid time unit f" { + t.Fatal("expected time unit 'f' to fail but it did not") + } + + for _, u := range []string{"d", "D"} { + d, err = parseDurationString("1.1" + u) + checkErr(t, err, "failed to parse 1.1%s duration: %s", u, err) + if d.Hours() != 26 { + t.Fatalf("expected 1 hour from 1.1%s duration, got %v", u, d) + } + } + + d, err = parseDurationString("1.1M") + checkErr(t, err, "failed to parse 1.1M duration: %s", err) + if d.Hours() != 1.1*24*30 { + t.Fatalf("expected 30 days from 1.1M duration, got %v", d) + } + + for _, u := range []string{"y", "Y"} { + d, err = parseDurationString("1.1" + u) + checkErr(t, err, "failed to parse 1.1%s duration: %s", u, err) + if d.Hours() != 1.1*24*365 { + t.Fatalf("expected 1.1 year from 1.1%s duration, got %v", u, d) + } + } + + d, err = parseDurationString("1.1h") + checkErr(t, err, "failed to parse 1.1h duration: %s", err) + if d.Minutes() != 66 { + t.Fatalf("expected 1.1 hour from 1.1h duration, got %v", d) + } +}