Add some load testing tools to the sandbox.

Signed-off-by: Hiram Chirino <[email protected]>

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["limitador", "limitador-server"]
+members = ["limitador", "limitador-server", "limitador-server/sandbox/loadtest"]
 resolver = "2"
 
 [profile.release]

Dockerfile

@@ -23,7 +23,9 @@ ENV CARGO_ARGS=${CARGO_ARGS}
 COPY Cargo.toml Cargo.lock ./
 COPY limitador/Cargo.toml ./limitador/
 COPY limitador-server/Cargo.toml ./limitador-server/
+COPY limitador-server/sandbox/loadtest/Cargo.toml ./limitador-server/sandbox/loadtest/
 RUN mkdir -p limitador-server/src && echo 'fn main() {}' > limitador-server/src/main.rs
+RUN mkdir -p limitador-server/sandbox/loadtest/src && echo 'fn main() {}' > limitador-server/sandbox/loadtest/src/main.rs
 RUN cargo build --release ${CARGO_ARGS}
 
 COPY ./limitador ./limitador

limitador-server/sandbox/.gitignore

@@ -3,3 +3,4 @@
 *.key
 *.pem
 *.csr
+report.html

limitador-server/sandbox/Makefile

@@ -45,6 +45,9 @@ deploy-redis-otel: clean ## Uses Redis to store counters, instrumented with open
 deploy-disk: clean ## Uses disk to store counters
 	$(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-disk.yaml up
 
+deploy-distributed: clean ## Counters are held in Limitador (ephemeral) but replicated to other Limitador servers.
+	$(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-distributed.yaml up
+
 ##@ Helper targets
 
 build: ## Build "limitador-testing" image
@@ -88,6 +91,21 @@ $(GRPCURL):
 .PHONY: grpcurl
 grpcurl: $(GRPCURL) ## Download grpcurl locally if necessary.
 
+.PHONY: ghz
+ghz:
+	$(call go-install-tool,$(PROJECT_PATH)/bin/ghz,github.com/bojand/ghz/cmd/ghz@latest) 
+
+RPS?=1000
+.PHONY: load-test
+load-test: ghz
+	# see https://ghz.sh/docs/load for usage details
+	$(PROJECT_PATH)/bin/ghz 127.0.0.1:18081 --insecure \
+	--call envoy.service.ratelimit.v3.RateLimitService.ShouldRateLimit \
+	--async --concurrency=50 \
+	--rps=$(RPS) \
+	--total=$(RPS)0 \
+	--data-file load-test.json
+
 # go-install-tool will 'go install' any package $2 and install it to $1.
 define go-install-tool
 @[ -f $(1) ] || { \

limitador-server/sandbox/README.md

@@ -17,14 +17,39 @@ Check out `make help` for all the targets.
 
 ### Deployment options
 
-| Limitador's configuration | Command | Info |
-| ------------- | ----- | ----- |
-| In-memory configuration | `make deploy-in-memory` | Counters are held in Limitador (ephemeral) |
-| Redis | `make deploy-redis` | Uses Redis to store counters |
-| Redis Secured | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters |
-| Redis Cached | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache |
-| Redis Otel Instrumented | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md) |
-| Disk | `make deploy-disk` | Uses disk to store counters |
+| Limitador's configuration | Command | Info                                                                                                           |
+|--------------------------| ----- |----------------------------------------------------------------------------------------------------------------|
+| In-memory configuration  | `make deploy-in-memory` | Counters are held in Limitador (ephemeral)                                                                     |
+| Redis                    | `make deploy-redis` | Uses Redis to store counters                                                                                   |
+| Redis Secured            | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters                                                   |
+| Redis Cached             | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache                                                          |
+| Redis Otel Instrumented  | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md)                          |
+| Disk                     | `make deploy-disk` | Uses disk to store counters                                                                                    |
+| Distributed | `make deploy-distributed` | Counters are held in Limitador (ephemeral) but replicated to other Limitador servers. |
+
+
+### Running Multi Node Distributed Deployments
+
+The `make deploy-distributed` target can be connected to other Limitador servers but requires you to set the `PEER_ID` and `PEER_URLS` environment variables when you run the target.
+
+If you have 3 servers you want to replicate between, you would run the following commands:
+
+```bash
+# on server where: hostname=server1
+PEER_ID=`hostname` PEER_URLS="http://server2:15001 http://server3:15001" make deploy-distributed
+```
+
+```bash
+# on server where: hostname=server2
+PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server3:15001" make deploy-distributed
+```
+
+```bash
+# on server where: hostname=server3
+PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server2:15001" make deploy-distributed
+```
+
+The `PEER_ID` just need to be unique between the servers, and the `PEER_URLS` should be a space-separated list of the other servers' URLs.
 
 ### Limitador's admin HTTP endpoint
 
@@ -75,6 +100,10 @@ bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit.v3.RateLimit
                 {
                     "key": "req.method",
                     "value": "POST"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/"
                 }
             ]
         }
@@ -97,6 +126,10 @@ while :; do bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit.
                 {
                     "key": "req.method",
                     "value": "POST"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/"
                 }
             ]
         }
@@ -113,6 +146,24 @@ EOM
 curl -i -H "Host: example.com" http://127.0.0.1:18000/get
 ```
 
+### Load Testing the GRPC RateLimitService directly
+
+This load test will use `grpcurl`. You need [Go SDK](https://golang.org/doc/install) installed.
+
+Run a load test a 5000 requests per second (RPS) for 10 seconds:
+
+```bash
+RPS=5000 make load-test
+```
+
+### Load Testing via Envoy Proxy
+
+```bash
+cargo run --package loadtest --release -- --report-file=report.htm
+```
+
+The report will be saved in `report.htm` file.
+
 ### Limitador Image
 
 By default, the sandbox will run Limitador's `limitador-testing:latest` image.

limitador-server/sandbox/docker-compose-limitador-disk.yaml

@@ -3,6 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - envoy
     command:

limitador-server/sandbox/docker-compose-limitador-distributed.yaml

@@ -2,31 +2,26 @@
 version: '3.8'
 services:
   limitador:
-    image: ${LIMITADOR_IMAGE:-limitador-testing}
+    image: limitador-testing-all-features
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+      args:
+        - CARGO_ARGS=--all-features
     depends_on:
       - envoy
-      - redis
-    command:
-      - limitador-server
-      - --rls-ip
-      - 0.0.0.0
-      - --rls-port
-      - "8081"
-      - --http-ip
-      - 0.0.0.0
-      - --http-port
-      - "8080"
-      - -vvv
-      - --grpc-reflection-service
-      - /opt/kuadrant/limits/limits.yaml
-      - redis
-      - redis://redis:6379
+    command: |
+      limitador-server --rls-ip 0.0.0.0 --rls-port 8081 --http-ip 0.0.0.0 --http-port "8080" 
+      -vv --grpc-reflection-service /opt/kuadrant/limits/limits.yaml 
+      distributed ${PEER_ID:-node1} 0.0.0.0:5001 ${PEER_URLS:-}
     expose:
       - "8080"
       - "8081"
+      - "5001"
     ports:
       - "18080:8080"
       - "18081:8081"
+      - "15001:5001"
     volumes:
       - ./limits.yaml:/opt/kuadrant/limits/limits.yaml
   redis:

limitador-server/sandbox/docker-compose-limitador-memory.yaml

@@ -3,6 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - envoy
     command:

limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml

@@ -3,6 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - envoy
       - redis

limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml

@@ -3,6 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - jaeger
       - envoy

limitador-server/sandbox/docker-compose-limitador-redis-tls.yaml

@@ -3,6 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - envoy
       - redis

limitador-server/sandbox/docker-compose-limitador-redis.yaml

@@ -3,6 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - envoy
       - redis

limitador-server/sandbox/limits.yaml

@@ -4,10 +4,19 @@
   seconds: 60
   conditions:
     - "req.method == 'GET'"
+    - "req.path != '/json'"
   variables: []
 - namespace: test_namespace
   max_value: 5
   seconds: 60
   conditions:
     - "req.method == 'POST'"
+    - "req.path != '/json'"
+  variables: []
+- namespace: test_namespace
+  max_value: 50000
+  seconds: 10
+  conditions:
+    - "req.method == 'GET'"
+    - "req.path == '/json'"
   variables: []

limitador-server/sandbox/load-test.json

@@ -0,0 +1,18 @@
+{
+    "domain": "test_namespace",
+    "hits_addend": 1,
+    "descriptors": [
+        {   
+            "entries": [
+                {
+                    "key": "req.method",
+                    "value": "GET"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/json"
+                }
+            ]
+        }
+    ]
+}

limitador-server/sandbox/loadtest/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "loadtest"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+goose = "^0.17"
+tokio = "^1.12"

limitador-server/sandbox/loadtest/src/main.rs

@@ -0,0 +1,26 @@
+use goose::prelude::*;
+
+async fn loadtest_get_json(user: &mut GooseUser) -> TransactionResult {
+    let _goose_metrics = user.get("/json").await?;
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> Result<(), GooseError> {
+    GooseAttack::initialize()?
+        .register_scenario(
+            scenario!("LoadtestTransactions").register_transaction(transaction!(loadtest_get_json)),
+        )
+        .set_default(GooseDefault::Host, "http://localhost:18000")?
+        .set_default(GooseDefault::HatchRate, "2")?
+        .set_default(
+            GooseDefault::CoordinatedOmissionMitigation,
+            GooseCoordinatedOmissionMitigation::Average,
+        )?
+        .set_default(GooseDefault::RunTime, 20)?
+        .execute()
+        .await?;
+
+    Ok(())
+}