diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 65ac41695..1c8bc7d2a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -200,8 +200,8 @@ jobs: shell: bash run: | echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" - $(go env GOPATH)/bin/simple-transfer create -d simple-transfer-${GITHUB_RUN_NUMBER} -n simple-transfer-${GITHUB_RUN_NUMBER} -l environment=gh-fablab-smoketest,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version) - $(go env GOPATH)/bin/simple-transfer up + $(go env GOPATH)/bin/smoketest create -d smoketest-${GITHUB_RUN_NUMBER} -n smoketest-${GITHUB_RUN_NUMBER} -l environment=gh-fablab-smoketest,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version) + $(go env GOPATH)/bin/smoketest up - name: Test Ziti Command shell: bash @@ -217,15 +217,15 @@ jobs: FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }} shell: bash run: | - cp $(go env GOPATH)/bin/simple-transfer simple-transfer-${GITHUB_RUN_NUMBER} - cp ~/.fablab/config.yml simple-transfer-${GITHUB_RUN_NUMBER}/ - tar --exclude *terraform-provider-aws* -czvf - simple-transfer-${GITHUB_RUN_NUMBER}/ | gpg --symmetric --cipher-algo aes256 --batch --quiet --passphrase ${FABLAB_PASSPHRASE} -o simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg - aws s3 cp ./simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg s3://ziti-smoketest-fablab-instances/ + cp $(go env GOPATH)/bin/smoketest smoketest-${GITHUB_RUN_NUMBER} + cp ~/.fablab/config.yml smoketest-${GITHUB_RUN_NUMBER}/ + tar --exclude *terraform-provider-aws* -czvf - smoketest-${GITHUB_RUN_NUMBER}/ | gpg --symmetric --cipher-algo aes256 --batch --quiet --passphrase ${FABLAB_PASSPHRASE} -o smoketest-${GITHUB_RUN_NUMBER}.tar.gz.gpg + aws s3 cp ./smoketest-${GITHUB_RUN_NUMBER}.tar.gz.gpg s3://ziti-smoketest-fablab-instances/ - name: Create Logs Archive if: always() run: | - $(go env GOPATH)/bin/simple-transfer get files '*' "./logs/{{ .Id }}/" ./logs + $(go env GOPATH)/bin/smoketest get files '*' "./logs/{{ .Id }}/" ./logs - name: Upload artifacts uses: actions/upload-artifact@v4 @@ -275,16 +275,16 @@ jobs: run: | if aws s3api head-object \ --bucket ziti-smoketest-fablab-instances \ - --key simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg + --key smoketest-${GITHUB_RUN_NUMBER}.tar.gz.gpg then - aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg . - gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg - tar -xzf simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz - ./simple-transfer-${GITHUB_RUN_NUMBER}/simple-transfer import simple-transfer-${GITHUB_RUN_NUMBER} - ./simple-transfer-${GITHUB_RUN_NUMBER}/simple-transfer dispose - aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg + aws s3 cp s3://ziti-smoketest-fablab-instances/smoketest-${GITHUB_RUN_NUMBER}.tar.gz.gpg . + gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output smoketest-${GITHUB_RUN_NUMBER}.tar.gz smoketest-${GITHUB_RUN_NUMBER}.tar.gz.gpg + tar -xzf smoketest-${GITHUB_RUN_NUMBER}.tar.gz + ./smoketest-${GITHUB_RUN_NUMBER}/smoketest import smoketest-${GITHUB_RUN_NUMBER} + ./smoketest-${GITHUB_RUN_NUMBER}/smoketest dispose + aws s3 rm s3://ziti-smoketest-fablab-instances/smoketest-${GITHUB_RUN_NUMBER}.tar.gz.gpg else - echo "WARN: No instance archive found for simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2 + echo "WARN: No instance archive found for smoketest-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2 fi fablab-ha-smoketest: @@ -322,8 +322,8 @@ jobs: shell: bash run: | echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" - $(go env GOPATH)/bin/simple-transfer create -d simple-transfer-ha-${GITHUB_RUN_NUMBER} -n simple-transfer-ha-${GITHUB_RUN_NUMBER} -l ha=true,environment=gh-fablab-ha-smoketest,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version) - $(go env GOPATH)/bin/simple-transfer up + $(go env GOPATH)/bin/smoketest create -d smoketest-ha-${GITHUB_RUN_NUMBER} -n smoketest-ha-${GITHUB_RUN_NUMBER} -l ha=true,environment=gh-fablab-ha-smoketest,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version) + $(go env GOPATH)/bin/smoketest up # - name: Test Ziti Command # shell: bash @@ -337,10 +337,10 @@ jobs: FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }} shell: bash run: | - cp $(go env GOPATH)/bin/simple-transfer simple-transfer-ha-${GITHUB_RUN_NUMBER} - cp ~/.fablab/config.yml simple-transfer-ha-${GITHUB_RUN_NUMBER}/ - tar --exclude *terraform-provider-aws* -czvf - simple-transfer-ha-${GITHUB_RUN_NUMBER}/ | gpg --symmetric --cipher-algo aes256 --batch --quiet --passphrase ${FABLAB_PASSPHRASE} -o simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg - aws s3 cp ./simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg s3://ziti-smoketest-fablab-instances/ + cp $(go env GOPATH)/bin/smoketest smoketest-ha-${GITHUB_RUN_NUMBER} + cp ~/.fablab/config.yml smoketest-ha-${GITHUB_RUN_NUMBER}/ + tar --exclude *terraform-provider-aws* -czvf - smoketest-ha-${GITHUB_RUN_NUMBER}/ | gpg --symmetric --cipher-algo aes256 --batch --quiet --passphrase ${FABLAB_PASSPHRASE} -o smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg + aws s3 cp ./smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg s3://ziti-smoketest-fablab-instances/ # - name: Test Report Generation # if: always() @@ -381,16 +381,16 @@ jobs: run: | if aws s3api head-object \ --bucket ziti-smoketest-fablab-instances \ - --key simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg + --key smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg then - aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg . - gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg - tar -xzf simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz - ./simple-transfer-ha-${GITHUB_RUN_NUMBER}/simple-transfer import simple-transfer-ha-${GITHUB_RUN_NUMBER} - ./simple-transfer-ha-${GITHUB_RUN_NUMBER}/simple-transfer dispose - aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg + aws s3 cp s3://ziti-smoketest-fablab-instances/smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg . + gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg + tar -xzf smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz + ./smoketest-ha-${GITHUB_RUN_NUMBER}/smoketest import smoketest-ha-${GITHUB_RUN_NUMBER} + ./smoketest-ha-${GITHUB_RUN_NUMBER}/smoketest dispose + aws s3 rm s3://ziti-smoketest-fablab-instances/smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg else - echo "WARN: No instance archive found for simple-transfer-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2 + echo "WARN: No instance archive found for smoketest-ha-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2 fi publish: diff --git a/.github/workflows/release-quickstart.yml b/.github/workflows/release-quickstart.yml index c1ed7f739..cd26f572e 100644 --- a/.github/workflows/release-quickstart.yml +++ b/.github/workflows/release-quickstart.yml @@ -34,7 +34,7 @@ jobs: uses: hmarr/debug-action@v3.0.0 - name: Wait for other builds to complete - uses: lewagon/wait-on-check-action@v1.3.3 + uses: lewagon/wait-on-check-action@v1.3.4 with: ref: ${{ env.GITHUB_SHA }} repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-validation.yml b/.github/workflows/release-validation.yml new file mode 100644 index 000000000..b168e3260 --- /dev/null +++ b/.github/workflows/release-validation.yml @@ -0,0 +1,156 @@ +name: release-validation + +on: + workflow_dispatch: + +# cancel older, redundant runs of same workflow on same branch +concurrency: + group: ${{ github.workflow }}-${{github.event_name}}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + GOFLAGS: "-trimpath" + GOX_OUTPUT: "release/{{.Arch}}/{{.OS}}/{{.Dir}}" + GOX_TEST_OUTPUT: "test/{{.Arch}}/{{.OS}}/bin/{{.Dir}}" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "us-east-2" + gh_ci_key: ${{ secrets.GH_CI_KEY }} + BUILD_NUMBER: ${{ format('{0}-{1}-{2}', github.run_id, github.run_number, github.run_attempt) }} + + +jobs: + link-validation: + name: Link Validation + # not applicable to forks. shouldn't run on release build + if: github.repository_owner == 'openziti' + runs-on: ubuntu-20.04 + steps: + - name: Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: ./go.mod + + - name: Install Ziti CI + uses: openziti/ziti-ci@v1 + + - name: Build and Test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ziti_ci_gpg_key: ${{ secrets.ZITI_CI_GPG_KEY }} + ziti_ci_gpg_key_id: ${{ secrets.ZITI_CI_GPG_KEY_ID }} + shell: bash + run: | + $(go env GOPATH)/bin/ziti-ci configure-git + $(go env GOPATH)/bin/ziti-ci generate-build-info common/version/info_generated.go version + pushd zititest && go install ./... && popd + go install -tags=all,tests ./... + + - name: Create Test Environment + shell: bash + run: | + echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" + $(go env GOPATH)/bin/links-test create -d links-test-${GITHUB_RUN_NUMBER} -n links-test-${GITHUB_RUN_NUMBER} -l environment=gh-fablab-links-test,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version) + $(go env GOPATH)/bin/links-test up + $(go env GOPATH)/bin/links-test exec validateUp + + - name: Run Loop Validation + shell: bash + timeout-minutes: 380 + run: | + echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" + $(go env GOPATH)/bin/links-test exec-loop 4h sowChaos validateUp validateLinks + + - name: Create Logs Archive + if: always() + run: | + $(go env GOPATH)/bin/links-test get files '*' "./logs/{{ .Id }}/" ./logs + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: links-test-logs-${{ github.run_id }} + path: logs/ + compression-level: 7 + retention-days: 5 + + - name: Tear down Test Environment + timeout-minutes: 30 + if: always() + shell: bash + run: | + $(go env GOPATH)/bin/links-test dispose + + sdk-terminator-validation: + name: SDK Terminator Validation + # not applicable to forks. shouldn't run on release build + if: github.repository_owner == 'openziti' + runs-on: ubuntu-20.04 + steps: + - name: Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: ./go.mod + + - name: Install Ziti CI + uses: openziti/ziti-ci@v1 + + - name: Build and Test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ziti_ci_gpg_key: ${{ secrets.ZITI_CI_GPG_KEY }} + ziti_ci_gpg_key_id: ${{ secrets.ZITI_CI_GPG_KEY_ID }} + shell: bash + run: | + $(go env GOPATH)/bin/ziti-ci configure-git + $(go env GOPATH)/bin/ziti-ci generate-build-info common/version/info_generated.go version + pushd zititest && go install ./... && popd + go install -tags=all,tests ./... + + - name: Create Test Environment + shell: bash + run: | + echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" + $(go env GOPATH)/bin/sdk-hosting-test create -d sdk-hosting-test-${GITHUB_RUN_NUMBER} -n sdk-hosting-test-${GITHUB_RUN_NUMBER} -l environment=gh-fablab-sdk-hosting-test,ziti_version=$($(go env GOPATH)/bin/ziti-ci -q get-current-version) + $(go env GOPATH)/bin/sdk-hosting-test up + $(go env GOPATH)/bin/sdk-hosting-test exec validateUp + + - name: Run SDK Terminator Validation + shell: bash + timeout-minutes: 380 + run: | + echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" + $(go env GOPATH)/bin/sdk-hosting-test exec-loop 4h sowChaos validateUp validate + + - name: Create Logs Archive + if: always() + run: | + $(go env GOPATH)/bin/sdk-hosting-test get files '*' "./logs/{{ .Id }}/" ./logs + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: sdk-hosting-test-logs-${{ github.run_id }} + path: logs/ + compression-level: 7 + retention-days: 5 + + - name: Tear down Test Environment + timeout-minutes: 30 + if: always() + shell: bash + run: | + $(go env GOPATH)/bin/sdk-hosting-test dispose + diff --git a/CHANGELOG.md b/CHANGELOG.md index ae1fbc05a..8d44f6208 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,1326 +1,100 @@ -# Release 0.34.2 +# Release 1.0.0 -## What's New - -* The circuit id is now available in the SDK on the client and hosting side - * Requires 0.34.2+ routers - * Requests SDK support. Currently supported in the Go SDK 0.23.11+ -* Bug fixes - -## Component Updates and Bug Fixes -* github.com/openziti/edge-api: [v0.26.13 -> v0.26.14](https://github.com/openziti/edge-api/compare/v0.26.13...v0.26.14) -* github.com/openziti/sdk-golang: [v0.23.14 -> v0.23.15](https://github.com/openziti/sdk-golang/compare/v0.23.14...v0.23.15) -* github.com/openziti/secretstream: [v0.1.17 -> v0.1.18](https://github.com/openziti/secretstream/compare/v0.1.17...v0.1.18) - * [Issue #24](https://github.com/openziti/secretstream/issues/24) - Potential side channel issue - * [Issue #25](https://github.com/openziti/secretstream/issues/25) - Reads from crypto/rand not checked for errors - -* github.com/openziti/ziti: [v0.34.1 -> v0.34.2](https://github.com/openziti/ziti/compare/v0.34.1...v0.34.2) - * [Issue #1831](https://github.com/openziti/ziti/issues/1831) - Circuit ID should be returned in the response to a Dial request - * [Issue #1873](https://github.com/openziti/ziti/issues/1873) - host.v1 health check time.Duration unconvertible - -# Release 0.34.1 - -## What's New - -* Updates version of go to 1.22.x - * As usual when updating the go version, this is the only change in this release - -# Release 0.34.0 - -## What's New +## About 1.0 -* Bug fixes and performance enhancements -* Version number is bumped as a large chunk of HA was merged up. The next version bump is likely to bring HA to alpha status. +What does marking OpenZiti as 1.0 mean? -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.119 -> v2.0.122](https://github.com/openziti/channel/compare/v2.0.119...v2.0.122) -* github.com/openziti/edge-api: [v0.26.12 -> v0.26.14](https://github.com/openziti/edge-api/compare/v0.26.12...v0.26.14) -* github.com/openziti/foundation/v2: [v2.0.37 -> v2.0.40](https://github.com/openziti/foundation/compare/v2.0.37...v2.0.40) -* github.com/openziti/identity: [v1.0.70 -> v1.0.73](https://github.com/openziti/identity/compare/v1.0.70...v1.0.73) -* github.com/openziti/metrics: [v1.2.45 -> v1.2.48](https://github.com/openziti/metrics/compare/v1.2.45...v1.2.48) -* github.com/openziti/runzmd: [v1.0.38 -> v1.0.41](https://github.com/openziti/runzmd/compare/v1.0.38...v1.0.41) -* github.com/openziti/sdk-golang: [v0.22.28 -> v0.23.14](https://github.com/openziti/sdk-golang/compare/v0.22.28...v0.23.14) - * [Issue #524](https://github.com/openziti/sdk-golang/issues/524) - Add circuit id to edge.Conn, so sdk connections can be correlated with network traffic - * [Issue #515](https://github.com/openziti/sdk-golang/issues/515) - Service hosting improvements - * [Issue #501](https://github.com/openziti/sdk-golang/issues/501) - Improve hosting session management - -* github.com/openziti/secretstream: [v0.1.16 -> v0.1.17](https://github.com/openziti/secretstream/compare/v0.1.16...v0.1.17) -* github.com/openziti/storage: [v0.2.30 -> v0.2.33](https://github.com/openziti/storage/compare/v0.2.30...v0.2.33) -* github.com/openziti/transport/v2: [v2.0.122 -> v2.0.125](https://github.com/openziti/transport/compare/v2.0.122...v2.0.125) -* github.com/openziti/ziti: [v0.33.1 -> v0.34.0](https://github.com/openziti/ziti/compare/v0.33.1...v0.34.0) - * [Issue #1858](https://github.com/openziti/ziti/issues/1858) - add option to create a generic env file instead of a BASH script - * [Issue #1428](https://github.com/openziti/ziti/issues/1428) - Investigate policy integrity performance - * [Issue #1854](https://github.com/openziti/ziti/issues/1854) - Controller can try to send unroute to router which has since disconnected, causing panic - * [Issue #1576](https://github.com/openziti/ziti/issues/1576) - Don't scan for posture checks if there are no posture checks - * [Issue #1849](https://github.com/openziti/ziti/issues/1849) - Session Sync shouldn't be able to block the control channel - * [Issue #1846](https://github.com/openziti/ziti/issues/1846) - Looking up api session certs for api sessions is inefficient +### Backwards Compatibility +We've guaranteed API stability for SDK clients for years and worked hard to ensure that routers +and controllers would be backwards and forward compatible. However, we have had a variety of +management API changes and CLI changes. For post 1.0 releases we expect to make additions to the +APIs and CLI, but won't remove anything until it's been first marked as deprecated and then only +with a major version bump. -# Release 0.33.1 +### Stability and Scale +Recent releases have seen additional testing using chaos testing techniques. These tests involve +setting up relatively large scale environments, knocking out various components and then verifying +that the network is able to return to a stable state. These test are run for hours to try and +eliminate race conditions and distributed state machine problems. -## What's New - -* Backward compatibility router <-> controller fix to address metrics parsing panic +OpenZiti is also being used as underlying infrastrcture for the zrok public service. Use of this +network has grown quickly and proven that it's possible to build ziti native apps that can scale +up. -## Component Updates and Bug Fixes -* github.com/openziti/ziti: [v0.33.0 -> v0.33.1](https://github.com/openziti/ziti/compare/v0.33.0...v0.33.1) - * [Issue #1826](https://github.com/openziti/ziti/issues/1826) - 0.33.+ routers can cause panic in pre-0.33 controller with new metric +## Backward Incompatible Changes to pre-1.0 releases -# Release 0.33.0 - -## What's New - -* SDK Terminator stability improvements -* Minor feature updates and bug fixes - -## SDK Terminator stability improvements - -This release was focused on creating a chaos test for SDK terminators, running it and fixing any issues found. -The test repeatedly and randomly restarts the controller, routers and tunnelers then verifies that terminators -end up in the correct state. - -The following tools were also used/added to aid in diagnosing and fixing issues: - -* `ziti fabric validate router-sdk-terminators` - * Compares the controller state with the router state -* `ziti fabric validate terminators` - * Checks each selected terminator to ensure it's still valid on the router and/or sdk -* `ziti fabric inspect sdk-terminators` - * Allows inspecting each routers terminator state -* `ziti fabric inspect router-messaging` - * Allows inspecting what the controller has queued for router state sync and terminator validations -* `ziti edge validate service-hosting` - * Shows how many terminators each identity which can host a service has - -Several changes were made to the terminator code to ensure that terminators are properly created and cleaned up. -The routers now use an adaptive rate limiter to control how fast they send terminator related requests to the -controller. For this to work properly, the rate limiting on the controller must be enabled, so it can report -back to the routers when it's got too much work. - -## Component Updates and Bug Fixes - -* github.com/openziti/edge-api: [v0.26.10 -> v0.26.12](https://github.com/openziti/edge-api/compare/v0.26.10...v0.26.12) -* github.com/openziti/ziti: [v0.32.2 -> v0.33.0](https://github.com/openziti/ziti/compare/v0.32.2...v0.33.0) - * [Issue #1815](https://github.com/openziti/ziti/issues/1815) - Panic if api session sync failed handler is called twice in the router - * [Issue #1794](https://github.com/openziti/ziti/issues/1794) - Add SDK terminator chaos test and fix any bugs found as part of chaos testing - * [Issue #1781](https://github.com/openziti/ziti/issues/1781) - Improve performance when adding intercepted services - * [Issue #1369](https://github.com/openziti/ziti/issues/1369) - Allow filtering by policy type when listing identities for service or services for identity - * [Issue #1791](https://github.com/openziti/ziti/issues/1791) - route dial isn't checking for network timeouts correctly - * [Issue #1204](https://github.com/openziti/ziti/issues/1204) - ziti cli identity tags related flags misbehaving - * [Issue #987](https://github.com/openziti/ziti/issues/987) - "ziti create config router edge" doesn't know about --tunnelerMode proxy - * [Issue #652](https://github.com/openziti/ziti/issues/652) - Update CLI script M1 Support when github actions allows - -# Release 0.32.2 - -## What's New - -* Terminator performance improvements -* API Rate Limiter enabled by default - -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.117 -> v2.0.119](https://github.com/openziti/channel/compare/v2.0.117...v2.0.119) - * [Issue #127](https://github.com/openziti/channel/issues/127) - Support some additional types in message headers - -* github.com/openziti/foundation/v2: [v2.0.36 -> v2.0.37](https://github.com/openziti/foundation/compare/v2.0.36...v2.0.37) -* github.com/openziti/identity: [v1.0.69 -> v1.0.70](https://github.com/openziti/identity/compare/v1.0.69...v1.0.70) -* github.com/openziti/metrics: [v1.2.43 -> v1.2.45](https://github.com/openziti/metrics/compare/v1.2.43...v1.2.45) -* github.com/openziti/runzmd: [v1.0.37 -> v1.0.38](https://github.com/openziti/runzmd/compare/v1.0.37...v1.0.38) -* github.com/openziti/sdk-golang: [v0.22.21 -> v0.22.28](https://github.com/openziti/sdk-golang/compare/v0.22.21...v0.22.28) - * [Issue #495](https://github.com/openziti/sdk-golang/issues/495) - Refresh edge session if listen attempt fails, to ensure that session is still valid - -* github.com/openziti/storage: [v0.2.28 -> v0.2.30](https://github.com/openziti/storage/compare/v0.2.28...v0.2.30) -* github.com/openziti/transport/v2: [v2.0.121 -> v2.0.122](https://github.com/openziti/transport/compare/v2.0.121...v2.0.122) -* github.com/openziti/ziti: [v0.32.1 -> v0.32.2](https://github.com/openziti/ziti/compare/v0.32.1...v0.32.2) - * [Issue #1741](https://github.com/openziti/ziti/issues/1741) - Prevent stuck links - * [Issue #1736](https://github.com/openziti/ziti/issues/1736) - controller crashes generating create circuit responses - * [Issue #1733](https://github.com/openziti/ziti/issues/1733) - Improve terminator creation performance - * [Issue #1734](https://github.com/openziti/ziti/issues/1734) - Make API rate limiter enabled by default - * [Issue #1726](https://github.com/openziti/ziti/issues/1726) - Fix some sdk hosting logging - * [Issue #1725](https://github.com/openziti/ziti/issues/1725) - Fix panic in entity event processing - * [Issue #652](https://github.com/openziti/ziti/issues/652) - CI support for MacOS arm64 - -# Release 0.32.1 +Administrators no longer have access to dial/bind all services by default. See below for details. ## What's New +* Administrators no longer have access to dial/bind all services by default. +* TLS Handshakes can now be rate limited in the controller +* TLS Handshake timeouts can now be set on the controller when using ALPN * Bugfixes -* New router setting to control startup timeout - -## Router startup timeout - -The router now has a configuration setting to control how long it wait on startup to be able to -connect to a controller, before it gives up and exits. - -``` -ctrl: - endpoints: - - tls:localhost:1280 - startupTimeout: 5m -``` - -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.116 -> v2.0.117](https://github.com/openziti/channel/compare/v2.0.116...v2.0.117) - * [Issue #125](https://github.com/openziti/channel/issues/125) - Ensure reconnecting channel is marked as connected before calling reconnect callback -* github.com/openziti/edge-api: [v0.26.8 -> v0.26.10](https://github.com/openziti/edge-api/compare/v0.26.8...v0.26.10) -* github.com/openziti/sdk-golang: [v0.22.17 -> v0.22.21](https://github.com/openziti/sdk-golang/compare/v0.22.17...v0.22.21) -* github.com/openziti/ziti: [v0.32.0 -> v0.32.1](https://github.com/openziti/ziti/compare/v0.32.0...v0.32.1) - * [Issue #1709](https://github.com/openziti/ziti/issues/1709) - Fix link management race conditions found by chaos testing - * [Issue #1715](https://github.com/openziti/ziti/issues/1715) - Ensure controller raft peers don't end up with duplicate connections - * [Issue #1702](https://github.com/openziti/ziti/issues/1702) - Add link management chaos test - * [Issue #1691](https://github.com/openziti/ziti/issues/1691) multiple er re-enrolls creates multiple enrollments +## DEFAULT Bind/Dial SERVICE PERMISSIONS FOR Admin IDENTITIES HAVE CHANGED -# Release 0.32.0 +Admin identities were able to Dial and Bind all services regardless of the effective service policies +prior to this release. This could lead to a confusing situation where a tunneler that was assuming an Admin +identity would put itself into an infinite connect-loop when a service's host.v1 address overlapped with +any addresses in its intercept configuration. -## What's New - -* Auth Rate Limiter -* Link Management Fixes -* ziti edge quickstart command deprecates redundant --already-initialized flag. The identical behavior is implied by --home. - -## Backwards compatibility +Please create service policies to grant Bind or Dial permissions to Admin identities as needed. -This release includes new response types from the REST authentication APIS. They are now able to return -`429` (server too busy) responses to auth requests. As this is an API change, the version number is -being bumped to 0.32. +## TLS Handshake -If controller and router are both v0.32 or later, only the router which dialed a link will report it to the controller. -If the controller is older, newer routers will report links from both the dialing and listening side of the link. +A TLS handhshake rate limiter can be enabled. This is useful in cases where there's a flood of TLS requests and the +controller can't handle them all. It can get into a state where it can't respond to TLS handshakes quickly enough, +so the clients time out. They then retry, adding to the the load. The controller ends up wasting time doing work +that isn't use. -## Auth Rate Limiter +This uses the same rate limiting as the auth rate limiter. -In order to prevent clients from overwhelming the server with auth requests, an auth rate limiter has been introduced. -The rate limiter is adaptive, in that it will react to auth attempts timing out by shrinking the number of allowed -queued auth attempts. The number will slowly recover over time. +Additionally the server side handshake timeout can now be configured. -Example configuration: +Configuration: ``` -edge: - # This section allows configurating the rate limiter for auth attempts - authRateLimiter: - # if disabled, no auth rate limiting with be enforced +tls: + handshakeTimeout: 15s + + rateLimiter: + # if disabled, no tls handshake rate limiting with be enforced enabled: true - # the smallest window size for auth attempts + # the smallest window size for tls handshakes minSize: 5 - # the largest allowed window size for auth attempts - maxSize: 250 + # the largest allowed window size for tls handshakes + maxSize: 5000 + # after how long to consider a handshake abandoned if neither success nor failure was reported + timeout: 30s ``` New metrics: -* `auth.limiter.queued_count` - current number of queued auth attempts -* `auth.limiter.window_size` - current size at which new auth attempts will be rejected -* `auth.limiter.work_timer` - tracks the rate at which api sessions are being created and how long it's taking to create them - -## Link Management Fixes - -With long lived link ids, there was potential for link control message to be ambiguous, as the link id wasn't enough to identify -a specific iteration of that link. An iteration field has been added to links so that messaging is unambiguous. -Links will also only be reported from the dialing router now to reduce ambiguouity and race condition in link control channel -messaging. - -## Router SSL Handshake Timeout Config - -There is a new router config setting which allows setting the SSL handshake timeout for TLS connections, when using ALPN for listeners. - -``` -tls: - handshakeTimeout: 15s -``` - -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.111 -> v2.0.116](https://github.com/openziti/channel/compare/v2.0.111...v2.0.116) - * [Issue #123](https://github.com/openziti/channel/issues/123) - Ensure hello messages respect connect timeout - * [Issue #120](https://github.com/openziti/channel/issues/120) - Allow handling new underlay instances with function instead of channel - -* github.com/openziti/edge-api: [v0.26.6 -> v0.26.8](https://github.com/openziti/edge-api/compare/v0.26.6...v0.26.8) -* github.com/openziti/foundation/v2: [v2.0.35 -> v2.0.36](https://github.com/openziti/foundation/compare/v2.0.35...v2.0.36) - * [Issue #391](https://github.com/openziti/foundation/issues/391) - goroutine pool can stall if configured for 0 min workers and with single producer - -* github.com/openziti/identity: [v1.0.68 -> v1.0.69](https://github.com/openziti/identity/compare/v1.0.68...v1.0.69) -* github.com/openziti/metrics: [v1.2.41 -> v1.2.43](https://github.com/openziti/metrics/compare/v1.2.41...v1.2.43) -* github.com/openziti/runzmd: [v1.0.36 -> v1.0.37](https://github.com/openziti/runzmd/compare/v1.0.36...v1.0.37) -* github.com/openziti/sdk-golang: [v0.22.0 -> v0.22.17](https://github.com/openziti/sdk-golang/compare/v0.22.0...v0.22.17) - * [Issue #482](https://github.com/openziti/sdk-golang/issues/482) - Deprecate ListenOptions.MaxConnections in favor of MaxTerminators - -* github.com/openziti/secretstream: [v0.1.14 -> v0.1.16](https://github.com/openziti/secretstream/compare/v0.1.14...v0.1.16) -* github.com/openziti/storage: [v0.2.27 -> v0.2.28](https://github.com/openziti/storage/compare/v0.2.27...v0.2.28) -* github.com/openziti/transport/v2: [v2.0.119 -> v2.0.121](https://github.com/openziti/transport/compare/v2.0.119...v2.0.121) - * [Issue #73](https://github.com/openziti/transport/issues/73) - Allow overriding shared TLS/ALPN listener SSL handshake timeout - -* github.com/openziti/ziti: [v0.31.4 -> v0.32.0](https://github.com/openziti/ziti/compare/v0.31.4...v0.32.0) - * [Issue #1692](https://github.com/openziti/ziti/issues/1692) - Improve link stability with long lived link ids - * [Issue #1693](https://github.com/openziti/ziti/issues/1693) - Make links owned by the dialing router - * [Issue #1685](https://github.com/openziti/ziti/issues/1685) - Race condition where we try to create terminator after client connection is closed - * [Issue #1678](https://github.com/openziti/ziti/issues/1678) - Add link validation utility - * [Issue #1673](https://github.com/openziti/ziti/issues/1673) - xgress dialers not getting passed xgress config - * [Issue #1669](https://github.com/openziti/ziti/issues/1669) - Make sure link accepts are not single threaded - * [Issue #1657](https://github.com/openziti/ziti/issues/1657) - Add api session rate limiter - -# Release 0.31.4 - -## What's New - -* Bug fix for a data flow stall which is especially likely to happen on circuits with single router paths - -## Thanks - -* @marvkis - for providing high quality debug data which made tracking down a couple of flow control stall issues much easier - -## Component Updates and Bug Fixes - -* github.com/openziti/metrics: [v1.2.40 -> v1.2.41](https://github.com/openziti/metrics/compare/v1.2.40...v1.2.41) -* github.com/openziti/sdk-golang: [v0.21.2 -> v0.22.0](https://github.com/openziti/sdk-golang/compare/v0.21.2...v0.22.0) - * [Issue #468](https://github.com/openziti/sdk-golang/issues/468) - SDK does an unnecessary number of session refreshes - -* github.com/openziti/storage: [v0.2.26 -> v0.2.27](https://github.com/openziti/storage/compare/v0.2.26...v0.2.27) -* github.com/openziti/ziti: [v0.31.3 -> v0.31.4](https://github.com/openziti/ziti/compare/v0.31.3...v0.31.4) - * [Issue #1645](https://github.com/openziti/ziti/issues/1645) - Once routers share a link id, we can't use the link id to decide which duplicate link to discard - * [Issue #1642](https://github.com/openziti/ziti/issues/1642) - Revert posture check optimization - * [Issue #1586](https://github.com/openziti/ziti/issues/1586) - If ack is received before payload is processed by link send buffer, a stall can result - - -# Release 0.31.3 - -## What's New - -* Services Max Idle Time -* Add/Remove Peer and Transfer Leadership via REST - -## Service Max Idle Time - -A max idle time can now be configured on services. The default value of 0 indicates that no maximum will -be enforced. A circuit is considered idle when no traffic is flowing across through the initiating or -terminating router. - -``` -ziti edge create service test-service --max-idle-time 5m -``` - -Note that the idle time calculation is done on the router, so if max idle time on a service is less -than the configured scan interval on the router, it make take longer than expected for idle circuits -to be removed. - -## Raft Cluster Management via REST - -The controller now allows some Raft cluster management operations to be performed via REST. - -NOTE: If your cluster is not bootstrapped yet, the REST API won't be available. These will only work on a bootstrapped cluster! - -The following operations are now supported: - -* Add member -* Remove member -* Transfer leadership - -``` -ziti fabric raft add-member tls:localhost:6363 -ziti fabric raft add-member tls:localhost:6464 -ziti fabric raft transfer-leadership -ziti fabric raft transfer-leadership ctrl3 -ziti fabric raft remove-member ctrl2 -ziti fabric raft remove-member ctrl3 -``` - -## Component Updates and Bug Fixes - -* github.com/openziti/edge-api: [v0.26.1 -> v0.26.6](https://github.com/openziti/edge-api/compare/v0.26.1...v0.26.6) -* github.com/openziti/sdk-golang: [v0.20.139 -> v0.21.2](https://github.com/openziti/sdk-golang/compare/v0.20.139...v0.21.2) - * [Issue #465](https://github.com/openziti/sdk-golang/issues/465) - Allow listen options to specify how many listeners need to be established before returning - * [Issue #462](https://github.com/openziti/sdk-golang/issues/462) - Allow refreshing a single service - -* github.com/openziti/ziti: [v0.31.2 -> v0.31.3](https://github.com/openziti/ziti/compare/v0.31.2...v0.31.3) - * [Issue #1583](https://github.com/openziti/ziti/issues/1583) - xgress: Potential data stall due when processing acks after checking window size - * [Issue #1578](https://github.com/openziti/ziti/issues/1578) - Send BindSuccess notifications to SDK if supported - * [Issue #1544](https://github.com/openziti/ziti/issues/1544) - Support transfer raft leadership via REST - * [Issue #1543](https://github.com/openziti/ziti/issues/1543) - Support add/remove raft peer via REST - * [Issue #1496](https://github.com/openziti/ziti/issues/1496) - Configurable Timer needed to close idle circuits - * [Issue #1402](https://github.com/openziti/ziti/issues/1402) - Allow router to decomission itself - -# Release 0.31.2 - -## What's New - -* Go version updated from 1.20 to 1.21 - -# Release 0.31.1 - -## What's New - -* SDK Hosting Improvements -* Terminator validation utility -* Circuit/Link query support - -## SDK Hosting Improvements - -In previous versions of OpenZiti, if many SDK clients were attempting to establish hosting, the controller could get overwhelmed. -In this release, routers will use the rate limiter pool introduced in 0.27.6 when creating terminators on behalf of sdk clients -hosting applications. Additionally, routers now have the ability to verify terminator state with the sdk, if the sdk supports it. -In general, hosting large numbers of services using the sdk should now be less suceptible to thundering herd issues. - -## Manual Terminator Validation - -There is a new CLI command available to validate terminator state. This is primarily a developer tool to validate that terminator -setup logic is correct. However it may also be used to diagnose and resolve issues with production systems, should the need arise. - -``` -ziti fabric validate terminators -``` - -## Circuit/Link Query Support - -Previously listing circuit and links always showed the full list. This is because these types are in memory only and are not stored -in the bbolt datastore. There's now basic support for querying in-memory types and circuits and links can now be filtered/paged/sorted - the same as other entity types. - -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.105 -> v2.0.111](https://github.com/openziti/channel/compare/v2.0.105...v2.0.111) - * [Issue #118](https://github.com/openziti/channel/issues/118) - Allowing checking if reconnecting impl is currently connected - -* github.com/openziti/edge-api: [v0.26.0 -> v0.26.1](https://github.com/openziti/edge-api/compare/v0.26.0...v0.26.1) -* github.com/openziti/foundation/v2: [v2.0.33 -> v2.0.35](https://github.com/openziti/foundation/compare/v2.0.33...v2.0.35) -* github.com/openziti/identity: [v1.0.66 -> v1.0.68](https://github.com/openziti/identity/compare/v1.0.66...v1.0.68) -* github.com/openziti/metrics: [v1.2.37 -> v1.2.40](https://github.com/openziti/metrics/compare/v1.2.37...v1.2.40) -* github.com/openziti/runzmd: [v1.0.33 -> v1.0.36](https://github.com/openziti/runzmd/compare/v1.0.33...v1.0.36) -* github.com/openziti/sdk-golang: [v0.20.129 -> v0.20.139](https://github.com/openziti/sdk-golang/compare/v0.20.129...v0.20.139) - * [Issue #457](https://github.com/openziti/sdk-golang/issues/457) - Add inspect support - * [Issue #450](https://github.com/openziti/sdk-golang/issues/450) - Support idempotent terminator creation - -* github.com/openziti/secretstream: [v0.1.13 -> v0.1.14](https://github.com/openziti/secretstream/compare/v0.1.13...v0.1.14) -* github.com/openziti/storage: [v0.2.23 -> v0.2.26](https://github.com/openziti/storage/compare/v0.2.23...v0.2.26) - * [Issue #57](https://github.com/openziti/storage/issues/57) - Support querying collections of in memory objects - -* github.com/openziti/transport/v2: [v2.0.113 -> v2.0.119](https://github.com/openziti/transport/compare/v2.0.113...v2.0.119) -* github.com/openziti/ziti: [v0.31.0 -> v0.31.1](https://github.com/openziti/ziti/compare/v0.31.0...v0.31.1) - * [Issue #1555](https://github.com/openziti/ziti/issues/1555) - Consolidate fabric/edge persistence code - * [Issue #1547](https://github.com/openziti/ziti/issues/1547) - Support filtering, sorting and paging circuits and links - * [Issue #1446](https://github.com/openziti/ziti/issues/1446) - Allow for idempotent sdk based terminators - * [Issue #1540](https://github.com/openziti/ziti/issues/1540) - Transit router create fails in HA environment - * [Issue #1523](https://github.com/openziti/ziti/issues/1523) - Bootstrap members not working - * [Issue #1525](https://github.com/openziti/ziti/issues/1525) - Improve cluster list output - * [Issue #1519](https://github.com/openziti/ziti/issues/1519) - Simplify link ack handling - * [Issue #1513](https://github.com/openziti/ziti/issues/1513) - DNS service failure should not cause a router restart - * [Issue #1494](https://github.com/openziti/ziti/issues/1494) - Panic if applying raft log returns nil result - - -# Release 0.31.0 - -## What's New - -* Rate limited for model changes - -## Rate Limiter for Model Changes - -To prevent the controller from being overwhelmed by a flood of changes, a rate limiter -can be enabled in the configuration file. A maximum number of queued changes can also -be configured. The rate limited is disabled by default for now. If not specified the -default number of queued changes is 100. - -When the rate limit is hit, an error will be returned. If the request came in from -the REST API, the response will use HTTP status code 429 (too many requests). - -The OpenAPI specs have been updated, so if you're using a generated client to make -REST calls, it's recommended that you regenerate your client. - - -``` -commandRateLimiter: - enabled: true - maxQueued: 100 -``` - -If the rate limiter is enabled, the following metrics will be produced: - -* `command.limiter.queued_count` - gauge of the current number of queued operations -* `command.limiter.work_timer` - timer for operations. Includes the following: - * A histogram of how long operations take to complete - * A meter showing that rate at which operations are executed - * A count of how many operations have been executed - -## Component Updates and Bug Fixes - -* github.com/openziti/agent: [v1.0.15 -> v1.0.16](https://github.com/openziti/agent/compare/v1.0.15...v1.0.16) -* github.com/openziti/channel/v2: [v2.0.101 -> v2.0.105](https://github.com/openziti/channel/compare/v2.0.101...v2.0.105) -* github.com/openziti/edge-api: [v0.25.38 -> v0.26.0](https://github.com/openziti/edge-api/compare/v0.25.38...v0.26.0) - * [Issue #49](https://github.com/openziti/edge-api/issues/49) - Add 429 responses to allow indicating that the server is too busy - -* github.com/openziti/identity: [v1.0.64 -> v1.0.66](https://github.com/openziti/identity/compare/v1.0.64...v1.0.66) -* github.com/openziti/metrics: [v1.2.36 -> v1.2.37](https://github.com/openziti/metrics/compare/v1.2.36...v1.2.37) -* github.com/openziti/sdk-golang: [v0.20.122 -> v0.20.129](https://github.com/openziti/sdk-golang/compare/v0.20.122...v0.20.129) - * [Issue #443](https://github.com/openziti/sdk-golang/issues/443) - Don't send close in response to a close on a listener - -* github.com/openziti/secretstream: [v0.1.12 -> v0.1.13](https://github.com/openziti/secretstream/compare/v0.1.12...v0.1.13) -* github.com/openziti/storage: [v0.2.20 -> v0.2.23](https://github.com/openziti/storage/compare/v0.2.20...v0.2.23) -* github.com/openziti/transport/v2: [v2.0.109 -> v2.0.113](https://github.com/openziti/transport/compare/v2.0.109...v2.0.113) -* github.com/openziti/ziti: [v0.30.5 -> v0.31.0](https://github.com/openziti/ziti/compare/v0.30.5...v0.31.0) - * [Issue #1471](https://github.com/openziti/ziti/issues/1471) - Router links not resilient to controller crash - * [Issue #1468](https://github.com/openziti/ziti/issues/1468) - Quickstart quietly fails if password is < 5 characters long - * [Issue #1445](https://github.com/openziti/ziti/issues/1445) - Add controller update guardrail - * [Issue #1442](https://github.com/openziti/ziti/issues/1442) - Network watchdog not shutting down when controller shuts down - * [Issue #1465](https://github.com/openziti/ziti/issues/1465) - Upgrade functions `getZiti` and `performMigration` were only functional on Mac OS, now they are functional for Linux and Mac OSs. - * [Issue #1217](https://github.com/openziti/ziti/issues/1217) - Quickstart was improperly handling special characters in `ZITI_PWD`. Special characters are now supported for `ZITI_PWD` in quickstart functions. - - -# Release 0.30.5 - -## What's New - -* Initial proxy support in host.v1/host.v2 - -## Proxy Support in host.v1/host.v2 - -`host.v1` and `host.v2` configurations may now specify a proxy to use. -Currently only HTTP Connect proxies which don't require authentication are supported. - -**Example using `host.v1`** - -``` -{ - "address": "192.168.2.50", - "port": 1234, - "protocol": "tcp", - "proxy": { - "address": "192.168.1.110:3128", - "type": "http" - } -} -``` - - -## Component Updates and Bug Fixes -* github.com/openziti/channel/v2: [v2.0.99 -> v2.0.101](https://github.com/openziti/channel/compare/v2.0.99...v2.0.101) -* github.com/openziti/edge-api: [v0.25.37 -> v0.25.38](https://github.com/openziti/edge-api/compare/v0.25.37...v0.25.38) -* github.com/openziti/foundation/v2: [v2.0.32 -> v2.0.33](https://github.com/openziti/foundation/compare/v2.0.32...v2.0.33) -* github.com/openziti/identity: [v1.0.63 -> v1.0.64](https://github.com/openziti/identity/compare/v1.0.63...v1.0.64) -* github.com/openziti/metrics: [v1.2.35 -> v1.2.36](https://github.com/openziti/metrics/compare/v1.2.35...v1.2.36) -* github.com/openziti/runzmd: [v1.0.32 -> v1.0.33](https://github.com/openziti/runzmd/compare/v1.0.32...v1.0.33) -* github.com/openziti/sdk-golang: [v0.20.116 -> v0.20.122](https://github.com/openziti/sdk-golang/compare/v0.20.116...v0.20.122) - * [Issue #436](https://github.com/openziti/sdk-golang/issues/436) - HTTP calls should respect environment proxy settings - -* github.com/openziti/storage: [v0.2.18 -> v0.2.20](https://github.com/openziti/storage/compare/v0.2.18...v0.2.20) - * [Issue #52](https://github.com/openziti/storage/issues/52) - Grammar should expect single valid query followed by EOF - -* github.com/openziti/transport/v2: [v2.0.107 -> v2.0.109](https://github.com/openziti/transport/compare/v2.0.107...v2.0.109) -* github.com/openziti/ziti: [v0.30.4 -> v0.30.5](https://github.com/openziti/ziti/compare/v0.30.4...v0.30.5) - * [Issue #1336](https://github.com/openziti/ziti/issues/1336) - `ziti edge quickstart` did - not create the usual edge router/service edge router policy. - * [Issue #1397](https://github.com/openziti/ziti/issues/1397) - HTTP Proxy support for host.v1/host.v2 config types - * [Issue #1423](https://github.com/openziti/ziti/issues/1423) - Controller crashes when edge router reconnects (Client Hello) - * [Issue #1414](https://github.com/openziti/ziti/issues/1414) - Race condition in xgress_edge_tunnel tunneller at start but not seen in pre-compiled binary - * [Issue #1406](https://github.com/openziti/ziti/issues/1406) - Entity change event dispatcher isn't shutting down properly when controller shuts down - * [Issue #1382](https://github.com/openziti/ziti/issues/1382) - service failure costs are not shrinking over time - -# Release 0.30.4 - -## What's New - -* `ziti edge quickstart`](https://github.com/openziti/ziti/issues/1298). You can now - download the `ziti` CLI and have a functioning network with just one command. The - network it creates is ephemeral and is intended to be torn down when the process exits. - It is intended for quick evaluation and testing of an overlay network. It supports the - following flags: - - ``` - --already-initialized Specifies the PKI does not need to be created and the db does not need to be initialized. Recommended to be combined with --home. If --home is not specified the environment will be destroyed on shutdown! default: false - --ctrl-address string Sets the advertised address for the control plane and API - --ctrl-port int16 Sets the port to use for the control plane and API - -h, --help help for quickstart - --home string Sets the directory the environment should be installed into. Defaults to a temporary directory. If specified, the environment will not be removed on exit. - -p, --password string Password to use for authenticating to the Ziti Edge Controller. default: admin - --router-address string Sets the advertised address for the integrated router - --router-port int16 Sets the port to use for the integrated router - -u, --username string Username to use when creating the Ziti Edge Controller. default: admin - ``` - - Example Usage: - ``` - ziti edge quickstart \ - --ctrl-address potato \ - --ctrl-port 12345 \ - --router-address avocado \ - --router-port 23456 \ - --home $HOME/.ziti/pet-ziti \ - --already-initialized \ - --username someOtherUsername \ - --password someOtherPassword - ``` - -* Bugfixes - -## Component Updates and Bug Fixes - -* github.com/openziti/ziti: [v0.30.3 -> v0.30.4](https://github.com/openziti/ziti/compare/v0.30.3...v0.30.4) - * Fixed an issue causing router configs to be rewritten when docker compose was brought up with existing configs - -# Release 0.30.3 - -## What's New - -* Bugfixes - -## Component Updates and Bug Fixes - -* github.com/openziti/edge: [v0.24.401 -> v0.24.404](https://github.com/openziti/edge/compare/v0.24.401...v0.24.404) -* github.com/openziti/fabric: [v0.24.20 -> v0.24.23](https://github.com/openziti/fabric/compare/v0.24.20...v0.24.23) - * [Issue #786](https://github.com/openziti/fabric/issues/786) - entityChangeEventDispatcher.flushLoop doesn't shutdown when controller shuts down - * [Issue #785](https://github.com/openziti/fabric/issues/785) - Allow link groups to be single string value - * [Issue #783](https://github.com/openziti/fabric/issues/783) - Raft cluster connections not updated for ALPN - -* github.com/openziti/ziti: [v0.30.2 -> v0.30.3](https://github.com/openziti/ziti/compare/v0.30.2...v0.30.3) - -# Release 0.30.2 - -## What's New - -* Identity type consolidation -* HTTP Connect Proxy support for control channel and links - -## Identity Type Consolidation - -Prior to this release there were four identity types: - -* User -* Service -* Device -* Router - -Of these four types, only Router has any functional purpose. Given that, the other three have been merged into -a single `Default` identity type. Since Router identities can only be created by the system, it's no longer -necessary to specify the identity type when creating identities. - -The identity type may still be provided, but a deprecation warning will be emitted. - -**Backwards Compatibility** - -Existing non-Router identities will be migrated to the `Default` identity type. If an identity type other -than `Default` is provided when creating an identity, it will be coerced to the `Default` type. Existing -code may have issues with the new identity type being returned. - - -## HTTP Connect Proxy support - -Routers may now specify a proxy configuration which will be used when establishing connections to controllers -and data links to other routers. At this point only HTTP Connect Proxies with no authentication required are -supported. - -Example router config: - -```yaml -proxy: - type: http - address: localhost:3128 - -``` - -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.91 -> v2.0.95](https://github.com/openziti/channel/compare/v2.0.91...v2.0.95) -* github.com/openziti/edge: [v0.24.381 -> v0.24.401](https://github.com/openziti/edge/compare/v0.24.381...v0.24.401) - * [Issue #1597](https://github.com/openziti/edge/issues/1597) - ER/T cached API sessions aren't remembered when pulled from cache - * [Issue #1428](https://github.com/openziti/edge/issues/1428) - Make identity type optional. Consolidate User/Service/Device to 'Default'. - * [Issue #1584](https://github.com/openziti/edge/issues/1584) - AuthPolicyDetail is incompatible with API response - -* github.com/openziti/edge-api: [v0.25.31 -> v0.25.33](https://github.com/openziti/edge-api/compare/v0.25.31...v0.25.33) -* github.com/openziti/fabric: [v0.24.2 -> v0.24.20](https://github.com/openziti/fabric/compare/v0.24.2...v0.24.20) - * [Issue #775](https://github.com/openziti/fabric/issues/775) - Add support for proxies for control channel and links - -* github.com/openziti/foundation/v2: [v2.0.29 -> v2.0.30](https://github.com/openziti/foundation/compare/v2.0.29...v2.0.30) -* github.com/openziti/identity: [v1.0.60 -> v1.0.61](https://github.com/openziti/identity/compare/v1.0.60...v1.0.61) -* github.com/openziti/runzmd: [v1.0.29 -> v1.0.30](https://github.com/openziti/runzmd/compare/v1.0.29...v1.0.30) -* github.com/openziti/sdk-golang: [v0.20.90 -> v0.20.101](https://github.com/openziti/sdk-golang/compare/v0.20.90...v0.20.101) -* github.com/openziti/storage: [v0.2.12 -> v0.2.14](https://github.com/openziti/storage/compare/v0.2.12...v0.2.14) -* github.com/openziti/transport/v2: [v2.0.99 -> v2.0.103](https://github.com/openziti/transport/compare/v2.0.99...v2.0.103) - * [Issue #54](https://github.com/openziti/transport/issues/54) - Support HTTP Connect proxying for TLS connections - -* github.com/openziti/metrics: [v1.2.31 -> v1.2.33](https://github.com/openziti/metrics/compare/v1.2.31...v1.2.33) -* github.com/openziti/secretstream: [v0.1.10 -> v0.1.11](https://github.com/openziti/secretstream/compare/v0.1.10...v0.1.11) -* github.com/openziti/ziti: [v0.30.1 -> v0.30.2](https://github.com/openziti/ziti/compare/v0.30.1...v0.30.2) - * [Issue #1266](https://github.com/openziti/ziti/issues/1266) - Outdated README.md: Some links return "Page Not Found" - - -# Release 0.30.1 - -## What's New - -## Component Updates and Bug Fixes - -* github.com/openziti/ziti: [v0.30.0 -> v0.30.1](https://github.com/openziti/ziti/compare/v0.30.0...v0.30.1) - * [Issue #1225](https://github.com/openziti/ziti/issues/1225) - Updated ZITI_ROUTER_ADVERTISED_HOST to use the more common naming convention of ZITI_ROUTER_ADVERTISED_ADDRESS - * [Issue #1233](https://github.com/openziti/ziti/issues/1233) - Added `lsof` to the list of prerequisites to be checked during quickstart - -# Release 0.30.0 - -## What's New - -* Link management is now delegated to routers -* Controller and routers can operate with a single listening port - -## Link Management Updates - -Previously, the controller would do its best to determine where links needed to be established. -It would send messages to the routers, telling them which addresses to dial on other routes. -The routers would in turn let the controller know if link establishment was successful or -if the router already had a link to the given endpoint. - -With this release, the controller will only let routers know which routers exist, whether they -are currently connected to the controller, and what link listeners they are advertising. The -routers will now decide which links to make and let the controllers know as links are created -and broken. - -### Link Groups - -Both dialers and listeners can now specify a set of groups. If no groups are specified, the -dialer or listener will be placed in the `default` group. Dialers will only attempt to dial -listeners who have at least one group in common with them. - - -### Failed Links - -Previously when a link failed, the controller would show it in the link list as failed for a time -before removing it. Now failed links are removed immediately. There are existing link events for -link creation and link failure which can be used for forensics. - -### Duplicate Links - -There is a new link status `Duplicate` used when a router receives a link request and determines -that it's a duplicate of an existing link. This happens when two routers both have listeners -and dialers. They will often dial each other at the same time, resulting in a duplicate link. - -### Compatibility - -If you use a 0.30+ controller with older routers, the controller will still do link calculation -and send dial messages, as long as the `enableLegacyLinkMgmt` setting is set to true. - -If you use a pre 0.30.0 controller with newer routers, the new routers will still accept the -dial messages. - -### New Configuration - -#### Controller - -The controller has three new options: - -``` -network: - routerMessaging: - queueSize: 100 - maxWorkers: 100 - enableLegacyLinkMgmt: true -``` - -When a router connects or disconnects from the controller, we send two sets of updates. - -1. If a router has connected we send it the the state of the other routers -1. We send all the other routers the updated state of the connecting/disconnecting router - -These messages are sent using a worker pool. The size of the queue feeding the worker pool is controlled with -`routerMessaging.queueSize`. The max size of the worker pool is controlled used the `routerMessaging.maxWorkers` -option. - -* queueSize - * Min value: 0 - * Max value: 1,000,000 - * Default: 100 -* maxWorkers - * Min value: 1 - * Max value: 10,000 - * Default: 100 - -If you have routers older than 0.30.0, the controller will calculate which links to dial. This can be disabled -by setting `enableLegacyLinkMgmt` to false. This setting currently defaults to true, but will default to false -in a future release. In a subsequent release this functionality will be removed all together. - -#### Router - -The router has new configuration options for link dialing. - -``` -link: - dialers: - - binding: transport - groups: - - public - - vpc1234 - healthyDialBackoff: - retryBackoffFactor: 1.5 - minRetryInterval: 5s - maxRetryInterval: 5m - unhealthyDialBackoff: - retryBackoffFactor: 10 - minRetryInterval: 1m - maxRetryInterval: 1h - listeners: - - binding: transport - groups: vpc1234 -``` - -**Groups** - -See above for a description of link groups work. - -Default value: `default` - -**Dial Back-off** - -Dialers can be configured with custom back-off behavior. Each dialer has a back-off policy for dialing -healthy routers (those that are connected to a controller) and a separate policy for unhealthy routers. - -The back-off policies have the following attributes: - -* minRetryInterval - duration specifying the minimum time between dial attempts - * Min value: 10ms - * Max value: 24h - * Default: 5s for healthy, 1m for unhealthy - * Format: Golang Durations, see: https://pkg.go.dev/maze.io/x/duration#ParseDuration -* maxRetryInterval - duration specifying the maximum time between dial attempts - * Min value: 10ms - * Max value: 24h - * Default: 5m for healthy, 1h for unhealthy - * Format: Golang duration, see: https://pkg.go.dev/maze.io/x/duration#ParseDuration -* retryBackoffFactor - factor by which to increase the retry interval between failed dial attempts - * Min value: 1 - * Max value: 100 - * Default: 1.5 for healthy, 100 for unhealthy - - -## Single Port Support / ALPN -Ziti Controller and Routers can operate with a single open port. In order to implement this feature we use -ALPN ([Application Layer Protocol Negotiation](https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation)) -TLS extension. It allows TLS client to request and TLS server to select appropriate application protocol handler during -TLS handshake. - -### Protocol Details - -The following protocol identifiers are defined: - -| id | purpose | -| --------- | ------- | -| ziti-ctrl | Control plane connections | -| ziti-link | Fabric link connections | -| ziti-edge | Client SDK connection to Edge Routers | - -Standard HTTP protocol identifiers (`h2`, `http/1.1`) are used for Controller REST API and Websocket listeners. - -### Backward Compatibility - -This feature is designed to be backward compatible with SDK clients: older client will still be able to connect without -requesting `ziti-edge` protocol. - -**Breaking** - -Older routers won't be able to establish control channel or fabric links with updated network. -However, newer Edge Routers should be able to join older network in some circumstances -- only outbound links from new Routers would work. - -## Component Updates and Bug Fixes -* github.com/openziti/agent: [v1.0.14 -> v1.0.15](https://github.com/openziti/agent/compare/v1.0.14...v1.0.15) -* github.com/openziti/channel/v2: [v2.0.84 -> v2.0.91](https://github.com/openziti/channel/compare/v2.0.84...v2.0.91) - * [Issue #108](https://github.com/openziti/channel/issues/108) - Reconnecting underlay not returning headers from hello message - -* github.com/openziti/edge: [v0.24.364 -> v0.24.381](https://github.com/openziti/edge/compare/v0.24.364...v0.24.381) - * [Issue #1548](https://github.com/openziti/edge/issues/1548) - Panic in edge@v0.24.326/controller/sync_strats/sync_instant.go:194 - -* github.com/openziti/edge-api: [v0.25.30 -> v0.25.31](https://github.com/openziti/edge-api/compare/v0.25.30...v0.25.31) -* github.com/openziti/fabric: [v0.23.45 -> v0.24.2](https://github.com/openziti/fabric/compare/v0.23.45...v0.24.2) - * [Issue #766](https://github.com/openziti/fabric/issues/766) - Lookup of terminators with same instance id isn't filtering by instance id - * [Issue #692](https://github.com/openziti/fabric/issues/692) - Add ability to control link formation between devices more granularly - * [Issue #749](https://github.com/openziti/fabric/issues/749) - Move link control to router - * [Issue #343](https://github.com/openziti/fabric/issues/343) - Link state Failed on startup - -* github.com/openziti/foundation/v2: [v2.0.28 -> v2.0.29](https://github.com/openziti/foundation/compare/v2.0.28...v2.0.29) -* github.com/openziti/identity: [v1.0.59 -> v1.0.60](https://github.com/openziti/identity/compare/v1.0.59...v1.0.60) -* github.com/openziti/runzmd: [v1.0.28 -> v1.0.29](https://github.com/openziti/runzmd/compare/v1.0.28...v1.0.29) -* github.com/openziti/sdk-golang: [v0.20.78 -> v0.20.90](https://github.com/openziti/sdk-golang/compare/v0.20.78...v0.20.90) -* github.com/openziti/storage: [v0.2.11 -> v0.2.12](https://github.com/openziti/storage/compare/v0.2.11...v0.2.12) -* github.com/openziti/transport/v2: [v2.0.93 -> v2.0.99](https://github.com/openziti/transport/compare/v2.0.93...v2.0.99) -* github.com/openziti/xweb/v2: [v2.0.2 -> v2.1.0](https://github.com/openziti/xweb/compare/v2.0.2...v2.1.0) -* github.com/openziti/ziti-db-explorer: [v1.1.1 -> v1.1.3](https://github.com/openziti/ziti-db-explorer/compare/v1.1.1...v1.1.3) - * [Issue #4](https://github.com/openziti/ziti-db-explorer/issues/4) - db explore timeout error is uninformative - -* github.com/openziti/metrics: [v1.2.30 -> v1.2.31](https://github.com/openziti/metrics/compare/v1.2.30...v1.2.31) -* github.com/openziti/ziti: [v0.29.0 -> v0.30.0](https://github.com/openziti/ziti/compare/v0.29.0...v0.30.0) - * [Issue #1199](https://github.com/openziti/ziti/issues/1199) - ziti edge list enrollments - CLI gets 404 - * [Issue #1135](https://github.com/openziti/ziti/issues/1135) - Edge Router: Support multiple protocols on the same listener port - * [Issue #65](https://github.com/openziti/ziti/issues/65) - Add ECDSA support to PKI subcmd - * [Issue #1212](https://github.com/openziti/ziti/issues/1212) - getZiti fails on Mac OS - * [Issue #1220](https://github.com/openziti/ziti/issues/1220) - Fixed getZiti function not respecting user input for custom path - * [Issue #1219](https://github.com/openziti/ziti/issues/1219) - Added check for IPs provided as a DNS SANs entry, IPs will be ignored and not added as a DNS entry in the expressInstall PKI or router config generation. - -# Release 0.29.0 - -## What's New - -### Deprecated Binary Removal -This release removes the following deprecated binaries from the release archives. - -* `ziti-controller` - replaced by `ziti controller` -* `ziti-router` - replaced by `ziti router` -* `ziti-tunnel` - replaced by `ziti tunnel` - -The release archives now only contain the `ziti` executable. This executable is now at the root of the archive instead of nested under a `ziti` directory. - -### Ziti CLI Demo Consolidation - -The ziti CLI functions under `ziti learn`, namely `ziti learn demo` and `ziti learn tutorial` have been consolidated under `ziti demo`. - -### Continued Quickstart Changes - -The quickstart continues to evolve. A breaking change has occurred as numerous environment variables used to customize the quickstart -have changed again. A summary of changes is below - -* All `ZITI_EDGE_ROUTER_` variables have been changed to just `ZITI_ROUTER_`. - * `ZITI_EDGE_ROUTER_NAME` -> `ZITI_ROUTER_NAME` - * `ZITI_EDGE_ROUTER_PORT` -> `ZITI_ROUTER_PORT` - * `ZITI_EDGE_ROUTER_ADVERTISED_HOST` -> `ZITI_ROUTER_ADVERTISED_HOST` - * `ZITI_EDGE_ROUTER_IP_OVERRIDE` -> `ZITI_ROUTER_IP_OVERRIDE` - * `ZITI_EDGE_ROUTER_ENROLLMENT_DURATION` -> `ZITI_ROUTER_ENROLLMENT_DURATION` - * `ZITI_EDGE_ROUTER_ADVERTISED_HOST` -> `ZITI_ROUTER_ADVERTISED_HOST` - * `ZITI_EDGE_ROUTER_LISTENER_BIND_PORT` -> `ZITI_ROUTER_LISTENER_BIND_PORT` -* Additional variables have been added to support "alternative addresses" and "alternative PKI", for example - to support using Let's Encrypt certificates easily in the quickstarts. -* New variables were introduced to allow automatic generation of the `alt_server_certs` section. Both variables - must be supplied for the variables to impact the configurations. - * `ZITI_PKI_ALT_SERVER_CERT` - "Alternative server certificate. Must be specified with ZITI_PKI_ALT_SERVER_KEY" - * `ZITI_PKI_ALT_SERVER_KEY` - "Key to use with the alternative server certificate. Must be specified with ZITI_PKI_ALT_SERVER_CERT" -* New variables were introduced to allow one to override and customize the CSR section of routers which is used during enrollment. - * `ZITI_ROUTER_CSR_C` - "The country (C) to use for router CSRs" - * `ZITI_ROUTER_CSR_ST` - "The state/province (ST) to use for router CSRs" - * `ZITI_ROUTER_CSR_L` - "The locality (L) to use for router CSRs" - * `ZITI_ROUTER_CSR_O` - "The organization (O) to use for router CSRs" - * `ZITI_ROUTER_CSR_OU` - "The organization unit to use for router CSRs" - * `ZITI_ROUTER_CSR_SANS_DNS` - "The DNS name used in the CSR request" -* New variable `ZITI_CTRL_EDGE_BIND_ADDRESS` allows controlling the IP the edge API uses - -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.81 -> v2.0.84](https://github.com/openziti/channel/compare/v2.0.81...v2.0.84) -* github.com/openziti/edge: [v0.24.348 -> v0.24.364](https://github.com/openziti/edge/compare/v0.24.348...v0.24.364) - * [Issue #1543](https://github.com/openziti/edge/issues/1543) - controller ca normalization can go into infinite loop on startup with bad certs - -* github.com/openziti/edge-api: [v0.25.29 -> v0.25.30](https://github.com/openziti/edge-api/compare/v0.25.29...v0.25.30) -* github.com/openziti/fabric: [v0.23.39 -> v0.23.45](https://github.com/openziti/fabric/compare/v0.23.39...v0.23.45) -* github.com/openziti/foundation/v2: [v2.0.26 -> v2.0.28](https://github.com/openziti/foundation/compare/v2.0.26...v2.0.28) -* github.com/openziti/identity: [v1.0.57 -> v1.0.59](https://github.com/openziti/identity/compare/v1.0.57...v1.0.59) -* github.com/openziti/runzmd: [v1.0.26 -> v1.0.28](https://github.com/openziti/runzmd/compare/v1.0.26...v1.0.28) -* github.com/openziti/sdk-golang: [v0.20.67 -> v0.20.78](https://github.com/openziti/sdk-golang/compare/v0.20.67...v0.20.78) -* github.com/openziti/storage: [v0.2.8 -> v0.2.11](https://github.com/openziti/storage/compare/v0.2.8...v0.2.11) -* github.com/openziti/transport/v2: [v2.0.91 -> v2.0.93](https://github.com/openziti/transport/compare/v2.0.91...v2.0.93) -* github.com/openziti/metrics: [v1.2.27 -> v1.2.30](https://github.com/openziti/metrics/compare/v1.2.27...v1.2.30) -* github.com/openziti/secretstream: [v0.1.9 -> v0.1.10](https://github.com/openziti/secretstream/compare/v0.1.9...v0.1.10) -* github.com/openziti/ziti: [v0.28.4 -> v0.29.0](https://github.com/openziti/ziti/compare/v0.28.4...v0.29.0) - * [Issue #1180](https://github.com/openziti/ziti/issues/1180) - Add ability to debug failed smoketests - * [Issue #1169](https://github.com/openziti/ziti/issues/1169) - Consolidate demo and tutorial under demo - * [Issue #1168](https://github.com/openziti/ziti/issues/1168) - Remove ziti-controller, ziti-router and ziti-tunnel executables from build - * [Issue #1158](https://github.com/openziti/ziti/issues/1158) - Add iperf tests to ziti smoketest - -# Release 0.28.4 - -## Component Updates and Bug Fixes - -* Restores Ziti Edge Client API as the default handler for `/version` and as the root handler to support previously enrolled GO SDK clients - -# Release 0.28.3 - -## What's New - -Bug fix - -## Component Updates and Bug Fixes - -* github.com/openziti/ziti: [v0.28.2 -> v0.28.3](https://github.com/openziti/ziti/compare/v0.28.2...v0.28.3) - -# Release 0.28.2 - -## What's New - -* IMPORTANT: If you update your OpenZiti binaries to this version or later (which can be done easily with the `getZiti()` function, you will need to migrate any existing network that has been developed using OpenZiti v0.27.5 or earlier binaries as the new binaries will expect the new environment variable names. A function `performMigration()` has been provided in the `ziti-cli-script.sh` for this purpose. Simply source the latest `ziti-cli-script.sh`, and your current network's .env file, then run `performMigration()` to update environment variable name references. If the migration process cannot find your existing environment file in the default location, you will need to provide the path to the migration function, ex: `performMigration ` -* If you were using the `ZITI_HOME` environment variable to configure where your ziti CLI profiles were stored, you should now use `ZITI_CONFIG_DIR` instead. - - -## Component Updates and Bug Fixes - -* github.com/openziti/channel/v2: [v2.0.80 -> v2.0.81](https://github.com/openziti/channel/compare/v2.0.80...v2.0.81) -* github.com/openziti/edge: [v0.24.326 -> v0.24.345](https://github.com/openziti/edge/compare/v0.24.326...v0.24.345) - * [Issue #1528](https://github.com/openziti/edge/issues/1528) - edge unbind returns incorrect message if token is not supplied or invalid - * [Issue #1416](https://github.com/openziti/edge/issues/1416) - Allow MFA token name to be configured - -* github.com/openziti/edge-api: [v0.25.25 -> v0.25.29](https://github.com/openziti/edge-api/compare/v0.25.25...v0.25.29) -* github.com/openziti/fabric: [v0.23.35 -> v0.23.39](https://github.com/openziti/fabric/compare/v0.23.35...v0.23.39) - * [Issue #751](https://github.com/openziti/fabric/issues/751) - Use of Fprintf causing buffer pool corruption with amqp event output - -* github.com/openziti/foundation/v2: [v2.0.25 -> v2.0.26](https://github.com/openziti/foundation/compare/v2.0.25...v2.0.26) -* github.com/openziti/identity: [v1.0.56 -> v1.0.57](https://github.com/openziti/identity/compare/v1.0.56...v1.0.57) -* github.com/openziti/runzmd: [v1.0.25 -> v1.0.26](https://github.com/openziti/runzmd/compare/v1.0.25...v1.0.26) -* github.com/openziti/sdk-golang: [v0.20.58 -> v0.20.67](https://github.com/openziti/sdk-golang/compare/v0.20.58...v0.20.67) -* github.com/openziti/storage: [v0.2.7 -> v0.2.8](https://github.com/openziti/storage/compare/v0.2.7...v0.2.8) -* github.com/openziti/transport/v2: [v2.0.90 -> v2.0.91](https://github.com/openziti/transport/compare/v2.0.90...v2.0.91) -* github.com/openziti/metrics: [v1.2.26 -> v1.2.27](https://github.com/openziti/metrics/compare/v1.2.26...v1.2.27) -* github.com/openziti/secretstream: [v0.1.8 -> v0.1.9](https://github.com/openziti/secretstream/compare/v0.1.8...v0.1.9) -* github.com/openziti/ziti: [v0.28.1 -> v0.28.2](https://github.com/openziti/ziti/compare/v0.28.1...v0.28.2) - * [Issue #1144](https://github.com/openziti/ziti/issues/1144) - DB explore subcommand panic - * [Issue #986](https://github.com/openziti/ziti/issues/986) - Updated default ports in `.env` file to match documentation. - * [Issue #920](https://github.com/openziti/ziti/issues/920) - Fixed bug causing failure when re-running quickstart. - * [Issue #779](https://github.com/openziti/ziti/issues/779) - Add ability to upgrade ziti binaries using a quickstart function. - * [Issue #761](https://github.com/openziti/ziti/issues/761) - Remove Management Listener section from controller config. - * [Issue #650](https://github.com/openziti/ziti/issues/650) - Removed/Updated references to `ZITI_EDGE_CONTROLLER_API` - * Quickstart environment variable names have been cleaned up. - * [Issue #1030](https://github.com/openziti/ziti/issues/1030) - Provide an upgrade path for quickstart cleanup - -# Release 0.28.1 - -## What's New - -* `ziti` CLI now trims jwt files specified for login preventing a confusing invalid header field value for "Authorization" - error when trying to use `-e` flag - -## Router Health Check Changes - -The link health check on routers now supports an initial delay configuration. - -``` - -healthChecks: - linkCheck: - minLinks: 1 - interval: 30s - initialDelay: 5s -``` - -The health check will also now start with an initial state of unhealthy, unless `minLinks` is set to zero. - -Finally, link checks now include the addresses associated with the links: - -```json - { - "details": [ - { - "linkId": "6a72EtnLib5nUvjhVLuHOb", - "destRouterId": "5uUxuQ3u6Q", - "latency": 2732886.5, - "addresses": { - "ack": { - "localAddr": "tcp:127.0.0.1:4023", - "remoteAddr": "tcp:127.0.0.1:33520" - }, - "payload": { - "localAddr": "tcp:127.0.0.1:4023", - "remoteAddr": "tcp:127.0.0.1:33504" - } - } - } - ], - "healthy": true, - "id": "link.health", - "lastCheckDuration": "53.213µs", - "lastCheckTime": "2023-06-01T18:35:11Z" - } -``` - -## Event Changes - -### AMQP Event Writer Changes -A new field is available to the AMQP Event Writer. `bufferSize` denotes how many messages ziti will hold during AMQP connection outages. Any messages exceeding this limit will be logged and dropped. - -Example configuration: -``` -events: - jsonLogger: - subscriptions: - - type: fabric.circuits - handler: - type: amqp - format: json - url: "amqp://localhost:5672" - queue: ziti - durable: true //default:true - autoDelete: false //default:false - exclusive: false //default:false - noWait: false //default:false - bufferSize: 50 //default:50 -``` - -## Component Updates and Bug Fixes - -* github.com/openziti/agent: [v1.0.13 -> v1.0.14](https://github.com/openziti/agent/compare/v1.0.13...v1.0.14) -* github.com/openziti/channel/v2: [v2.0.78 -> v2.0.80](https://github.com/openziti/channel/compare/v2.0.78...v2.0.80) -* github.com/openziti/edge: [v0.24.309 -> v0.24.326](https://github.com/openziti/edge/compare/v0.24.309...v0.24.326) - * [Issue #1512](https://github.com/openziti/edge/issues/1512) - Panic when removing edge terminator with expired session - * [Issue #1509](https://github.com/openziti/edge/issues/1509) - SDK hosted terminators are being removed twice, causing spurious errors - * [Issue #1507](https://github.com/openziti/edge/issues/1507) - edge-router with encryption disabled fails - * [Issue #1517](https://github.com/openziti/edge/issues/1517) - allow wildcard domains in intercept.v1 addresses - -* github.com/openziti/edge-api: [v0.25.24 -> v0.25.25](https://github.com/openziti/edge-api/compare/v0.25.24...v0.25.25) -* github.com/openziti/fabric: [v0.23.29 -> v0.23.35](https://github.com/openziti/fabric/compare/v0.23.29...v0.23.35) - * [Issue #538](https://github.com/openziti/fabric/issues/538) - Allow quiescing/dequiescing routers - * [Issue #738](https://github.com/openziti/fabric/issues/738) - Timeout from routing is getting reported as conn refused instead of timeout - * [Issue #737](https://github.com/openziti/fabric/issues/737) - Router link check should support initial delay configuration - * [Issue #735](https://github.com/openziti/fabric/issues/735) - router link health check should only be passing initially if min links is zero - * [Issue #733](https://github.com/openziti/fabric/issues/733) - Show link addresses in health check - * [Issue #732](https://github.com/openziti/fabric/issues/732) - Added new `bufferSize` config option to amqp handler. Connection handling now happens in the background with exponential retries. - -* github.com/openziti/foundation/v2: [v2.0.24 -> v2.0.25](https://github.com/openziti/foundation/compare/v2.0.24...v2.0.25) -* github.com/openziti/identity: [v1.0.54 -> v1.0.56](https://github.com/openziti/identity/compare/v1.0.54...v1.0.56) -* github.com/openziti/runzmd: [v1.0.24 -> v1.0.25](https://github.com/openziti/runzmd/compare/v1.0.24...v1.0.25) -* github.com/openziti/sdk-golang: [v0.20.51 -> v0.20.58](https://github.com/openziti/sdk-golang/compare/v0.20.51...v0.20.58) - * [Issue #409](https://github.com/openziti/sdk-golang/issues/409) - sdk-golang v0.20.49 loops forever with older 'ws://' edge router - -* github.com/openziti/storage: [v0.2.6 -> v0.2.7](https://github.com/openziti/storage/compare/v0.2.6...v0.2.7) -* github.com/openziti/transport/v2: [v2.0.88 -> v2.0.90](https://github.com/openziti/transport/compare/v2.0.88...v2.0.90) -* github.com/openziti/metrics: [v1.2.25 -> v1.2.26](https://github.com/openziti/metrics/compare/v1.2.25...v1.2.26) -* github.com/openziti/ziti: [v0.28.0 -> v0.28.1](https://github.com/openziti/ziti/compare/v0.28.0...v0.28.1) - * [Issue #1132](https://github.com/openziti/ziti/issues/1132) - Updated `ws` protocol to `wss` as `ws` is no longer supported. - -# Release 0.28.0 - -## What's New - -* Event changes - * Added AMQP event writer for events - * Add entity change events for auditing or external integration - * Add usage event filtering - * Add annotations to circuit events -* CLI additions for `ziti` to login with certificates or external-jwt-signers -* NOTE: ziti edge login flag changes: - * `-c` flag has been changed to map to `--client-cert` - * `--cert` is now `--ca` and has no short flag representation - * `-e/--ext-jwt` allows a user to supply a file containing a jwt used with ext-jwt-signers to login - * `-c/--client-cert` allows a certificate to be supplied to login (used with `-k/--client-key`) - * `-k/--client-key` allows a key to be supplied to login (used with `-c/--client-cert`) -* Config type changes - * address fields in `intercept.v1`, `host.v1`, and `host.v2` config types now permit hostnames with underscores. -* Edge Router/Tunneler now supports setting default UDP idle timeout/check interval - -## Event Changes - -### AMQP Event Writer -Previously events could only be emitted to a file. They can now also be emitted to an AMQP endpoint. - -Example configuration: -``` -events: - jsonLogger: - subscriptions: - - type: fabric.circuits - handler: - type: amqp - format: json - url: "amqp://localhost:5672" - queue: ziti - durable: true //default:true - autoDelete: false //default:false - exclusive: false //default:false - noWait: false //default:false -``` - -### Entity Change Events -OpenZiti can now be configured to emit entity change events. These events describe the changes when entities stored in the -bbolt database are created, updated or deleted. - -Note that events are emitted during the transaction. They are emitted at the end, so it's unlikely, but possible that an event will be emitted for a change which is rolled back. For this reason a following event will emitted when the change is committed. If a system crashes after commit, but before the committed event can be emitted, it will be emitted on the next startup. - -Example configuration: - -``` -events: - jsonLogger: - subscriptions: - - type: entityChange - include: - - services - - identities - handler: - type: file - format: json - path: /tmp/ziti-events.log -``` - -See the related issue for discussion: https://github.com/openziti/fabric/issues/562 - -Example output: - -``` -{ - "namespace": "entityChange", - "eventId": "326faf6c-8123-42ae-9ed8-6fd9560eb567", - "eventType": "created", - "timestamp": "2023-05-11T21:41:47.128588927-04:00", - "metadata": { - "author": { - "type": "identity", - "id": "ji2Rt8KJ4", - "name": "Default Admin" - }, - "source": { - "type": "rest", - "auth": "edge", - "localAddr": "localhost:1280", - "remoteAddr": "127.0.0.1:37578", - "method": "POST" - }, - "version": "v0.0.0" - }, - "entityType": "services", - "isParentEvent": false, - "initialState": null, - "finalState": { - "id": "6S0bCGWb6yrAutXwSQaLiv", - "createdAt": "2023-05-12T01:41:47.128138887Z", - "updatedAt": "2023-05-12T01:41:47.128138887Z", - "tags": {}, - "isSystem": false, - "name": "test", - "terminatorStrategy": "smartrouting", - "roleAttributes": [ - "goodbye", - "hello" - ], - "configs": null, - "encryptionRequired": true - } -} - -{ - "namespace": "entityChange", - "eventId": "326faf6c-8123-42ae-9ed8-6fd9560eb567", - "eventType": "committed", - "timestamp": "2023-05-11T21:41:47.129235443-04:00" -} -``` - -### Usage Event Filtering -Usage events, version 3, can now be filtered based on type. - -The valid types include: - -* ingress.rx -* ingress.tx -* egress.rx -* egress.tx -* fabric.rx -* fabric.tx - -Example configuration: - -``` -events: - jsonLogger: - subscriptions: - - type: fabric.usage - version: 3 - include: - - ingress.rx - - egress.rx -``` - -### Circuit Event Annotations -Circuit events initiated from the edge are now annotated with clientId, hostId and serviceId, to match usage events. The client and host ids are identity ids. - -Example output: - -``` - { - "namespace": "fabric.circuits", - "version": 2, - "event_type": "created", - "circuit_id": "0CEjWYiw6", - "timestamp": "2023-05-05T11:44:03.242399585-04:00", - "client_id": "clhaq7u7600o4ucgdpxy9i4t1", - "service_id": "QARLLTKjqfLZytmSsIqba", - "terminator_id": "7ddcd421-2b00-4b49-9ac0-8c78fe388c30", - "instance_id": "", - "creation_timespan": 1014280, - "path": { - "nodes": [ - "U7OwPtfjg", - "a4rC9DrZ3" - ], - "links": [ - "7Ru3hoxsssZzUNOyvd8Jcb" - ], - "ingress_id": "K9lD", - "egress_id": "rQLK", - "initiator_local_addr": "100.64.0.1:1234", - "initiator_remote_addr": "100.64.0.1:37640", - "terminator_local_addr": "127.0.0.1:45566", - "terminator_remote_addr": "127.0.0.1:1234" - }, - "link_count": 1, - "path_cost": 392151, - "tags": { - "clientId": "U7OwPtfjg", - "hostId": "a4rC9DrZ3", - "serviceId": "QARLLTKjqfLZytmSsIqba" - } -} -``` - -## ER/T UDP Settings - -The edge router tunneler now allows configuring a timeout and check interval for tproxy UDP intercepts. By default intercepted UDP -connections will be closed after five minutes of no traffic, checking every thirty seconds. The configuration is done in the router -config file, in the options for the tunnel module. Note that these configuration options only apply to tproxy intercepts, not to -proxy or host side UDP connections. - -Example configuration: - -```yaml -listeners: - - binding: tunnel - options: - mode: tproxy - udpIdleTimeout: 10s - udpCheckInterval: 5s -``` - -## Component Updates and Bug Fixes -* github.com/openziti/agent: [v1.0.10 -> v1.0.13](https://github.com/openziti/agent/compare/v1.0.10...v1.0.13) -* github.com/openziti/channel/v2: [v2.0.58 -> v2.0.78](https://github.com/openziti/channel/compare/v2.0.58...v2.0.78) - * [Issue #98](https://github.com/openziti/channel/issues/98) - Set default connect timeout to 5 seconds - -* github.com/openziti/edge: [v0.24.239 -> v0.24.309](https://github.com/openziti/edge/compare/v0.24.239...v0.24.309) - * [Issue #1503](https://github.com/openziti/edge/issues/1503) - Support configurable UDP idle timeout and check interval for tproxy in edge router tunneler - * [Issue #1471](https://github.com/openziti/edge/issues/1471) - UDP intercept connections report incorrect local/remote addresses, making confusing events - * [Issue #629](https://github.com/openziti/edge/issues/629) - emit entity change events - * [Issue #1295](https://github.com/openziti/edge/issues/1295) - Ensure DB migrations work properly in a clustered setup (edge) - * [Issue #1418](https://github.com/openziti/edge/issues/1418) - Checks for session edge router availability are inefficient - -* github.com/openziti/edge-api: [v0.25.11 -> v0.25.24](https://github.com/openziti/edge-api/compare/v0.25.11...v0.25.24) -* github.com/openziti/fabric: [v0.22.87 -> v0.23.29](https://github.com/openziti/fabric/compare/v0.22.87...v0.23.29) - * [Issue #724](https://github.com/openziti/fabric/issues/724) - Controller should be notified of forwarding faults on links - * [Issue #725](https://github.com/openziti/fabric/issues/725) - If reroute fails, circuit should be torn down - * [Issue #706](https://github.com/openziti/fabric/issues/706) - Fix panic in link close - * [Issue #700](https://github.com/openziti/fabric/issues/700) - Additional Health Checks exposed on Edge Router - * [Issue #595](https://github.com/openziti/fabric/issues/595) - Add include filtering for V3 usage metrics - * [Issue #684](https://github.com/openziti/fabric/issues/684) - Add tag annotations to circuit events, similar to usage events - * [Issue #562](https://github.com/openziti/fabric/issues/562) - Add entity change events - * [Issue #677](https://github.com/openziti/fabric/issues/677) - Rework raft startup - * [Issue #582](https://github.com/openziti/fabric/issues/582) - Ensure DB migrations work properly in a clustered setup (fabric) - * [Issue #668](https://github.com/openziti/fabric/issues/668) - Add network.Run watchdog, to warn if processing is delayed - -* github.com/openziti/foundation/v2: [v2.0.21 -> v2.0.24](https://github.com/openziti/foundation/compare/v2.0.21...v2.0.24) -* github.com/openziti/identity: [v1.0.45 -> v1.0.54](https://github.com/openziti/identity/compare/v1.0.45...v1.0.54) -* github.com/openziti/runzmd: [v1.0.20 -> v1.0.24](https://github.com/openziti/runzmd/compare/v1.0.20...v1.0.24) -* github.com/openziti/sdk-golang: [v0.18.76 -> v0.20.51](https://github.com/openziti/sdk-golang/compare/v0.18.76...v0.20.51) - * [Issue #407](https://github.com/openziti/sdk-golang/issues/407) - Allowing filtering which edge router urls the sdk uses - * [Issue #394](https://github.com/openziti/sdk-golang/issues/394) - SDK does not recover from API session expiration (during app/computer suspend) - -* github.com/openziti/storage: [v0.1.49 -> v0.2.6](https://github.com/openziti/storage/compare/v0.1.49...v0.2.6) -* github.com/openziti/transport/v2: [v2.0.72 -> v2.0.88](https://github.com/openziti/transport/compare/v2.0.72...v2.0.88) -* github.com/openziti/metrics: [v1.2.19 -> v1.2.25](https://github.com/openziti/metrics/compare/v1.2.19...v1.2.25) -* github.com/openziti/secretstream: v0.1.8 (new) -* github.com/openziti/ziti: [v0.27.9 -> v0.28.0](https://github.com/openziti/ziti/compare/v0.27.9...v0.28.0) - * [Issue #1112](https://github.com/openziti/ziti/issues/1112) - `ziti pki create` creates CA's and intermediates w/ the same DN - * [Issue #1087](https://github.com/openziti/ziti/issues/1087) - re-enable CI in forks - * [Issue #1013](https://github.com/openziti/ziti/issues/1013) - docker env password is renewed at each `docker-compose up` - * [Issue #1077](https://github.com/openziti/ziti/issues/1077) - Show auth-policy name on identity list instead of id - * [Issue #1119](https://github.com/openziti/ziti/issues/1119) - intercept.v1 config should permit underscores in the address - * [Issue #1123](https://github.com/openziti/ziti/issues/1123) - cannot update config types with ziti cli - -# Archived Changelogs - -Archives are in the [changelogs](./changelogs/) folder. - +* `tls_handshake_limiter.in_process` - number of TLS handshakes in progress +* `tls_handshake_limiter.window_size` - number of TLS handhshakes allowed concurrently +* `tls_handshake_limiter.work_timer` - timer tracking how long TLS handshakes are taking + + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.122 -> v2.0.128](https://github.com/openziti/channel/compare/v2.0.122...v2.0.128) +* github.com/openziti/edge-api: [v0.26.14 -> v0.26.16](https://github.com/openziti/edge-api/compare/v0.26.14...v0.26.16) +* github.com/openziti/foundation/v2: [v2.0.40 -> v2.0.42](https://github.com/openziti/foundation/compare/v2.0.40...v2.0.42) +* github.com/openziti/identity: [v1.0.73 -> v1.0.75](https://github.com/openziti/identity/compare/v1.0.73...v1.0.75) +* github.com/openziti/metrics: [v1.2.48 -> v1.2.51](https://github.com/openziti/metrics/compare/v1.2.48...v1.2.51) +* github.com/openziti/runzmd: [v1.0.41 -> v1.0.43](https://github.com/openziti/runzmd/compare/v1.0.41...v1.0.43) +* github.com/openziti/sdk-golang: [v0.23.15 -> v0.23.19](https://github.com/openziti/sdk-golang/compare/v0.23.15...v0.23.19) +* github.com/openziti/secretstream: [v0.1.18 -> v0.1.19](https://github.com/openziti/secretstream/compare/v0.1.18...v0.1.19) +* github.com/openziti/storage: [v0.2.33 -> v0.2.36](https://github.com/openziti/storage/compare/v0.2.33...v0.2.36) +* github.com/openziti/transport/v2: [v2.0.125 -> v2.0.131](https://github.com/openziti/transport/compare/v2.0.125...v2.0.131) + * [Issue #79](https://github.com/openziti/transport/issues/79) - Add adaptive rate limiting to shared tls listener + +* github.com/openziti/ziti: [v0.34.2 -> v1.0.0](https://github.com/openziti/ziti/compare/v0.34.2...v1.0.0) + * [Issue #1923](https://github.com/openziti/ziti/issues/1923) - Add release validation test suite + * [Issue #1904](https://github.com/openziti/ziti/issues/1904) - Add TLS handshake rate limiter + * [Issue #1921](https://github.com/openziti/ziti/issues/1921) - Tidy CLI + * [Issue #1916](https://github.com/openziti/ziti/issues/1916) - SDK dials fails with 'token is malformed' error + * [Issue #1911](https://github.com/openziti/ziti/issues/1911) - Fix panic on first HA controller startup + * [Issue #1914](https://github.com/openziti/ziti/issues/1914) - Fix panic in PeerConnected + * [Issue #1781](https://github.com/openziti/ziti/issues/1781) - Admin identities have bind and dial permissions to services diff --git a/changelogs/CHANGELOG.0.28.md b/changelogs/CHANGELOG.0.28.md new file mode 100644 index 000000000..22ff0a397 --- /dev/null +++ b/changelogs/CHANGELOG.0.28.md @@ -0,0 +1,407 @@ +# Release 0.28.4 + +## Component Updates and Bug Fixes + +* Restores Ziti Edge Client API as the default handler for `/version` and as the root handler to support previously enrolled GO SDK clients + +# Release 0.28.3 + +## What's New + +Bug fix + +## Component Updates and Bug Fixes + +* github.com/openziti/ziti: [v0.28.2 -> v0.28.3](https://github.com/openziti/ziti/compare/v0.28.2...v0.28.3) + +# Release 0.28.2 + +## What's New + +* IMPORTANT: If you update your OpenZiti binaries to this version or later (which can be done easily with the `getZiti()` function, you will need to migrate any existing network that has been developed using OpenZiti v0.27.5 or earlier binaries as the new binaries will expect the new environment variable names. A function `performMigration()` has been provided in the `ziti-cli-script.sh` for this purpose. Simply source the latest `ziti-cli-script.sh`, and your current network's .env file, then run `performMigration()` to update environment variable name references. If the migration process cannot find your existing environment file in the default location, you will need to provide the path to the migration function, ex: `performMigration ` +* If you were using the `ZITI_HOME` environment variable to configure where your ziti CLI profiles were stored, you should now use `ZITI_CONFIG_DIR` instead. + + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.80 -> v2.0.81](https://github.com/openziti/channel/compare/v2.0.80...v2.0.81) +* github.com/openziti/edge: [v0.24.326 -> v0.24.345](https://github.com/openziti/edge/compare/v0.24.326...v0.24.345) + * [Issue #1528](https://github.com/openziti/edge/issues/1528) - edge unbind returns incorrect message if token is not supplied or invalid + * [Issue #1416](https://github.com/openziti/edge/issues/1416) - Allow MFA token name to be configured + +* github.com/openziti/edge-api: [v0.25.25 -> v0.25.29](https://github.com/openziti/edge-api/compare/v0.25.25...v0.25.29) +* github.com/openziti/fabric: [v0.23.35 -> v0.23.39](https://github.com/openziti/fabric/compare/v0.23.35...v0.23.39) + * [Issue #751](https://github.com/openziti/fabric/issues/751) - Use of Fprintf causing buffer pool corruption with amqp event output + +* github.com/openziti/foundation/v2: [v2.0.25 -> v2.0.26](https://github.com/openziti/foundation/compare/v2.0.25...v2.0.26) +* github.com/openziti/identity: [v1.0.56 -> v1.0.57](https://github.com/openziti/identity/compare/v1.0.56...v1.0.57) +* github.com/openziti/runzmd: [v1.0.25 -> v1.0.26](https://github.com/openziti/runzmd/compare/v1.0.25...v1.0.26) +* github.com/openziti/sdk-golang: [v0.20.58 -> v0.20.67](https://github.com/openziti/sdk-golang/compare/v0.20.58...v0.20.67) +* github.com/openziti/storage: [v0.2.7 -> v0.2.8](https://github.com/openziti/storage/compare/v0.2.7...v0.2.8) +* github.com/openziti/transport/v2: [v2.0.90 -> v2.0.91](https://github.com/openziti/transport/compare/v2.0.90...v2.0.91) +* github.com/openziti/metrics: [v1.2.26 -> v1.2.27](https://github.com/openziti/metrics/compare/v1.2.26...v1.2.27) +* github.com/openziti/secretstream: [v0.1.8 -> v0.1.9](https://github.com/openziti/secretstream/compare/v0.1.8...v0.1.9) +* github.com/openziti/ziti: [v0.28.1 -> v0.28.2](https://github.com/openziti/ziti/compare/v0.28.1...v0.28.2) + * [Issue #1144](https://github.com/openziti/ziti/issues/1144) - DB explore subcommand panic + * [Issue #986](https://github.com/openziti/ziti/issues/986) - Updated default ports in `.env` file to match documentation. + * [Issue #920](https://github.com/openziti/ziti/issues/920) - Fixed bug causing failure when re-running quickstart. + * [Issue #779](https://github.com/openziti/ziti/issues/779) - Add ability to upgrade ziti binaries using a quickstart function. + * [Issue #761](https://github.com/openziti/ziti/issues/761) - Remove Management Listener section from controller config. + * [Issue #650](https://github.com/openziti/ziti/issues/650) - Removed/Updated references to `ZITI_EDGE_CONTROLLER_API` + * Quickstart environment variable names have been cleaned up. + * [Issue #1030](https://github.com/openziti/ziti/issues/1030) - Provide an upgrade path for quickstart cleanup + +# Release 0.28.1 + +## What's New + +* `ziti` CLI now trims jwt files specified for login preventing a confusing invalid header field value for "Authorization" + error when trying to use `-e` flag + +## Router Health Check Changes + +The link health check on routers now supports an initial delay configuration. + +``` + +healthChecks: + linkCheck: + minLinks: 1 + interval: 30s + initialDelay: 5s +``` + +The health check will also now start with an initial state of unhealthy, unless `minLinks` is set to zero. + +Finally, link checks now include the addresses associated with the links: + +```json + { + "details": [ + { + "linkId": "6a72EtnLib5nUvjhVLuHOb", + "destRouterId": "5uUxuQ3u6Q", + "latency": 2732886.5, + "addresses": { + "ack": { + "localAddr": "tcp:127.0.0.1:4023", + "remoteAddr": "tcp:127.0.0.1:33520" + }, + "payload": { + "localAddr": "tcp:127.0.0.1:4023", + "remoteAddr": "tcp:127.0.0.1:33504" + } + } + } + ], + "healthy": true, + "id": "link.health", + "lastCheckDuration": "53.213µs", + "lastCheckTime": "2023-06-01T18:35:11Z" + } +``` + +## Event Changes + +### AMQP Event Writer Changes +A new field is available to the AMQP Event Writer. `bufferSize` denotes how many messages ziti will hold during AMQP connection outages. Any messages exceeding this limit will be logged and dropped. + +Example configuration: +``` +events: + jsonLogger: + subscriptions: + - type: fabric.circuits + handler: + type: amqp + format: json + url: "amqp://localhost:5672" + queue: ziti + durable: true //default:true + autoDelete: false //default:false + exclusive: false //default:false + noWait: false //default:false + bufferSize: 50 //default:50 +``` + +## Component Updates and Bug Fixes + +* github.com/openziti/agent: [v1.0.13 -> v1.0.14](https://github.com/openziti/agent/compare/v1.0.13...v1.0.14) +* github.com/openziti/channel/v2: [v2.0.78 -> v2.0.80](https://github.com/openziti/channel/compare/v2.0.78...v2.0.80) +* github.com/openziti/edge: [v0.24.309 -> v0.24.326](https://github.com/openziti/edge/compare/v0.24.309...v0.24.326) + * [Issue #1512](https://github.com/openziti/edge/issues/1512) - Panic when removing edge terminator with expired session + * [Issue #1509](https://github.com/openziti/edge/issues/1509) - SDK hosted terminators are being removed twice, causing spurious errors + * [Issue #1507](https://github.com/openziti/edge/issues/1507) - edge-router with encryption disabled fails + * [Issue #1517](https://github.com/openziti/edge/issues/1517) - allow wildcard domains in intercept.v1 addresses + +* github.com/openziti/edge-api: [v0.25.24 -> v0.25.25](https://github.com/openziti/edge-api/compare/v0.25.24...v0.25.25) +* github.com/openziti/fabric: [v0.23.29 -> v0.23.35](https://github.com/openziti/fabric/compare/v0.23.29...v0.23.35) + * [Issue #538](https://github.com/openziti/fabric/issues/538) - Allow quiescing/dequiescing routers + * [Issue #738](https://github.com/openziti/fabric/issues/738) - Timeout from routing is getting reported as conn refused instead of timeout + * [Issue #737](https://github.com/openziti/fabric/issues/737) - Router link check should support initial delay configuration + * [Issue #735](https://github.com/openziti/fabric/issues/735) - router link health check should only be passing initially if min links is zero + * [Issue #733](https://github.com/openziti/fabric/issues/733) - Show link addresses in health check + * [Issue #732](https://github.com/openziti/fabric/issues/732) - Added new `bufferSize` config option to amqp handler. Connection handling now happens in the background with exponential retries. + +* github.com/openziti/foundation/v2: [v2.0.24 -> v2.0.25](https://github.com/openziti/foundation/compare/v2.0.24...v2.0.25) +* github.com/openziti/identity: [v1.0.54 -> v1.0.56](https://github.com/openziti/identity/compare/v1.0.54...v1.0.56) +* github.com/openziti/runzmd: [v1.0.24 -> v1.0.25](https://github.com/openziti/runzmd/compare/v1.0.24...v1.0.25) +* github.com/openziti/sdk-golang: [v0.20.51 -> v0.20.58](https://github.com/openziti/sdk-golang/compare/v0.20.51...v0.20.58) + * [Issue #409](https://github.com/openziti/sdk-golang/issues/409) - sdk-golang v0.20.49 loops forever with older 'ws://' edge router + +* github.com/openziti/storage: [v0.2.6 -> v0.2.7](https://github.com/openziti/storage/compare/v0.2.6...v0.2.7) +* github.com/openziti/transport/v2: [v2.0.88 -> v2.0.90](https://github.com/openziti/transport/compare/v2.0.88...v2.0.90) +* github.com/openziti/metrics: [v1.2.25 -> v1.2.26](https://github.com/openziti/metrics/compare/v1.2.25...v1.2.26) +* github.com/openziti/ziti: [v0.28.0 -> v0.28.1](https://github.com/openziti/ziti/compare/v0.28.0...v0.28.1) + * [Issue #1132](https://github.com/openziti/ziti/issues/1132) - Updated `ws` protocol to `wss` as `ws` is no longer supported. + +# Release 0.28.0 + +## What's New + +* Event changes + * Added AMQP event writer for events + * Add entity change events for auditing or external integration + * Add usage event filtering + * Add annotations to circuit events +* CLI additions for `ziti` to login with certificates or external-jwt-signers +* NOTE: ziti edge login flag changes: + * `-c` flag has been changed to map to `--client-cert` + * `--cert` is now `--ca` and has no short flag representation + * `-e/--ext-jwt` allows a user to supply a file containing a jwt used with ext-jwt-signers to login + * `-c/--client-cert` allows a certificate to be supplied to login (used with `-k/--client-key`) + * `-k/--client-key` allows a key to be supplied to login (used with `-c/--client-cert`) +* Config type changes + * address fields in `intercept.v1`, `host.v1`, and `host.v2` config types now permit hostnames with underscores. +* Edge Router/Tunneler now supports setting default UDP idle timeout/check interval + +## Event Changes + +### AMQP Event Writer +Previously events could only be emitted to a file. They can now also be emitted to an AMQP endpoint. + +Example configuration: +``` +events: + jsonLogger: + subscriptions: + - type: fabric.circuits + handler: + type: amqp + format: json + url: "amqp://localhost:5672" + queue: ziti + durable: true //default:true + autoDelete: false //default:false + exclusive: false //default:false + noWait: false //default:false +``` + +### Entity Change Events +OpenZiti can now be configured to emit entity change events. These events describe the changes when entities stored in the +bbolt database are created, updated or deleted. + +Note that events are emitted during the transaction. They are emitted at the end, so it's unlikely, but possible that an event will be emitted for a change which is rolled back. For this reason a following event will emitted when the change is committed. If a system crashes after commit, but before the committed event can be emitted, it will be emitted on the next startup. + +Example configuration: + +``` +events: + jsonLogger: + subscriptions: + - type: entityChange + include: + - services + - identities + handler: + type: file + format: json + path: /tmp/ziti-events.log +``` + +See the related issue for discussion: https://github.com/openziti/fabric/issues/562 + +Example output: + +``` +{ + "namespace": "entityChange", + "eventId": "326faf6c-8123-42ae-9ed8-6fd9560eb567", + "eventType": "created", + "timestamp": "2023-05-11T21:41:47.128588927-04:00", + "metadata": { + "author": { + "type": "identity", + "id": "ji2Rt8KJ4", + "name": "Default Admin" + }, + "source": { + "type": "rest", + "auth": "edge", + "localAddr": "localhost:1280", + "remoteAddr": "127.0.0.1:37578", + "method": "POST" + }, + "version": "v0.0.0" + }, + "entityType": "services", + "isParentEvent": false, + "initialState": null, + "finalState": { + "id": "6S0bCGWb6yrAutXwSQaLiv", + "createdAt": "2023-05-12T01:41:47.128138887Z", + "updatedAt": "2023-05-12T01:41:47.128138887Z", + "tags": {}, + "isSystem": false, + "name": "test", + "terminatorStrategy": "smartrouting", + "roleAttributes": [ + "goodbye", + "hello" + ], + "configs": null, + "encryptionRequired": true + } +} + +{ + "namespace": "entityChange", + "eventId": "326faf6c-8123-42ae-9ed8-6fd9560eb567", + "eventType": "committed", + "timestamp": "2023-05-11T21:41:47.129235443-04:00" +} +``` + +### Usage Event Filtering +Usage events, version 3, can now be filtered based on type. + +The valid types include: + +* ingress.rx +* ingress.tx +* egress.rx +* egress.tx +* fabric.rx +* fabric.tx + +Example configuration: + +``` +events: + jsonLogger: + subscriptions: + - type: fabric.usage + version: 3 + include: + - ingress.rx + - egress.rx +``` + +### Circuit Event Annotations +Circuit events initiated from the edge are now annotated with clientId, hostId and serviceId, to match usage events. The client and host ids are identity ids. + +Example output: + +``` + { + "namespace": "fabric.circuits", + "version": 2, + "event_type": "created", + "circuit_id": "0CEjWYiw6", + "timestamp": "2023-05-05T11:44:03.242399585-04:00", + "client_id": "clhaq7u7600o4ucgdpxy9i4t1", + "service_id": "QARLLTKjqfLZytmSsIqba", + "terminator_id": "7ddcd421-2b00-4b49-9ac0-8c78fe388c30", + "instance_id": "", + "creation_timespan": 1014280, + "path": { + "nodes": [ + "U7OwPtfjg", + "a4rC9DrZ3" + ], + "links": [ + "7Ru3hoxsssZzUNOyvd8Jcb" + ], + "ingress_id": "K9lD", + "egress_id": "rQLK", + "initiator_local_addr": "100.64.0.1:1234", + "initiator_remote_addr": "100.64.0.1:37640", + "terminator_local_addr": "127.0.0.1:45566", + "terminator_remote_addr": "127.0.0.1:1234" + }, + "link_count": 1, + "path_cost": 392151, + "tags": { + "clientId": "U7OwPtfjg", + "hostId": "a4rC9DrZ3", + "serviceId": "QARLLTKjqfLZytmSsIqba" + } +} +``` + +## ER/T UDP Settings + +The edge router tunneler now allows configuring a timeout and check interval for tproxy UDP intercepts. By default intercepted UDP +connections will be closed after five minutes of no traffic, checking every thirty seconds. The configuration is done in the router +config file, in the options for the tunnel module. Note that these configuration options only apply to tproxy intercepts, not to +proxy or host side UDP connections. + +Example configuration: + +```yaml +listeners: + - binding: tunnel + options: + mode: tproxy + udpIdleTimeout: 10s + udpCheckInterval: 5s +``` + +## Component Updates and Bug Fixes +* github.com/openziti/agent: [v1.0.10 -> v1.0.13](https://github.com/openziti/agent/compare/v1.0.10...v1.0.13) +* github.com/openziti/channel/v2: [v2.0.58 -> v2.0.78](https://github.com/openziti/channel/compare/v2.0.58...v2.0.78) + * [Issue #98](https://github.com/openziti/channel/issues/98) - Set default connect timeout to 5 seconds + +* github.com/openziti/edge: [v0.24.239 -> v0.24.309](https://github.com/openziti/edge/compare/v0.24.239...v0.24.309) + * [Issue #1503](https://github.com/openziti/edge/issues/1503) - Support configurable UDP idle timeout and check interval for tproxy in edge router tunneler + * [Issue #1471](https://github.com/openziti/edge/issues/1471) - UDP intercept connections report incorrect local/remote addresses, making confusing events + * [Issue #629](https://github.com/openziti/edge/issues/629) - emit entity change events + * [Issue #1295](https://github.com/openziti/edge/issues/1295) - Ensure DB migrations work properly in a clustered setup (edge) + * [Issue #1418](https://github.com/openziti/edge/issues/1418) - Checks for session edge router availability are inefficient + +* github.com/openziti/edge-api: [v0.25.11 -> v0.25.24](https://github.com/openziti/edge-api/compare/v0.25.11...v0.25.24) +* github.com/openziti/fabric: [v0.22.87 -> v0.23.29](https://github.com/openziti/fabric/compare/v0.22.87...v0.23.29) + * [Issue #724](https://github.com/openziti/fabric/issues/724) - Controller should be notified of forwarding faults on links + * [Issue #725](https://github.com/openziti/fabric/issues/725) - If reroute fails, circuit should be torn down + * [Issue #706](https://github.com/openziti/fabric/issues/706) - Fix panic in link close + * [Issue #700](https://github.com/openziti/fabric/issues/700) - Additional Health Checks exposed on Edge Router + * [Issue #595](https://github.com/openziti/fabric/issues/595) - Add include filtering for V3 usage metrics + * [Issue #684](https://github.com/openziti/fabric/issues/684) - Add tag annotations to circuit events, similar to usage events + * [Issue #562](https://github.com/openziti/fabric/issues/562) - Add entity change events + * [Issue #677](https://github.com/openziti/fabric/issues/677) - Rework raft startup + * [Issue #582](https://github.com/openziti/fabric/issues/582) - Ensure DB migrations work properly in a clustered setup (fabric) + * [Issue #668](https://github.com/openziti/fabric/issues/668) - Add network.Run watchdog, to warn if processing is delayed + +* github.com/openziti/foundation/v2: [v2.0.21 -> v2.0.24](https://github.com/openziti/foundation/compare/v2.0.21...v2.0.24) +* github.com/openziti/identity: [v1.0.45 -> v1.0.54](https://github.com/openziti/identity/compare/v1.0.45...v1.0.54) +* github.com/openziti/runzmd: [v1.0.20 -> v1.0.24](https://github.com/openziti/runzmd/compare/v1.0.20...v1.0.24) +* github.com/openziti/sdk-golang: [v0.18.76 -> v0.20.51](https://github.com/openziti/sdk-golang/compare/v0.18.76...v0.20.51) + * [Issue #407](https://github.com/openziti/sdk-golang/issues/407) - Allowing filtering which edge router urls the sdk uses + * [Issue #394](https://github.com/openziti/sdk-golang/issues/394) - SDK does not recover from API session expiration (during app/computer suspend) + +* github.com/openziti/storage: [v0.1.49 -> v0.2.6](https://github.com/openziti/storage/compare/v0.1.49...v0.2.6) +* github.com/openziti/transport/v2: [v2.0.72 -> v2.0.88](https://github.com/openziti/transport/compare/v2.0.72...v2.0.88) +* github.com/openziti/metrics: [v1.2.19 -> v1.2.25](https://github.com/openziti/metrics/compare/v1.2.19...v1.2.25) +* github.com/openziti/secretstream: v0.1.8 (new) +* github.com/openziti/ziti: [v0.27.9 -> v0.28.0](https://github.com/openziti/ziti/compare/v0.27.9...v0.28.0) + * [Issue #1112](https://github.com/openziti/ziti/issues/1112) - `ziti pki create` creates CA's and intermediates w/ the same DN + * [Issue #1087](https://github.com/openziti/ziti/issues/1087) - re-enable CI in forks + * [Issue #1013](https://github.com/openziti/ziti/issues/1013) - docker env password is renewed at each `docker-compose up` + * [Issue #1077](https://github.com/openziti/ziti/issues/1077) - Show auth-policy name on identity list instead of id + * [Issue #1119](https://github.com/openziti/ziti/issues/1119) - intercept.v1 config should permit underscores in the address + * [Issue #1123](https://github.com/openziti/ziti/issues/1123) - cannot update config types with ziti cli + +# Archived Changelogs + +Archives are in the [changelogs](./changelogs/) folder. + diff --git a/changelogs/CHANGELOG.0.29.md b/changelogs/CHANGELOG.0.29.md new file mode 100644 index 000000000..07063d07e --- /dev/null +++ b/changelogs/CHANGELOG.0.29.md @@ -0,0 +1,66 @@ +# Release 0.29.0 + +## What's New + +### Deprecated Binary Removal +This release removes the following deprecated binaries from the release archives. + +* `ziti-controller` - replaced by `ziti controller` +* `ziti-router` - replaced by `ziti router` +* `ziti-tunnel` - replaced by `ziti tunnel` + +The release archives now only contain the `ziti` executable. This executable is now at the root of the archive instead of nested under a `ziti` directory. + +### Ziti CLI Demo Consolidation + +The ziti CLI functions under `ziti learn`, namely `ziti learn demo` and `ziti learn tutorial` have been consolidated under `ziti demo`. + +### Continued Quickstart Changes + +The quickstart continues to evolve. A breaking change has occurred as numerous environment variables used to customize the quickstart +have changed again. A summary of changes is below + +* All `ZITI_EDGE_ROUTER_` variables have been changed to just `ZITI_ROUTER_`. + * `ZITI_EDGE_ROUTER_NAME` -> `ZITI_ROUTER_NAME` + * `ZITI_EDGE_ROUTER_PORT` -> `ZITI_ROUTER_PORT` + * `ZITI_EDGE_ROUTER_ADVERTISED_HOST` -> `ZITI_ROUTER_ADVERTISED_HOST` + * `ZITI_EDGE_ROUTER_IP_OVERRIDE` -> `ZITI_ROUTER_IP_OVERRIDE` + * `ZITI_EDGE_ROUTER_ENROLLMENT_DURATION` -> `ZITI_ROUTER_ENROLLMENT_DURATION` + * `ZITI_EDGE_ROUTER_ADVERTISED_HOST` -> `ZITI_ROUTER_ADVERTISED_HOST` + * `ZITI_EDGE_ROUTER_LISTENER_BIND_PORT` -> `ZITI_ROUTER_LISTENER_BIND_PORT` +* Additional variables have been added to support "alternative addresses" and "alternative PKI", for example + to support using Let's Encrypt certificates easily in the quickstarts. +* New variables were introduced to allow automatic generation of the `alt_server_certs` section. Both variables + must be supplied for the variables to impact the configurations. + * `ZITI_PKI_ALT_SERVER_CERT` - "Alternative server certificate. Must be specified with ZITI_PKI_ALT_SERVER_KEY" + * `ZITI_PKI_ALT_SERVER_KEY` - "Key to use with the alternative server certificate. Must be specified with ZITI_PKI_ALT_SERVER_CERT" +* New variables were introduced to allow one to override and customize the CSR section of routers which is used during enrollment. + * `ZITI_ROUTER_CSR_C` - "The country (C) to use for router CSRs" + * `ZITI_ROUTER_CSR_ST` - "The state/province (ST) to use for router CSRs" + * `ZITI_ROUTER_CSR_L` - "The locality (L) to use for router CSRs" + * `ZITI_ROUTER_CSR_O` - "The organization (O) to use for router CSRs" + * `ZITI_ROUTER_CSR_OU` - "The organization unit to use for router CSRs" + * `ZITI_ROUTER_CSR_SANS_DNS` - "The DNS name used in the CSR request" +* New variable `ZITI_CTRL_EDGE_BIND_ADDRESS` allows controlling the IP the edge API uses + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.81 -> v2.0.84](https://github.com/openziti/channel/compare/v2.0.81...v2.0.84) +* github.com/openziti/edge: [v0.24.348 -> v0.24.364](https://github.com/openziti/edge/compare/v0.24.348...v0.24.364) + * [Issue #1543](https://github.com/openziti/edge/issues/1543) - controller ca normalization can go into infinite loop on startup with bad certs + +* github.com/openziti/edge-api: [v0.25.29 -> v0.25.30](https://github.com/openziti/edge-api/compare/v0.25.29...v0.25.30) +* github.com/openziti/fabric: [v0.23.39 -> v0.23.45](https://github.com/openziti/fabric/compare/v0.23.39...v0.23.45) +* github.com/openziti/foundation/v2: [v2.0.26 -> v2.0.28](https://github.com/openziti/foundation/compare/v2.0.26...v2.0.28) +* github.com/openziti/identity: [v1.0.57 -> v1.0.59](https://github.com/openziti/identity/compare/v1.0.57...v1.0.59) +* github.com/openziti/runzmd: [v1.0.26 -> v1.0.28](https://github.com/openziti/runzmd/compare/v1.0.26...v1.0.28) +* github.com/openziti/sdk-golang: [v0.20.67 -> v0.20.78](https://github.com/openziti/sdk-golang/compare/v0.20.67...v0.20.78) +* github.com/openziti/storage: [v0.2.8 -> v0.2.11](https://github.com/openziti/storage/compare/v0.2.8...v0.2.11) +* github.com/openziti/transport/v2: [v2.0.91 -> v2.0.93](https://github.com/openziti/transport/compare/v2.0.91...v2.0.93) +* github.com/openziti/metrics: [v1.2.27 -> v1.2.30](https://github.com/openziti/metrics/compare/v1.2.27...v1.2.30) +* github.com/openziti/secretstream: [v0.1.9 -> v0.1.10](https://github.com/openziti/secretstream/compare/v0.1.9...v0.1.10) +* github.com/openziti/ziti: [v0.28.4 -> v0.29.0](https://github.com/openziti/ziti/compare/v0.28.4...v0.29.0) + * [Issue #1180](https://github.com/openziti/ziti/issues/1180) - Add ability to debug failed smoketests + * [Issue #1169](https://github.com/openziti/ziti/issues/1169) - Consolidate demo and tutorial under demo + * [Issue #1168](https://github.com/openziti/ziti/issues/1168) - Remove ziti-controller, ziti-router and ziti-tunnel executables from build + * [Issue #1158](https://github.com/openziti/ziti/issues/1158) - Add iperf tests to ziti smoketest diff --git a/changelogs/CHANGELOG.0.30.md b/changelogs/CHANGELOG.0.30.md new file mode 100644 index 000000000..31f5384d5 --- /dev/null +++ b/changelogs/CHANGELOG.0.30.md @@ -0,0 +1,383 @@ +# Release 0.30.5 + +## What's New + +* Initial proxy support in host.v1/host.v2 + +## Proxy Support in host.v1/host.v2 + +`host.v1` and `host.v2` configurations may now specify a proxy to use. +Currently only HTTP Connect proxies which don't require authentication are supported. + +**Example using `host.v1`** + +``` +{ + "address": "192.168.2.50", + "port": 1234, + "protocol": "tcp", + "proxy": { + "address": "192.168.1.110:3128", + "type": "http" + } +} +``` + + +## Component Updates and Bug Fixes +* github.com/openziti/channel/v2: [v2.0.99 -> v2.0.101](https://github.com/openziti/channel/compare/v2.0.99...v2.0.101) +* github.com/openziti/edge-api: [v0.25.37 -> v0.25.38](https://github.com/openziti/edge-api/compare/v0.25.37...v0.25.38) +* github.com/openziti/foundation/v2: [v2.0.32 -> v2.0.33](https://github.com/openziti/foundation/compare/v2.0.32...v2.0.33) +* github.com/openziti/identity: [v1.0.63 -> v1.0.64](https://github.com/openziti/identity/compare/v1.0.63...v1.0.64) +* github.com/openziti/metrics: [v1.2.35 -> v1.2.36](https://github.com/openziti/metrics/compare/v1.2.35...v1.2.36) +* github.com/openziti/runzmd: [v1.0.32 -> v1.0.33](https://github.com/openziti/runzmd/compare/v1.0.32...v1.0.33) +* github.com/openziti/sdk-golang: [v0.20.116 -> v0.20.122](https://github.com/openziti/sdk-golang/compare/v0.20.116...v0.20.122) + * [Issue #436](https://github.com/openziti/sdk-golang/issues/436) - HTTP calls should respect environment proxy settings + +* github.com/openziti/storage: [v0.2.18 -> v0.2.20](https://github.com/openziti/storage/compare/v0.2.18...v0.2.20) + * [Issue #52](https://github.com/openziti/storage/issues/52) - Grammar should expect single valid query followed by EOF + +* github.com/openziti/transport/v2: [v2.0.107 -> v2.0.109](https://github.com/openziti/transport/compare/v2.0.107...v2.0.109) +* github.com/openziti/ziti: [v0.30.4 -> v0.30.5](https://github.com/openziti/ziti/compare/v0.30.4...v0.30.5) + * [Issue #1336](https://github.com/openziti/ziti/issues/1336) - `ziti edge quickstart` did + not create the usual edge router/service edge router policy. + * [Issue #1397](https://github.com/openziti/ziti/issues/1397) - HTTP Proxy support for host.v1/host.v2 config types + * [Issue #1423](https://github.com/openziti/ziti/issues/1423) - Controller crashes when edge router reconnects (Client Hello) + * [Issue #1414](https://github.com/openziti/ziti/issues/1414) - Race condition in xgress_edge_tunnel tunneller at start but not seen in pre-compiled binary + * [Issue #1406](https://github.com/openziti/ziti/issues/1406) - Entity change event dispatcher isn't shutting down properly when controller shuts down + * [Issue #1382](https://github.com/openziti/ziti/issues/1382) - service failure costs are not shrinking over time + +# Release 0.30.4 + +## What's New + +* `ziti edge quickstart`](https://github.com/openziti/ziti/issues/1298). You can now + download the `ziti` CLI and have a functioning network with just one command. The + network it creates is ephemeral and is intended to be torn down when the process exits. + It is intended for quick evaluation and testing of an overlay network. It supports the + following flags: + + ``` + --already-initialized Specifies the PKI does not need to be created and the db does not need to be initialized. Recommended to be combined with --home. If --home is not specified the environment will be destroyed on shutdown! default: false + --ctrl-address string Sets the advertised address for the control plane and API + --ctrl-port int16 Sets the port to use for the control plane and API + -h, --help help for quickstart + --home string Sets the directory the environment should be installed into. Defaults to a temporary directory. If specified, the environment will not be removed on exit. + -p, --password string Password to use for authenticating to the Ziti Edge Controller. default: admin + --router-address string Sets the advertised address for the integrated router + --router-port int16 Sets the port to use for the integrated router + -u, --username string Username to use when creating the Ziti Edge Controller. default: admin + ``` + + Example Usage: + ``` + ziti edge quickstart \ + --ctrl-address potato \ + --ctrl-port 12345 \ + --router-address avocado \ + --router-port 23456 \ + --home $HOME/.ziti/pet-ziti \ + --already-initialized \ + --username someOtherUsername \ + --password someOtherPassword + ``` + +* Bugfixes + +## Component Updates and Bug Fixes + +* github.com/openziti/ziti: [v0.30.3 -> v0.30.4](https://github.com/openziti/ziti/compare/v0.30.3...v0.30.4) + * Fixed an issue causing router configs to be rewritten when docker compose was brought up with existing configs + +# Release 0.30.3 + +## What's New + +* Bugfixes + +## Component Updates and Bug Fixes + +* github.com/openziti/edge: [v0.24.401 -> v0.24.404](https://github.com/openziti/edge/compare/v0.24.401...v0.24.404) +* github.com/openziti/fabric: [v0.24.20 -> v0.24.23](https://github.com/openziti/fabric/compare/v0.24.20...v0.24.23) + * [Issue #786](https://github.com/openziti/fabric/issues/786) - entityChangeEventDispatcher.flushLoop doesn't shutdown when controller shuts down + * [Issue #785](https://github.com/openziti/fabric/issues/785) - Allow link groups to be single string value + * [Issue #783](https://github.com/openziti/fabric/issues/783) - Raft cluster connections not updated for ALPN + +* github.com/openziti/ziti: [v0.30.2 -> v0.30.3](https://github.com/openziti/ziti/compare/v0.30.2...v0.30.3) + +# Release 0.30.2 + +## What's New + +* Identity type consolidation +* HTTP Connect Proxy support for control channel and links + +## Identity Type Consolidation + +Prior to this release there were four identity types: + +* User +* Service +* Device +* Router + +Of these four types, only Router has any functional purpose. Given that, the other three have been merged into +a single `Default` identity type. Since Router identities can only be created by the system, it's no longer +necessary to specify the identity type when creating identities. + +The identity type may still be provided, but a deprecation warning will be emitted. + +**Backwards Compatibility** + +Existing non-Router identities will be migrated to the `Default` identity type. If an identity type other +than `Default` is provided when creating an identity, it will be coerced to the `Default` type. Existing +code may have issues with the new identity type being returned. + + +## HTTP Connect Proxy support + +Routers may now specify a proxy configuration which will be used when establishing connections to controllers +and data links to other routers. At this point only HTTP Connect Proxies with no authentication required are +supported. + +Example router config: + +```yaml +proxy: + type: http + address: localhost:3128 + +``` + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.91 -> v2.0.95](https://github.com/openziti/channel/compare/v2.0.91...v2.0.95) +* github.com/openziti/edge: [v0.24.381 -> v0.24.401](https://github.com/openziti/edge/compare/v0.24.381...v0.24.401) + * [Issue #1597](https://github.com/openziti/edge/issues/1597) - ER/T cached API sessions aren't remembered when pulled from cache + * [Issue #1428](https://github.com/openziti/edge/issues/1428) - Make identity type optional. Consolidate User/Service/Device to 'Default'. + * [Issue #1584](https://github.com/openziti/edge/issues/1584) - AuthPolicyDetail is incompatible with API response + +* github.com/openziti/edge-api: [v0.25.31 -> v0.25.33](https://github.com/openziti/edge-api/compare/v0.25.31...v0.25.33) +* github.com/openziti/fabric: [v0.24.2 -> v0.24.20](https://github.com/openziti/fabric/compare/v0.24.2...v0.24.20) + * [Issue #775](https://github.com/openziti/fabric/issues/775) - Add support for proxies for control channel and links + +* github.com/openziti/foundation/v2: [v2.0.29 -> v2.0.30](https://github.com/openziti/foundation/compare/v2.0.29...v2.0.30) +* github.com/openziti/identity: [v1.0.60 -> v1.0.61](https://github.com/openziti/identity/compare/v1.0.60...v1.0.61) +* github.com/openziti/runzmd: [v1.0.29 -> v1.0.30](https://github.com/openziti/runzmd/compare/v1.0.29...v1.0.30) +* github.com/openziti/sdk-golang: [v0.20.90 -> v0.20.101](https://github.com/openziti/sdk-golang/compare/v0.20.90...v0.20.101) +* github.com/openziti/storage: [v0.2.12 -> v0.2.14](https://github.com/openziti/storage/compare/v0.2.12...v0.2.14) +* github.com/openziti/transport/v2: [v2.0.99 -> v2.0.103](https://github.com/openziti/transport/compare/v2.0.99...v2.0.103) + * [Issue #54](https://github.com/openziti/transport/issues/54) - Support HTTP Connect proxying for TLS connections + +* github.com/openziti/metrics: [v1.2.31 -> v1.2.33](https://github.com/openziti/metrics/compare/v1.2.31...v1.2.33) +* github.com/openziti/secretstream: [v0.1.10 -> v0.1.11](https://github.com/openziti/secretstream/compare/v0.1.10...v0.1.11) +* github.com/openziti/ziti: [v0.30.1 -> v0.30.2](https://github.com/openziti/ziti/compare/v0.30.1...v0.30.2) + * [Issue #1266](https://github.com/openziti/ziti/issues/1266) - Outdated README.md: Some links return "Page Not Found" + + +# Release 0.30.1 + +## What's New + +## Component Updates and Bug Fixes + +* github.com/openziti/ziti: [v0.30.0 -> v0.30.1](https://github.com/openziti/ziti/compare/v0.30.0...v0.30.1) + * [Issue #1225](https://github.com/openziti/ziti/issues/1225) - Updated ZITI_ROUTER_ADVERTISED_HOST to use the more common naming convention of ZITI_ROUTER_ADVERTISED_ADDRESS + * [Issue #1233](https://github.com/openziti/ziti/issues/1233) - Added `lsof` to the list of prerequisites to be checked during quickstart + +# Release 0.30.0 + +## What's New + +* Link management is now delegated to routers +* Controller and routers can operate with a single listening port + +## Link Management Updates + +Previously, the controller would do its best to determine where links needed to be established. +It would send messages to the routers, telling them which addresses to dial on other routes. +The routers would in turn let the controller know if link establishment was successful or +if the router already had a link to the given endpoint. + +With this release, the controller will only let routers know which routers exist, whether they +are currently connected to the controller, and what link listeners they are advertising. The +routers will now decide which links to make and let the controllers know as links are created +and broken. + +### Link Groups + +Both dialers and listeners can now specify a set of groups. If no groups are specified, the +dialer or listener will be placed in the `default` group. Dialers will only attempt to dial +listeners who have at least one group in common with them. + + +### Failed Links + +Previously when a link failed, the controller would show it in the link list as failed for a time +before removing it. Now failed links are removed immediately. There are existing link events for +link creation and link failure which can be used for forensics. + +### Duplicate Links + +There is a new link status `Duplicate` used when a router receives a link request and determines +that it's a duplicate of an existing link. This happens when two routers both have listeners +and dialers. They will often dial each other at the same time, resulting in a duplicate link. + +### Compatibility + +If you use a 0.30+ controller with older routers, the controller will still do link calculation +and send dial messages, as long as the `enableLegacyLinkMgmt` setting is set to true. + +If you use a pre 0.30.0 controller with newer routers, the new routers will still accept the +dial messages. + +### New Configuration + +#### Controller + +The controller has three new options: + +``` +network: + routerMessaging: + queueSize: 100 + maxWorkers: 100 + enableLegacyLinkMgmt: true +``` + +When a router connects or disconnects from the controller, we send two sets of updates. + +1. If a router has connected we send it the the state of the other routers +1. We send all the other routers the updated state of the connecting/disconnecting router + +These messages are sent using a worker pool. The size of the queue feeding the worker pool is controlled with +`routerMessaging.queueSize`. The max size of the worker pool is controlled used the `routerMessaging.maxWorkers` +option. + +* queueSize + * Min value: 0 + * Max value: 1,000,000 + * Default: 100 +* maxWorkers + * Min value: 1 + * Max value: 10,000 + * Default: 100 + +If you have routers older than 0.30.0, the controller will calculate which links to dial. This can be disabled +by setting `enableLegacyLinkMgmt` to false. This setting currently defaults to true, but will default to false +in a future release. In a subsequent release this functionality will be removed all together. + +#### Router + +The router has new configuration options for link dialing. + +``` +link: + dialers: + - binding: transport + groups: + - public + - vpc1234 + healthyDialBackoff: + retryBackoffFactor: 1.5 + minRetryInterval: 5s + maxRetryInterval: 5m + unhealthyDialBackoff: + retryBackoffFactor: 10 + minRetryInterval: 1m + maxRetryInterval: 1h + listeners: + - binding: transport + groups: vpc1234 +``` + +**Groups** + +See above for a description of link groups work. + +Default value: `default` + +**Dial Back-off** + +Dialers can be configured with custom back-off behavior. Each dialer has a back-off policy for dialing +healthy routers (those that are connected to a controller) and a separate policy for unhealthy routers. + +The back-off policies have the following attributes: + +* minRetryInterval - duration specifying the minimum time between dial attempts + * Min value: 10ms + * Max value: 24h + * Default: 5s for healthy, 1m for unhealthy + * Format: Golang Durations, see: https://pkg.go.dev/maze.io/x/duration#ParseDuration +* maxRetryInterval - duration specifying the maximum time between dial attempts + * Min value: 10ms + * Max value: 24h + * Default: 5m for healthy, 1h for unhealthy + * Format: Golang duration, see: https://pkg.go.dev/maze.io/x/duration#ParseDuration +* retryBackoffFactor - factor by which to increase the retry interval between failed dial attempts + * Min value: 1 + * Max value: 100 + * Default: 1.5 for healthy, 100 for unhealthy + + +## Single Port Support / ALPN +Ziti Controller and Routers can operate with a single open port. In order to implement this feature we use +ALPN ([Application Layer Protocol Negotiation](https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation)) +TLS extension. It allows TLS client to request and TLS server to select appropriate application protocol handler during +TLS handshake. + +### Protocol Details + +The following protocol identifiers are defined: + +| id | purpose | +| --------- | ------- | +| ziti-ctrl | Control plane connections | +| ziti-link | Fabric link connections | +| ziti-edge | Client SDK connection to Edge Routers | + +Standard HTTP protocol identifiers (`h2`, `http/1.1`) are used for Controller REST API and Websocket listeners. + +### Backward Compatibility + +This feature is designed to be backward compatible with SDK clients: older client will still be able to connect without +requesting `ziti-edge` protocol. + +**Breaking** + +Older routers won't be able to establish control channel or fabric links with updated network. +However, newer Edge Routers should be able to join older network in some circumstances -- only outbound links from new Routers would work. + +## Component Updates and Bug Fixes +* github.com/openziti/agent: [v1.0.14 -> v1.0.15](https://github.com/openziti/agent/compare/v1.0.14...v1.0.15) +* github.com/openziti/channel/v2: [v2.0.84 -> v2.0.91](https://github.com/openziti/channel/compare/v2.0.84...v2.0.91) + * [Issue #108](https://github.com/openziti/channel/issues/108) - Reconnecting underlay not returning headers from hello message + +* github.com/openziti/edge: [v0.24.364 -> v0.24.381](https://github.com/openziti/edge/compare/v0.24.364...v0.24.381) + * [Issue #1548](https://github.com/openziti/edge/issues/1548) - Panic in edge@v0.24.326/controller/sync_strats/sync_instant.go:194 + +* github.com/openziti/edge-api: [v0.25.30 -> v0.25.31](https://github.com/openziti/edge-api/compare/v0.25.30...v0.25.31) +* github.com/openziti/fabric: [v0.23.45 -> v0.24.2](https://github.com/openziti/fabric/compare/v0.23.45...v0.24.2) + * [Issue #766](https://github.com/openziti/fabric/issues/766) - Lookup of terminators with same instance id isn't filtering by instance id + * [Issue #692](https://github.com/openziti/fabric/issues/692) - Add ability to control link formation between devices more granularly + * [Issue #749](https://github.com/openziti/fabric/issues/749) - Move link control to router + * [Issue #343](https://github.com/openziti/fabric/issues/343) - Link state Failed on startup + +* github.com/openziti/foundation/v2: [v2.0.28 -> v2.0.29](https://github.com/openziti/foundation/compare/v2.0.28...v2.0.29) +* github.com/openziti/identity: [v1.0.59 -> v1.0.60](https://github.com/openziti/identity/compare/v1.0.59...v1.0.60) +* github.com/openziti/runzmd: [v1.0.28 -> v1.0.29](https://github.com/openziti/runzmd/compare/v1.0.28...v1.0.29) +* github.com/openziti/sdk-golang: [v0.20.78 -> v0.20.90](https://github.com/openziti/sdk-golang/compare/v0.20.78...v0.20.90) +* github.com/openziti/storage: [v0.2.11 -> v0.2.12](https://github.com/openziti/storage/compare/v0.2.11...v0.2.12) +* github.com/openziti/transport/v2: [v2.0.93 -> v2.0.99](https://github.com/openziti/transport/compare/v2.0.93...v2.0.99) +* github.com/openziti/xweb/v2: [v2.0.2 -> v2.1.0](https://github.com/openziti/xweb/compare/v2.0.2...v2.1.0) +* github.com/openziti/ziti-db-explorer: [v1.1.1 -> v1.1.3](https://github.com/openziti/ziti-db-explorer/compare/v1.1.1...v1.1.3) + * [Issue #4](https://github.com/openziti/ziti-db-explorer/issues/4) - db explore timeout error is uninformative + +* github.com/openziti/metrics: [v1.2.30 -> v1.2.31](https://github.com/openziti/metrics/compare/v1.2.30...v1.2.31) +* github.com/openziti/ziti: [v0.29.0 -> v0.30.0](https://github.com/openziti/ziti/compare/v0.29.0...v0.30.0) + * [Issue #1199](https://github.com/openziti/ziti/issues/1199) - ziti edge list enrollments - CLI gets 404 + * [Issue #1135](https://github.com/openziti/ziti/issues/1135) - Edge Router: Support multiple protocols on the same listener port + * [Issue #65](https://github.com/openziti/ziti/issues/65) - Add ECDSA support to PKI subcmd + * [Issue #1212](https://github.com/openziti/ziti/issues/1212) - getZiti fails on Mac OS + * [Issue #1220](https://github.com/openziti/ziti/issues/1220) - Fixed getZiti function not respecting user input for custom path + * [Issue #1219](https://github.com/openziti/ziti/issues/1219) - Added check for IPs provided as a DNS SANs entry, IPs will be ignored and not added as a DNS entry in the expressInstall PKI or router config generation. diff --git a/changelogs/CHANGELOG.0.31.md b/changelogs/CHANGELOG.0.31.md new file mode 100644 index 000000000..f6630dcda --- /dev/null +++ b/changelogs/CHANGELOG.0.31.md @@ -0,0 +1,203 @@ +# Release 0.31.4 + +## What's New + +* Bug fix for a data flow stall which is especially likely to happen on circuits with single router paths + +## Thanks + +* @marvkis - for providing high quality debug data which made tracking down a couple of flow control stall issues much easier + +## Component Updates and Bug Fixes + +* github.com/openziti/metrics: [v1.2.40 -> v1.2.41](https://github.com/openziti/metrics/compare/v1.2.40...v1.2.41) +* github.com/openziti/sdk-golang: [v0.21.2 -> v0.22.0](https://github.com/openziti/sdk-golang/compare/v0.21.2...v0.22.0) + * [Issue #468](https://github.com/openziti/sdk-golang/issues/468) - SDK does an unnecessary number of session refreshes + +* github.com/openziti/storage: [v0.2.26 -> v0.2.27](https://github.com/openziti/storage/compare/v0.2.26...v0.2.27) +* github.com/openziti/ziti: [v0.31.3 -> v0.31.4](https://github.com/openziti/ziti/compare/v0.31.3...v0.31.4) + * [Issue #1645](https://github.com/openziti/ziti/issues/1645) - Once routers share a link id, we can't use the link id to decide which duplicate link to discard + * [Issue #1642](https://github.com/openziti/ziti/issues/1642) - Revert posture check optimization + * [Issue #1586](https://github.com/openziti/ziti/issues/1586) - If ack is received before payload is processed by link send buffer, a stall can result + + +# Release 0.31.3 + +## What's New + +* Services Max Idle Time +* Add/Remove Peer and Transfer Leadership via REST + +## Service Max Idle Time + +A max idle time can now be configured on services. The default value of 0 indicates that no maximum will +be enforced. A circuit is considered idle when no traffic is flowing across through the initiating or +terminating router. + +``` +ziti edge create service test-service --max-idle-time 5m +``` + +Note that the idle time calculation is done on the router, so if max idle time on a service is less +than the configured scan interval on the router, it make take longer than expected for idle circuits +to be removed. + +## Raft Cluster Management via REST + +The controller now allows some Raft cluster management operations to be performed via REST. + +NOTE: If your cluster is not bootstrapped yet, the REST API won't be available. These will only work on a bootstrapped cluster! + +The following operations are now supported: + +* Add member +* Remove member +* Transfer leadership + +``` +ziti fabric raft add-member tls:localhost:6363 +ziti fabric raft add-member tls:localhost:6464 +ziti fabric raft transfer-leadership +ziti fabric raft transfer-leadership ctrl3 +ziti fabric raft remove-member ctrl2 +ziti fabric raft remove-member ctrl3 +``` + +## Component Updates and Bug Fixes + +* github.com/openziti/edge-api: [v0.26.1 -> v0.26.6](https://github.com/openziti/edge-api/compare/v0.26.1...v0.26.6) +* github.com/openziti/sdk-golang: [v0.20.139 -> v0.21.2](https://github.com/openziti/sdk-golang/compare/v0.20.139...v0.21.2) + * [Issue #465](https://github.com/openziti/sdk-golang/issues/465) - Allow listen options to specify how many listeners need to be established before returning + * [Issue #462](https://github.com/openziti/sdk-golang/issues/462) - Allow refreshing a single service + +* github.com/openziti/ziti: [v0.31.2 -> v0.31.3](https://github.com/openziti/ziti/compare/v0.31.2...v0.31.3) + * [Issue #1583](https://github.com/openziti/ziti/issues/1583) - xgress: Potential data stall due when processing acks after checking window size + * [Issue #1578](https://github.com/openziti/ziti/issues/1578) - Send BindSuccess notifications to SDK if supported + * [Issue #1544](https://github.com/openziti/ziti/issues/1544) - Support transfer raft leadership via REST + * [Issue #1543](https://github.com/openziti/ziti/issues/1543) - Support add/remove raft peer via REST + * [Issue #1496](https://github.com/openziti/ziti/issues/1496) - Configurable Timer needed to close idle circuits + * [Issue #1402](https://github.com/openziti/ziti/issues/1402) - Allow router to decomission itself + +# Release 0.31.2 + +## What's New + +* Go version updated from 1.20 to 1.21 + +# Release 0.31.1 + +## What's New + +* SDK Hosting Improvements +* Terminator validation utility +* Circuit/Link query support + +## SDK Hosting Improvements + +In previous versions of OpenZiti, if many SDK clients were attempting to establish hosting, the controller could get overwhelmed. +In this release, routers will use the rate limiter pool introduced in 0.27.6 when creating terminators on behalf of sdk clients +hosting applications. Additionally, routers now have the ability to verify terminator state with the sdk, if the sdk supports it. +In general, hosting large numbers of services using the sdk should now be less suceptible to thundering herd issues. + +## Manual Terminator Validation + +There is a new CLI command available to validate terminator state. This is primarily a developer tool to validate that terminator +setup logic is correct. However it may also be used to diagnose and resolve issues with production systems, should the need arise. + +``` +ziti fabric validate terminators +``` + +## Circuit/Link Query Support + +Previously listing circuit and links always showed the full list. This is because these types are in memory only and are not stored +in the bbolt datastore. There's now basic support for querying in-memory types and circuits and links can now be filtered/paged/sorted + the same as other entity types. + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.105 -> v2.0.111](https://github.com/openziti/channel/compare/v2.0.105...v2.0.111) + * [Issue #118](https://github.com/openziti/channel/issues/118) - Allowing checking if reconnecting impl is currently connected + +* github.com/openziti/edge-api: [v0.26.0 -> v0.26.1](https://github.com/openziti/edge-api/compare/v0.26.0...v0.26.1) +* github.com/openziti/foundation/v2: [v2.0.33 -> v2.0.35](https://github.com/openziti/foundation/compare/v2.0.33...v2.0.35) +* github.com/openziti/identity: [v1.0.66 -> v1.0.68](https://github.com/openziti/identity/compare/v1.0.66...v1.0.68) +* github.com/openziti/metrics: [v1.2.37 -> v1.2.40](https://github.com/openziti/metrics/compare/v1.2.37...v1.2.40) +* github.com/openziti/runzmd: [v1.0.33 -> v1.0.36](https://github.com/openziti/runzmd/compare/v1.0.33...v1.0.36) +* github.com/openziti/sdk-golang: [v0.20.129 -> v0.20.139](https://github.com/openziti/sdk-golang/compare/v0.20.129...v0.20.139) + * [Issue #457](https://github.com/openziti/sdk-golang/issues/457) - Add inspect support + * [Issue #450](https://github.com/openziti/sdk-golang/issues/450) - Support idempotent terminator creation + +* github.com/openziti/secretstream: [v0.1.13 -> v0.1.14](https://github.com/openziti/secretstream/compare/v0.1.13...v0.1.14) +* github.com/openziti/storage: [v0.2.23 -> v0.2.26](https://github.com/openziti/storage/compare/v0.2.23...v0.2.26) + * [Issue #57](https://github.com/openziti/storage/issues/57) - Support querying collections of in memory objects + +* github.com/openziti/transport/v2: [v2.0.113 -> v2.0.119](https://github.com/openziti/transport/compare/v2.0.113...v2.0.119) +* github.com/openziti/ziti: [v0.31.0 -> v0.31.1](https://github.com/openziti/ziti/compare/v0.31.0...v0.31.1) + * [Issue #1555](https://github.com/openziti/ziti/issues/1555) - Consolidate fabric/edge persistence code + * [Issue #1547](https://github.com/openziti/ziti/issues/1547) - Support filtering, sorting and paging circuits and links + * [Issue #1446](https://github.com/openziti/ziti/issues/1446) - Allow for idempotent sdk based terminators + * [Issue #1540](https://github.com/openziti/ziti/issues/1540) - Transit router create fails in HA environment + * [Issue #1523](https://github.com/openziti/ziti/issues/1523) - Bootstrap members not working + * [Issue #1525](https://github.com/openziti/ziti/issues/1525) - Improve cluster list output + * [Issue #1519](https://github.com/openziti/ziti/issues/1519) - Simplify link ack handling + * [Issue #1513](https://github.com/openziti/ziti/issues/1513) - DNS service failure should not cause a router restart + * [Issue #1494](https://github.com/openziti/ziti/issues/1494) - Panic if applying raft log returns nil result + + +# Release 0.31.0 + +## What's New + +* Rate limited for model changes + +## Rate Limiter for Model Changes + +To prevent the controller from being overwhelmed by a flood of changes, a rate limiter +can be enabled in the configuration file. A maximum number of queued changes can also +be configured. The rate limited is disabled by default for now. If not specified the +default number of queued changes is 100. + +When the rate limit is hit, an error will be returned. If the request came in from +the REST API, the response will use HTTP status code 429 (too many requests). + +The OpenAPI specs have been updated, so if you're using a generated client to make +REST calls, it's recommended that you regenerate your client. + + +``` +commandRateLimiter: + enabled: true + maxQueued: 100 +``` + +If the rate limiter is enabled, the following metrics will be produced: + +* `command.limiter.queued_count` - gauge of the current number of queued operations +* `command.limiter.work_timer` - timer for operations. Includes the following: + * A histogram of how long operations take to complete + * A meter showing that rate at which operations are executed + * A count of how many operations have been executed + +## Component Updates and Bug Fixes + +* github.com/openziti/agent: [v1.0.15 -> v1.0.16](https://github.com/openziti/agent/compare/v1.0.15...v1.0.16) +* github.com/openziti/channel/v2: [v2.0.101 -> v2.0.105](https://github.com/openziti/channel/compare/v2.0.101...v2.0.105) +* github.com/openziti/edge-api: [v0.25.38 -> v0.26.0](https://github.com/openziti/edge-api/compare/v0.25.38...v0.26.0) + * [Issue #49](https://github.com/openziti/edge-api/issues/49) - Add 429 responses to allow indicating that the server is too busy + +* github.com/openziti/identity: [v1.0.64 -> v1.0.66](https://github.com/openziti/identity/compare/v1.0.64...v1.0.66) +* github.com/openziti/metrics: [v1.2.36 -> v1.2.37](https://github.com/openziti/metrics/compare/v1.2.36...v1.2.37) +* github.com/openziti/sdk-golang: [v0.20.122 -> v0.20.129](https://github.com/openziti/sdk-golang/compare/v0.20.122...v0.20.129) + * [Issue #443](https://github.com/openziti/sdk-golang/issues/443) - Don't send close in response to a close on a listener + +* github.com/openziti/secretstream: [v0.1.12 -> v0.1.13](https://github.com/openziti/secretstream/compare/v0.1.12...v0.1.13) +* github.com/openziti/storage: [v0.2.20 -> v0.2.23](https://github.com/openziti/storage/compare/v0.2.20...v0.2.23) +* github.com/openziti/transport/v2: [v2.0.109 -> v2.0.113](https://github.com/openziti/transport/compare/v2.0.109...v2.0.113) +* github.com/openziti/ziti: [v0.30.5 -> v0.31.0](https://github.com/openziti/ziti/compare/v0.30.5...v0.31.0) + * [Issue #1471](https://github.com/openziti/ziti/issues/1471) - Router links not resilient to controller crash + * [Issue #1468](https://github.com/openziti/ziti/issues/1468) - Quickstart quietly fails if password is < 5 characters long + * [Issue #1445](https://github.com/openziti/ziti/issues/1445) - Add controller update guardrail + * [Issue #1442](https://github.com/openziti/ziti/issues/1442) - Network watchdog not shutting down when controller shuts down + * [Issue #1465](https://github.com/openziti/ziti/issues/1465) - Upgrade functions `getZiti` and `performMigration` were only functional on Mac OS, now they are functional for Linux and Mac OSs. + * [Issue #1217](https://github.com/openziti/ziti/issues/1217) - Quickstart was improperly handling special characters in `ZITI_PWD`. Special characters are now supported for `ZITI_PWD` in quickstart functions. diff --git a/changelogs/CHANGELOG.0.32.md b/changelogs/CHANGELOG.0.32.md new file mode 100644 index 000000000..8beab2c6a --- /dev/null +++ b/changelogs/CHANGELOG.0.32.md @@ -0,0 +1,150 @@ +# Release 0.32.2 + +## What's New + +* Terminator performance improvements +* API Rate Limiter enabled by default + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.117 -> v2.0.119](https://github.com/openziti/channel/compare/v2.0.117...v2.0.119) + * [Issue #127](https://github.com/openziti/channel/issues/127) - Support some additional types in message headers + +* github.com/openziti/foundation/v2: [v2.0.36 -> v2.0.37](https://github.com/openziti/foundation/compare/v2.0.36...v2.0.37) +* github.com/openziti/identity: [v1.0.69 -> v1.0.70](https://github.com/openziti/identity/compare/v1.0.69...v1.0.70) +* github.com/openziti/metrics: [v1.2.43 -> v1.2.45](https://github.com/openziti/metrics/compare/v1.2.43...v1.2.45) +* github.com/openziti/runzmd: [v1.0.37 -> v1.0.38](https://github.com/openziti/runzmd/compare/v1.0.37...v1.0.38) +* github.com/openziti/sdk-golang: [v0.22.21 -> v0.22.28](https://github.com/openziti/sdk-golang/compare/v0.22.21...v0.22.28) + * [Issue #495](https://github.com/openziti/sdk-golang/issues/495) - Refresh edge session if listen attempt fails, to ensure that session is still valid + +* github.com/openziti/storage: [v0.2.28 -> v0.2.30](https://github.com/openziti/storage/compare/v0.2.28...v0.2.30) +* github.com/openziti/transport/v2: [v2.0.121 -> v2.0.122](https://github.com/openziti/transport/compare/v2.0.121...v2.0.122) +* github.com/openziti/ziti: [v0.32.1 -> v0.32.2](https://github.com/openziti/ziti/compare/v0.32.1...v0.32.2) + * [Issue #1741](https://github.com/openziti/ziti/issues/1741) - Prevent stuck links + * [Issue #1736](https://github.com/openziti/ziti/issues/1736) - controller crashes generating create circuit responses + * [Issue #1733](https://github.com/openziti/ziti/issues/1733) - Improve terminator creation performance + * [Issue #1734](https://github.com/openziti/ziti/issues/1734) - Make API rate limiter enabled by default + * [Issue #1726](https://github.com/openziti/ziti/issues/1726) - Fix some sdk hosting logging + * [Issue #1725](https://github.com/openziti/ziti/issues/1725) - Fix panic in entity event processing + * [Issue #652](https://github.com/openziti/ziti/issues/652) - CI support for MacOS arm64 + +# Release 0.32.1 + +## What's New + +* Bugfixes +* New router setting to control startup timeout + +## Router startup timeout + +The router now has a configuration setting to control how long it wait on startup to be able to +connect to a controller, before it gives up and exits. + +``` +ctrl: + endpoints: + - tls:localhost:1280 + startupTimeout: 5m +``` + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.116 -> v2.0.117](https://github.com/openziti/channel/compare/v2.0.116...v2.0.117) + * [Issue #125](https://github.com/openziti/channel/issues/125) - Ensure reconnecting channel is marked as connected before calling reconnect callback + +* github.com/openziti/edge-api: [v0.26.8 -> v0.26.10](https://github.com/openziti/edge-api/compare/v0.26.8...v0.26.10) +* github.com/openziti/sdk-golang: [v0.22.17 -> v0.22.21](https://github.com/openziti/sdk-golang/compare/v0.22.17...v0.22.21) +* github.com/openziti/ziti: [v0.32.0 -> v0.32.1](https://github.com/openziti/ziti/compare/v0.32.0...v0.32.1) + * [Issue #1709](https://github.com/openziti/ziti/issues/1709) - Fix link management race conditions found by chaos testing + * [Issue #1715](https://github.com/openziti/ziti/issues/1715) - Ensure controller raft peers don't end up with duplicate connections + * [Issue #1702](https://github.com/openziti/ziti/issues/1702) - Add link management chaos test + * [Issue #1691](https://github.com/openziti/ziti/issues/1691) multiple er re-enrolls creates multiple enrollments + +# Release 0.32.0 + +## What's New + +* Auth Rate Limiter +* Link Management Fixes +* ziti edge quickstart command deprecates redundant --already-initialized flag. The identical behavior is implied by --home. + +## Backwards compatibility + +This release includes new response types from the REST authentication APIS. They are now able to return +`429` (server too busy) responses to auth requests. As this is an API change, the version number is +being bumped to 0.32. + +If controller and router are both v0.32 or later, only the router which dialed a link will report it to the controller. +If the controller is older, newer routers will report links from both the dialing and listening side of the link. + +## Auth Rate Limiter + +In order to prevent clients from overwhelming the server with auth requests, an auth rate limiter has been introduced. +The rate limiter is adaptive, in that it will react to auth attempts timing out by shrinking the number of allowed +queued auth attempts. The number will slowly recover over time. + +Example configuration: + +``` +edge: + # This section allows configurating the rate limiter for auth attempts + authRateLimiter: + # if disabled, no auth rate limiting with be enforced + enabled: true + # the smallest window size for auth attempts + minSize: 5 + # the largest allowed window size for auth attempts + maxSize: 250 +``` + +New metrics: + +* `auth.limiter.queued_count` - current number of queued auth attempts +* `auth.limiter.window_size` - current size at which new auth attempts will be rejected +* `auth.limiter.work_timer` - tracks the rate at which api sessions are being created and how long it's taking to create them + +## Link Management Fixes + +With long lived link ids, there was potential for link control message to be ambiguous, as the link id wasn't enough to identify +a specific iteration of that link. An iteration field has been added to links so that messaging is unambiguous. +Links will also only be reported from the dialing router now to reduce ambiguouity and race condition in link control channel +messaging. + +## Router SSL Handshake Timeout Config + +There is a new router config setting which allows setting the SSL handshake timeout for TLS connections, when using ALPN for listeners. + +``` +tls: + handshakeTimeout: 15s +``` + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.111 -> v2.0.116](https://github.com/openziti/channel/compare/v2.0.111...v2.0.116) + * [Issue #123](https://github.com/openziti/channel/issues/123) - Ensure hello messages respect connect timeout + * [Issue #120](https://github.com/openziti/channel/issues/120) - Allow handling new underlay instances with function instead of channel + +* github.com/openziti/edge-api: [v0.26.6 -> v0.26.8](https://github.com/openziti/edge-api/compare/v0.26.6...v0.26.8) +* github.com/openziti/foundation/v2: [v2.0.35 -> v2.0.36](https://github.com/openziti/foundation/compare/v2.0.35...v2.0.36) + * [Issue #391](https://github.com/openziti/foundation/issues/391) - goroutine pool can stall if configured for 0 min workers and with single producer + +* github.com/openziti/identity: [v1.0.68 -> v1.0.69](https://github.com/openziti/identity/compare/v1.0.68...v1.0.69) +* github.com/openziti/metrics: [v1.2.41 -> v1.2.43](https://github.com/openziti/metrics/compare/v1.2.41...v1.2.43) +* github.com/openziti/runzmd: [v1.0.36 -> v1.0.37](https://github.com/openziti/runzmd/compare/v1.0.36...v1.0.37) +* github.com/openziti/sdk-golang: [v0.22.0 -> v0.22.17](https://github.com/openziti/sdk-golang/compare/v0.22.0...v0.22.17) + * [Issue #482](https://github.com/openziti/sdk-golang/issues/482) - Deprecate ListenOptions.MaxConnections in favor of MaxTerminators + +* github.com/openziti/secretstream: [v0.1.14 -> v0.1.16](https://github.com/openziti/secretstream/compare/v0.1.14...v0.1.16) +* github.com/openziti/storage: [v0.2.27 -> v0.2.28](https://github.com/openziti/storage/compare/v0.2.27...v0.2.28) +* github.com/openziti/transport/v2: [v2.0.119 -> v2.0.121](https://github.com/openziti/transport/compare/v2.0.119...v2.0.121) + * [Issue #73](https://github.com/openziti/transport/issues/73) - Allow overriding shared TLS/ALPN listener SSL handshake timeout + +* github.com/openziti/ziti: [v0.31.4 -> v0.32.0](https://github.com/openziti/ziti/compare/v0.31.4...v0.32.0) + * [Issue #1692](https://github.com/openziti/ziti/issues/1692) - Improve link stability with long lived link ids + * [Issue #1693](https://github.com/openziti/ziti/issues/1693) - Make links owned by the dialing router + * [Issue #1685](https://github.com/openziti/ziti/issues/1685) - Race condition where we try to create terminator after client connection is closed + * [Issue #1678](https://github.com/openziti/ziti/issues/1678) - Add link validation utility + * [Issue #1673](https://github.com/openziti/ziti/issues/1673) - xgress dialers not getting passed xgress config + * [Issue #1669](https://github.com/openziti/ziti/issues/1669) - Make sure link accepts are not single threaded + * [Issue #1657](https://github.com/openziti/ziti/issues/1657) - Add api session rate limiter diff --git a/changelogs/CHANGELOG.0.33.md b/changelogs/CHANGELOG.0.33.md new file mode 100644 index 000000000..c1ce3de07 --- /dev/null +++ b/changelogs/CHANGELOG.0.33.md @@ -0,0 +1,53 @@ +# Release 0.33.1 + +## What's New + +* Backward compatibility router <-> controller fix to address metrics parsing panic + +## Component Updates and Bug Fixes +* github.com/openziti/ziti: [v0.33.0 -> v0.33.1](https://github.com/openziti/ziti/compare/v0.33.0...v0.33.1) + * [Issue #1826](https://github.com/openziti/ziti/issues/1826) - 0.33.+ routers can cause panic in pre-0.33 controller with new metric + +# Release 0.33.0 + +## What's New + +* SDK Terminator stability improvements +* Minor feature updates and bug fixes + +## SDK Terminator stability improvements + +This release was focused on creating a chaos test for SDK terminators, running it and fixing any issues found. +The test repeatedly and randomly restarts the controller, routers and tunnelers then verifies that terminators +end up in the correct state. + +The following tools were also used/added to aid in diagnosing and fixing issues: + +* `ziti fabric validate router-sdk-terminators` + * Compares the controller state with the router state +* `ziti fabric validate terminators` + * Checks each selected terminator to ensure it's still valid on the router and/or sdk +* `ziti fabric inspect sdk-terminators` + * Allows inspecting each routers terminator state +* `ziti fabric inspect router-messaging` + * Allows inspecting what the controller has queued for router state sync and terminator validations +* `ziti edge validate service-hosting` + * Shows how many terminators each identity which can host a service has + +Several changes were made to the terminator code to ensure that terminators are properly created and cleaned up. +The routers now use an adaptive rate limiter to control how fast they send terminator related requests to the +controller. For this to work properly, the rate limiting on the controller must be enabled, so it can report +back to the routers when it's got too much work. + +## Component Updates and Bug Fixes + +* github.com/openziti/edge-api: [v0.26.10 -> v0.26.12](https://github.com/openziti/edge-api/compare/v0.26.10...v0.26.12) +* github.com/openziti/ziti: [v0.32.2 -> v0.33.0](https://github.com/openziti/ziti/compare/v0.32.2...v0.33.0) + * [Issue #1815](https://github.com/openziti/ziti/issues/1815) - Panic if api session sync failed handler is called twice in the router + * [Issue #1794](https://github.com/openziti/ziti/issues/1794) - Add SDK terminator chaos test and fix any bugs found as part of chaos testing + * [Issue #1781](https://github.com/openziti/ziti/issues/1781) - Improve performance when adding intercepted services + * [Issue #1369](https://github.com/openziti/ziti/issues/1369) - Allow filtering by policy type when listing identities for service or services for identity + * [Issue #1791](https://github.com/openziti/ziti/issues/1791) - route dial isn't checking for network timeouts correctly + * [Issue #1204](https://github.com/openziti/ziti/issues/1204) - ziti cli identity tags related flags misbehaving + * [Issue #987](https://github.com/openziti/ziti/issues/987) - "ziti create config router edge" doesn't know about --tunnelerMode proxy + * [Issue #652](https://github.com/openziti/ziti/issues/652) - Update CLI script M1 Support when github actions allows diff --git a/changelogs/CHANGELOG.0.34.md b/changelogs/CHANGELOG.0.34.md new file mode 100644 index 000000000..dcd67e575 --- /dev/null +++ b/changelogs/CHANGELOG.0.34.md @@ -0,0 +1,57 @@ +# Release 0.34.2 + +## What's New + +* The circuit id is now available in the SDK on the client and hosting side + * Requires 0.34.2+ routers + * Requests SDK support. Currently supported in the Go SDK 0.23.11+ +* Bug fixes + +## Component Updates and Bug Fixes +* github.com/openziti/edge-api: [v0.26.13 -> v0.26.14](https://github.com/openziti/edge-api/compare/v0.26.13...v0.26.14) +* github.com/openziti/sdk-golang: [v0.23.14 -> v0.23.15](https://github.com/openziti/sdk-golang/compare/v0.23.14...v0.23.15) +* github.com/openziti/secretstream: [v0.1.17 -> v0.1.18](https://github.com/openziti/secretstream/compare/v0.1.17...v0.1.18) + * [Issue #24](https://github.com/openziti/secretstream/issues/24) - Potential side channel issue + * [Issue #25](https://github.com/openziti/secretstream/issues/25) - Reads from crypto/rand not checked for errors + +* github.com/openziti/ziti: [v0.34.1 -> v0.34.2](https://github.com/openziti/ziti/compare/v0.34.1...v0.34.2) + * [Issue #1831](https://github.com/openziti/ziti/issues/1831) - Circuit ID should be returned in the response to a Dial request + * [Issue #1873](https://github.com/openziti/ziti/issues/1873) - host.v1 health check time.Duration unconvertible + +# Release 0.34.1 + +## What's New + +* Updates version of go to 1.22.x + * As usual when updating the go version, this is the only change in this release + +# Release 0.34.0 + +## What's New + +* Bug fixes and performance enhancements +* Version number is bumped as a large chunk of HA was merged up. The next version bump is likely to bring HA to alpha status. + +## Component Updates and Bug Fixes + +* github.com/openziti/channel/v2: [v2.0.119 -> v2.0.122](https://github.com/openziti/channel/compare/v2.0.119...v2.0.122) +* github.com/openziti/edge-api: [v0.26.12 -> v0.26.14](https://github.com/openziti/edge-api/compare/v0.26.12...v0.26.14) +* github.com/openziti/foundation/v2: [v2.0.37 -> v2.0.40](https://github.com/openziti/foundation/compare/v2.0.37...v2.0.40) +* github.com/openziti/identity: [v1.0.70 -> v1.0.73](https://github.com/openziti/identity/compare/v1.0.70...v1.0.73) +* github.com/openziti/metrics: [v1.2.45 -> v1.2.48](https://github.com/openziti/metrics/compare/v1.2.45...v1.2.48) +* github.com/openziti/runzmd: [v1.0.38 -> v1.0.41](https://github.com/openziti/runzmd/compare/v1.0.38...v1.0.41) +* github.com/openziti/sdk-golang: [v0.22.28 -> v0.23.14](https://github.com/openziti/sdk-golang/compare/v0.22.28...v0.23.14) + * [Issue #524](https://github.com/openziti/sdk-golang/issues/524) - Add circuit id to edge.Conn, so sdk connections can be correlated with network traffic + * [Issue #515](https://github.com/openziti/sdk-golang/issues/515) - Service hosting improvements + * [Issue #501](https://github.com/openziti/sdk-golang/issues/501) - Improve hosting session management + +* github.com/openziti/secretstream: [v0.1.16 -> v0.1.17](https://github.com/openziti/secretstream/compare/v0.1.16...v0.1.17) +* github.com/openziti/storage: [v0.2.30 -> v0.2.33](https://github.com/openziti/storage/compare/v0.2.30...v0.2.33) +* github.com/openziti/transport/v2: [v2.0.122 -> v2.0.125](https://github.com/openziti/transport/compare/v2.0.122...v2.0.125) +* github.com/openziti/ziti: [v0.33.1 -> v0.34.0](https://github.com/openziti/ziti/compare/v0.33.1...v0.34.0) + * [Issue #1858](https://github.com/openziti/ziti/issues/1858) - add option to create a generic env file instead of a BASH script + * [Issue #1428](https://github.com/openziti/ziti/issues/1428) - Investigate policy integrity performance + * [Issue #1854](https://github.com/openziti/ziti/issues/1854) - Controller can try to send unroute to router which has since disconnected, causing panic + * [Issue #1576](https://github.com/openziti/ziti/issues/1576) - Don't scan for posture checks if there are no posture checks + * [Issue #1849](https://github.com/openziti/ziti/issues/1849) - Session Sync shouldn't be able to block the control channel + * [Issue #1846](https://github.com/openziti/ziti/issues/1846) - Looking up api session certs for api sessions is inefficient diff --git a/common/ctrl_msg/messages.go b/common/ctrl_msg/messages.go index 00e0ff02b..f59754973 100644 --- a/common/ctrl_msg/messages.go +++ b/common/ctrl_msg/messages.go @@ -51,6 +51,7 @@ const ( CreateCircuitReqSessionTokenHeader = 11 CreateCircuitReqFingerprintsHeader = 12 CreateCircuitReqTerminatorInstanceIdHeader = 13 + CreateCircuitReqApiSessionTokenHeader = 14 CreateCircuitRespCircuitId = 11 CreateCircuitRespAddress = 12 @@ -117,6 +118,7 @@ func (self *CreateCircuitRequest) GetPeerData() map[uint32][]byte { func (self *CreateCircuitRequest) ToMessage() *channel.Message { msg := channel.NewMessage(int32(edge_ctrl_pb.ContentType_CreateCircuitV2RequestType), nil) msg.PutStringHeader(CreateCircuitReqSessionTokenHeader, self.SessionToken) + msg.PutStringHeader(CreateCircuitReqApiSessionTokenHeader, self.ApiSessionToken) msg.PutStringSliceHeader(CreateCircuitReqFingerprintsHeader, self.Fingerprints) msg.PutStringHeader(CreateCircuitReqTerminatorInstanceIdHeader, self.TerminatorInstanceId) msg.PutU32ToBytesMapHeader(CreateCircuitPeerDataHeader, self.PeerData) @@ -129,6 +131,8 @@ func DecodeCreateCircuitRequest(m *channel.Message) (*CreateCircuitRequest, erro return nil, errors.New("no session token provided in create circuit request") } + apiSessionToken, _ := m.GetStringHeader(CreateCircuitReqApiSessionTokenHeader) + fingerprints, _, err := m.GetStringSliceHeader(CreateCircuitReqFingerprintsHeader) if err != nil { return nil, fmt.Errorf("unable to get create circuit request fingerprints (%w)", err) @@ -141,6 +145,7 @@ func DecodeCreateCircuitRequest(m *channel.Message) (*CreateCircuitRequest, erro } return &CreateCircuitRequest{ + ApiSessionToken: apiSessionToken, SessionToken: sessionToken, Fingerprints: fingerprints, TerminatorInstanceId: terminatorInstanceId, diff --git a/controller/api_impl/link_api_model.go b/controller/api_impl/link_api_model.go index 2059df50d..d8338813b 100644 --- a/controller/api_impl/link_api_model.go +++ b/controller/api_impl/link_api_model.go @@ -44,11 +44,7 @@ func (factory *LinkLinkFactoryIml) Links(entity LinkEntity) rest_model.Links { func MapLinkToRestModel(n *network.Network, _ api.RequestContext, link *network.Link) (*rest_model.LinkDetail, error) { iteration := int64(link.Iteration) staticCost := int64(link.StaticCost) - linkState := link.CurrentState() - linkStateStr := "" - if linkState != nil { - linkStateStr = linkState.Mode.String() - } + linkStateStr := link.CurrentState().Mode.String() down := link.IsDown() diff --git a/controller/api_impl/link_router.go b/controller/api_impl/link_router.go index cf2e6062c..7b9e4d42f 100644 --- a/controller/api_impl/link_router.go +++ b/controller/api_impl/link_router.go @@ -122,7 +122,7 @@ func (r *LinkRouter) Patch(n *network.Network, rc api.RequestContext, params lin if fields.IsUpdated("down") { l.SetDown(params.Link.Down) } - n.LinkChanged(l) + n.RerouteLink(l) return nil }) } diff --git a/controller/command/command.go b/controller/command/command.go index d90af365f..deda9ca69 100644 --- a/controller/command/command.go +++ b/controller/command/command.go @@ -20,6 +20,7 @@ import ( "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/v2" "github.com/openziti/foundation/v2/debugz" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/storage/boltz" "github.com/openziti/ziti/controller/change" "github.com/sirupsen/logrus" @@ -51,13 +52,13 @@ type Dispatcher interface { Dispatch(command Command) error IsLeaderOrLeaderless() bool GetPeers() map[string]channel.Channel - GetRateLimiter() RateLimiter + GetRateLimiter() rate.RateLimiter } // LocalDispatcher should be used when running a non-clustered system type LocalDispatcher struct { EncodeDecodeCommands bool - Limiter RateLimiter + Limiter rate.RateLimiter } func (self *LocalDispatcher) IsLeaderOrLeaderless() bool { @@ -68,7 +69,7 @@ func (self *LocalDispatcher) GetPeers() map[string]channel.Channel { return nil } -func (self *LocalDispatcher) GetRateLimiter() RateLimiter { +func (self *LocalDispatcher) GetRateLimiter() rate.RateLimiter { return self.Limiter } diff --git a/controller/command/rate_limiter.go b/controller/command/rate_limiter.go index f62202c78..2fabe36e7 100644 --- a/controller/command/rate_limiter.go +++ b/controller/command/rate_limiter.go @@ -21,6 +21,7 @@ import ( "github.com/google/uuid" "github.com/michaelquigley/pfxlog" "github.com/openziti/foundation/v2/errorz" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/metrics" "github.com/openziti/ziti/controller/apierror" "github.com/pkg/errors" @@ -48,7 +49,7 @@ type RateLimiterConfig struct { QueueSize uint32 } -func NewRateLimiter(config RateLimiterConfig, registry metrics.Registry, closeNotify <-chan struct{}) RateLimiter { +func NewRateLimiter(config RateLimiterConfig, registry metrics.Registry, closeNotify <-chan struct{}) rate.RateLimiter { if !config.Enabled { return NoOpRateLimiter{} } @@ -77,45 +78,6 @@ func NewRateLimiter(config RateLimiterConfig, registry metrics.Registry, closeNo return result } -// A RateLimiter allows running arbitrary, sequential operations with a limiter, so that only N operations -// can be queued to run at any given time. If the system is too busy, the rate limiter will return -// an ApiError indicating that the server is too busy -type RateLimiter interface { - RunRateLimited(func() error) error - GetQueueFillPct() float64 -} - -// An AdaptiveRateLimiter allows running arbitrary, sequential operations with a limiter, so that only N operations -// can be queued to run at any given time. If the system is too busy, the rate limiter will return -// an ApiError indicating that the server is too busy. -// -// The rate limiter returns a RateLimitControl, allow the calling code to indicate if the operation finished in -// time. If operations are timing out before the results are available, the rate limiter should allow fewer -// operations in, as they will likely time out before the results can be used. -// -// The rate limiter doesn't have a set queue size, it has a window which can grow and shrink. When -// a timeout is signaled, using the RateLimitControl, it shrinks the window based on queue position -// of the timed out operation. For example, if an operation was queued at position 200, but the times -// out, we assume that we need to limit the queue size to something less than 200 for now. -// -// The limiter will also reject already queued operations if the window size changes and the operation -// was queued at a position larger than the current window size. -// -// The window size will slowly grow back towards the max as successes are noted in the RateLimitControl. -type AdaptiveRateLimiter interface { - RunRateLimited(f func() error) (RateLimitControl, error) -} - -// An AdaptiveRateLimitTracker works similarly to an AdaptiveRateLimiter, except it just manages the rate -// limiting without actually running the work. Because it doesn't run the work itself, it has to account -// for the possibility that some work may never report as complete or failed. It thus has a configurable -// timeout at which point outstanding work will be marked as failed. -type AdaptiveRateLimitTracker interface { - RunRateLimited(label string) (RateLimitControl, error) - RunRateLimitedF(label string, f func(control RateLimitControl) error) error - IsRateLimited() bool -} - type NoOpRateLimiter struct{} func (self NoOpRateLimiter) RunRateLimited(f func() error) error { @@ -128,18 +90,18 @@ func (self NoOpRateLimiter) GetQueueFillPct() float64 { type NoOpAdaptiveRateLimiter struct{} -func (self NoOpAdaptiveRateLimiter) RunRateLimited(f func() error) (RateLimitControl, error) { - return noOpRateLimitControl{}, f() +func (self NoOpAdaptiveRateLimiter) RunRateLimited(f func() error) (rate.RateLimitControl, error) { + return rate.NoOpRateLimitControl(), f() } type NoOpAdaptiveRateLimitTracker struct{} -func (n NoOpAdaptiveRateLimitTracker) RunRateLimited(string) (RateLimitControl, error) { - return noOpRateLimitControl{}, nil +func (n NoOpAdaptiveRateLimitTracker) RunRateLimited(string) (rate.RateLimitControl, error) { + return rate.NoOpRateLimitControl(), nil } -func (n NoOpAdaptiveRateLimitTracker) RunRateLimitedF(_ string, f func(control RateLimitControl) error) error { - return f(noOpRateLimitControl{}) +func (n NoOpAdaptiveRateLimitTracker) RunRateLimitedF(_ string, f func(control rate.RateLimitControl) error) error { + return f(rate.NoOpRateLimitControl()) } func (n NoOpAdaptiveRateLimitTracker) IsRateLimited() bool { @@ -278,7 +240,7 @@ func LoadAdaptiveRateLimiterConfig(cfg *AdaptiveRateLimiterConfig, cfgmap map[in return nil } -func NewAdaptiveRateLimiter(config AdaptiveRateLimiterConfig, registry metrics.Registry, closeNotify <-chan struct{}) AdaptiveRateLimiter { +func NewAdaptiveRateLimiter(config AdaptiveRateLimiterConfig, registry metrics.Registry, closeNotify <-chan struct{}) rate.AdaptiveRateLimiter { if !config.Enabled { return NoOpAdaptiveRateLimiter{} } @@ -366,7 +328,7 @@ func (self *adaptiveRateLimiter) backoff(queuePosition int32) { } } -func (self *adaptiveRateLimiter) RunRateLimited(f func() error) (RateLimitControl, error) { +func (self *adaptiveRateLimiter) RunRateLimited(f func() error) (rate.RateLimitControl, error) { work := &adaptiveRateLimitedWork{ wrapped: f, result: make(chan error, 1), @@ -381,7 +343,7 @@ func (self *adaptiveRateLimiter) RunRateLimited(f func() error) (RateLimitContro self.lock.Unlock() if !hasRoom { - return noOpRateLimitControl{}, apierror.NewTooManyUpdatesError() + return rate.NoOpRateLimitControl(), apierror.NewTooManyUpdatesError() } work.queuePosition = queuePosition @@ -394,12 +356,12 @@ func (self *adaptiveRateLimiter) RunRateLimited(f func() error) (RateLimitContro case result := <-work.result: return rateLimitControl{limiter: self, queuePosition: work.queuePosition}, result case <-self.closeNotify: - return noOpRateLimitControl{}, errors.New("rate limiter shutting down") + return rate.NoOpRateLimitControl(), errors.New("rate limiter shutting down") } case <-self.closeNotify: - return noOpRateLimitControl{}, errors.New("rate limiter shutting down") + return rate.NoOpRateLimitControl(), errors.New("rate limiter shutting down") default: - return noOpRateLimitControl{}, apierror.NewTooManyUpdatesError() + return rate.NoOpRateLimitControl(), apierror.NewTooManyUpdatesError() } } @@ -431,17 +393,6 @@ func (self *adaptiveRateLimiter) run() { } } -type RateLimitControl interface { - // Success indicates the operation was a success - Success() - - // Backoff indicates that we need to backoff - Backoff() - - // Failed indicates the operation was not a success, but a backoff isn't required - Failed() -} - type rateLimitControl struct { limiter *adaptiveRateLimiter queuePosition int32 @@ -459,18 +410,6 @@ func (r rateLimitControl) Failed() { // no-op for this type } -func NoOpRateLimitControl() RateLimitControl { - return noOpRateLimitControl{} -} - -type noOpRateLimitControl struct{} - -func (noOpRateLimitControl) Success() {} - -func (noOpRateLimitControl) Backoff() {} - -func (noOpRateLimitControl) Failed() {} - func WasRateLimited(err error) bool { var apiErr *errorz.ApiError if errors.As(err, &apiErr) { @@ -479,7 +418,7 @@ func WasRateLimited(err error) bool { return false } -func NewAdaptiveRateLimitTracker(config AdaptiveRateLimiterConfig, registry metrics.Registry, closeNotify <-chan struct{}) AdaptiveRateLimitTracker { +func NewAdaptiveRateLimitTracker(config AdaptiveRateLimiterConfig, registry metrics.Registry, closeNotify <-chan struct{}) rate.AdaptiveRateLimitTracker { if !config.Enabled { return NoOpAdaptiveRateLimitTracker{} } @@ -580,13 +519,13 @@ func (self *adaptiveRateLimitTracker) complete(work *adaptiveRateLimitTrackerWor delete(self.outstandingWork, work.id) } -func (self *adaptiveRateLimitTracker) RunRateLimited(label string) (RateLimitControl, error) { +func (self *adaptiveRateLimitTracker) RunRateLimited(label string) (rate.RateLimitControl, error) { self.lock.Lock() defer self.lock.Unlock() queuePosition := self.currentSize.Add(1) if queuePosition > self.currentWindow.Load() { self.currentSize.Add(-1) - return noOpRateLimitControl{}, apierror.NewTooManyUpdatesError() + return rate.NoOpRateLimitControl(), apierror.NewTooManyUpdatesError() } work := &adaptiveRateLimitTrackerWork{ @@ -602,7 +541,7 @@ func (self *adaptiveRateLimitTracker) RunRateLimited(label string) (RateLimitCon return work, nil } -func (self *adaptiveRateLimitTracker) RunRateLimitedF(label string, f func(control RateLimitControl) error) error { +func (self *adaptiveRateLimitTracker) RunRateLimitedF(label string, f func(control rate.RateLimitControl) error) error { ctrl, err := self.RunRateLimited(label) if err != nil { return err @@ -642,7 +581,7 @@ func (self *adaptiveRateLimitTracker) cleanExpired() { for _, work := range toRemove { pfxlog.Logger().WithField("label", work.label). WithField("duration", time.Since(work.createTime)). - Error("rate limit work expired") + Info("rate limit work expired") work.Backoff() } } @@ -660,7 +599,7 @@ func (self *adaptiveRateLimitTrackerWork) Success() { if self.completed.CompareAndSwap(false, true) { pfxlog.Logger().WithField("label", self.label). WithField("duration", time.Since(self.createTime)). - Info("success") + Debug("success") self.limiter.success(self) } } @@ -669,7 +608,7 @@ func (self *adaptiveRateLimitTrackerWork) Backoff() { if self.completed.CompareAndSwap(false, true) { pfxlog.Logger().WithField("label", self.label). WithField("duration", time.Since(self.createTime)). - Info("backoff") + Debug("backoff") self.limiter.backoff(self) } } @@ -678,7 +617,7 @@ func (self *adaptiveRateLimitTrackerWork) Failed() { if self.completed.CompareAndSwap(false, true) { pfxlog.Logger().WithField("label", self.label). WithField("duration", time.Since(self.createTime)). - Info("failed") + Debug("failed") self.limiter.complete(self) } } diff --git a/controller/config.go b/controller/config.go index 62ca4bec5..3a14c1325 100644 --- a/controller/config.go +++ b/controller/config.go @@ -27,6 +27,7 @@ import ( "github.com/openziti/identity" "github.com/openziti/storage/boltz" "github.com/openziti/transport/v2" + transporttls "github.com/openziti/transport/v2/tls" "github.com/openziti/ziti/common/config" "github.com/openziti/ziti/common/pb/ctrl_pb" "github.com/openziti/ziti/common/pb/mgmt_pb" @@ -50,6 +51,29 @@ const ( DefaultHealthChecksBoltCheckInitialDelay = 30 * time.Second DefaultRaftCommandHandlerMaxQueueSize = 1000 + + // DefaultTlsHandshakeRateLimiterEnabled is whether the tls handshake rate limiter is enabled by default + DefaultTlsHandshakeRateLimiterEnabled = false + + // TlsHandshakeRateLimiterMinSizeValue is the minimum size that can be configured for the tls handshake rate limiter + // window range + TlsHandshakeRateLimiterMinSizeValue = 5 + + // TlsHandshakeRateLimiterMaxSizeValue is the maximum size that can be configured for the tls handshake rate limiter + // window range + TlsHandshakeRateLimiterMaxSizeValue = 10000 + + // TlsHandshakeRateLimiterMetricOutstandingCount is the name of the metric tracking how many tasks are in process + TlsHandshakeRateLimiterMetricOutstandingCount = "tls_handshake_limiter.in_process" + + // TlsHandshakeRateLimiterMetricCurrentWindowSize is the name of the metric tracking the current window size + TlsHandshakeRateLimiterMetricCurrentWindowSize = "tls_handshake_limiter.window_size" + + // TlsHandshakeRateLimiterMetricWorkTimer is the name of the metric tracking how long successful tasks are taking to complete + TlsHandshakeRateLimiterMetricWorkTimer = "tls_handshake_limiter.work_timer" + + // DefaultTlsHandshakeRateLimiterMaxWindow is the default max size for the tls handshake rate limiter + DefaultTlsHandshakeRateLimiterMaxWindow = 1000 ) type Config struct { @@ -80,8 +104,9 @@ type Config struct { InitialDelay time.Duration } } - CommandRateLimiter command.RateLimiterConfig - src map[interface{}]interface{} + CommandRateLimiter command.RateLimiterConfig + TlsHandshakeRateLimiter command.AdaptiveRateLimiterConfig + src map[interface{}]interface{} } // CtrlOptions extends channel.Options to include support for additional, non-channel specific options @@ -488,9 +513,62 @@ func LoadConfig(path string) (*Config, error) { } } + controllerConfig.TlsHandshakeRateLimiter.SetDefaults() + controllerConfig.TlsHandshakeRateLimiter.Enabled = DefaultTlsHandshakeRateLimiterEnabled + controllerConfig.TlsHandshakeRateLimiter.MaxSize = DefaultTlsHandshakeRateLimiterMaxWindow + controllerConfig.TlsHandshakeRateLimiter.QueueSizeMetric = TlsHandshakeRateLimiterMetricOutstandingCount + controllerConfig.TlsHandshakeRateLimiter.WindowSizeMetric = TlsHandshakeRateLimiterMetricCurrentWindowSize + controllerConfig.TlsHandshakeRateLimiter.WorkTimerMetric = TlsHandshakeRateLimiterMetricWorkTimer + + if value, found := cfgmap["tls"]; found { + if tlsMap, ok := value.(map[interface{}]interface{}); ok { + if value, found := tlsMap["handshakeTimeout"]; found { + if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { + transporttls.SetSharedListenerHandshakeTimeout(val) + } else { + return nil, errors.Wrapf(err, "failed to parse tls.handshakeTimeout value '%v", value) + } + } + if err = loadTlsHandshakeRateLimiterConfig(&controllerConfig.TlsHandshakeRateLimiter, tlsMap); err != nil { + return nil, err + } + } + } + return controllerConfig, nil } +func loadTlsHandshakeRateLimiterConfig(rateLimitConfig *command.AdaptiveRateLimiterConfig, cfgmap map[interface{}]interface{}) error { + if value, found := cfgmap["rateLimiter"]; found { + if submap, ok := value.(map[interface{}]interface{}); ok { + if err := command.LoadAdaptiveRateLimiterConfig(rateLimitConfig, submap); err != nil { + return err + } + if rateLimitConfig.MaxSize < TlsHandshakeRateLimiterMinSizeValue { + return errors.Errorf("invalid value %v for tls.rateLimiter.maxSize, must be at least %v", + rateLimitConfig.MaxSize, TlsHandshakeRateLimiterMinSizeValue) + } + if rateLimitConfig.MaxSize > TlsHandshakeRateLimiterMaxSizeValue { + return errors.Errorf("invalid value %v for tls.rateLimiter.maxSize, must be at most %v", + rateLimitConfig.MaxSize, TlsHandshakeRateLimiterMaxSizeValue) + } + + if rateLimitConfig.MinSize < TlsHandshakeRateLimiterMinSizeValue { + return errors.Errorf("invalid value %v for tls.rateLimiter.minSize, must be at least %v", + rateLimitConfig.MinSize, TlsHandshakeRateLimiterMinSizeValue) + } + if rateLimitConfig.MinSize > TlsHandshakeRateLimiterMaxSizeValue { + return errors.Errorf("invalid value %v for tls.rateLimiter.minSize, must be at most %v", + rateLimitConfig.MinSize, TlsHandshakeRateLimiterMaxSizeValue) + } + } else { + return errors.Errorf("invalid type for tls.rateLimiter, should be map instead of %T", value) + } + } + + return nil +} + // verifyNewListenerInServerCert verifies that the hostname (ip/dns) for addr is present as an IP/DNS SAN in the first // certificate provided in the controller's identity server certificates. This is to avoid scenarios where // newListener propagated to routers who will never be able to verify the controller's certificates due to SAN issues. diff --git a/controller/controller.go b/controller/controller.go index 06fcf39f1..fdfbf1a52 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "github.com/openziti/transport/v2" + "github.com/openziti/transport/v2/tls" "github.com/openziti/ziti/common/capabilities" "github.com/openziti/ziti/common/config" "github.com/openziti/ziti/controller/event" @@ -202,6 +203,9 @@ func NewController(cfg *Config, versionProvider versions.VersionProvider) (*Cont shutdownC := make(chan struct{}) + tlsHandshakeRateLimiter := command.NewAdaptiveRateLimitTracker(cfg.TlsHandshakeRateLimiter, metricRegistry, shutdownC) + tls.SetSharedListenerRateLimiter(tlsHandshakeRateLimiter) + log := pfxlog.Logger() c := &Controller{ @@ -230,6 +234,8 @@ func NewController(cfg *Config, versionProvider versions.VersionProvider) (*Cont return nil, err } + c.initWeb() // need to init web before bootstrapping, so we can provide our endpoints to peers + if c.raftController != nil { if err := c.raftController.Bootstrap(); err != nil { log.WithError(err).Panic("error bootstrapping raft") @@ -258,8 +264,6 @@ func NewController(cfg *Config, versionProvider versions.VersionProvider) (*Cont return nil, err } - c.initWeb() - return c, nil } diff --git a/controller/db/stores.go b/controller/db/stores.go index 837c1c3a4..1d48dbd4d 100644 --- a/controller/db/stores.go +++ b/controller/db/stores.go @@ -21,10 +21,10 @@ import ( "errors" "github.com/michaelquigley/pfxlog" "github.com/openziti/foundation/v2/errorz" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/storage/ast" "github.com/openziti/storage/boltz" "github.com/openziti/ziti/controller/change" - "github.com/openziti/ziti/controller/command" "go.etcd.io/bbolt" "go4.org/sort" "reflect" @@ -223,7 +223,7 @@ type stores struct { apiSessionCertificate *ApiSessionCertificateStoreImpl mfa *MfaStoreImpl - rateLimiter command.RateLimiter + rateLimiter rate.RateLimiter } type DbProvider interface { @@ -236,7 +236,7 @@ func (f DbProviderF) GetDb() boltz.Db { return f() } -func InitStores(db boltz.Db, rateLimiter command.RateLimiter) (*Stores, error) { +func InitStores(db boltz.Db, rateLimiter rate.RateLimiter) (*Stores, error) { dbProvider := DbProviderF(func() boltz.Db { return db }) diff --git a/controller/env/appenv.go b/controller/env/appenv.go index cec23bb1e..cc1fcd93a 100644 --- a/controller/env/appenv.go +++ b/controller/env/appenv.go @@ -39,6 +39,7 @@ import ( managementOperations "github.com/openziti/edge-api/rest_management_api_server/operations" "github.com/openziti/edge-api/rest_model" "github.com/openziti/foundation/v2/errorz" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/foundation/v2/stringz" "github.com/openziti/identity" "github.com/openziti/metrics" @@ -102,7 +103,7 @@ type AppEnv struct { identityRefreshMeter metrics.Meter StartupTime time.Time InstanceId string - AuthRateLimiter command.AdaptiveRateLimiter + AuthRateLimiter rate.AdaptiveRateLimiter serverSigner jwtsigner.Signer ServerCert *tls.Certificate diff --git a/controller/handler_ctrl/fault.go b/controller/handler_ctrl/fault.go index ab040c946..b3f2a5b34 100644 --- a/controller/handler_ctrl/fault.go +++ b/controller/handler_ctrl/fault.go @@ -122,7 +122,9 @@ func (h *faultHandler) handleFaultedLink(log *logrus.Entry, fault *ctrl_pb.Fault wasConnected := link.IsUsable() if err := h.network.LinkFaulted(link, fault.Subject == ctrl_pb.FaultSubject_LinkDuplicate); err == nil { - h.network.LinkChanged(link) + if wasConnected { + h.network.RerouteLink(link) + } otherRouter := link.Src if link.Src.Id == h.r.Id { otherRouter = link.GetDest() diff --git a/controller/internal/routes/authenticate_router.go b/controller/internal/routes/authenticate_router.go index 3ce97c675..fa0fe0e4d 100644 --- a/controller/internal/routes/authenticate_router.go +++ b/controller/internal/routes/authenticate_router.go @@ -26,9 +26,9 @@ import ( "github.com/openziti/edge-api/rest_model" "github.com/openziti/foundation/v2/concurrenz" "github.com/openziti/foundation/v2/errorz" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/metrics" "github.com/openziti/ziti/controller/apierror" - "github.com/openziti/ziti/controller/command" "github.com/openziti/ziti/controller/env" "github.com/openziti/ziti/controller/internal/permissions" "github.com/openziti/ziti/controller/model" @@ -187,12 +187,12 @@ func (ro *AuthRouter) authHandler(ae *env.AppEnv, rc *response.RequestContext, h allowAdminBypass := identity.IsAdmin && time.Since(lastAdminAuth) > 10*time.Second && ro.lastAdminAuth.CompareAndSwap(lastAdminAuth, time.Now()) - var ctrl command.RateLimitControl + var ctrl rate.RateLimitControl if allowAdminBypass { var sessionId string sessionId, err = ae.Managers.ApiSession.Create(changeCtx.NewMutateContext(), newApiSession, sessionCerts) sessionIdHolder.Store(sessionId) - ctrl = command.NoOpRateLimitControl() + ctrl = rate.NoOpRateLimitControl() } else { ctrl, err = ae.AuthRateLimiter.RunRateLimited(func() error { sessionId, err := ae.Managers.ApiSession.Create(changeCtx.NewMutateContext(), newApiSession, sessionCerts) diff --git a/controller/model/controller_manager.go b/controller/model/controller_manager.go index 3c9c89fd4..27b458d7b 100644 --- a/controller/model/controller_manager.go +++ b/controller/model/controller_manager.go @@ -290,6 +290,7 @@ func (self *ControllerManager) PeersConnected(peers []*event.ClusterPeer) { CtrlAddress: peer.Addr, IsOnline: true, LastJoinedAt: &now, + ApiAddresses: map[string][]ApiAddress{}, } for apiKey, instances := range peer.ApiAddresses { diff --git a/controller/model/edge_service_manager.go b/controller/model/edge_service_manager.go index 4c16b76d7..2acd96590 100644 --- a/controller/model/edge_service_manager.go +++ b/controller/model/edge_service_manager.go @@ -119,34 +119,15 @@ func (self *EdgeServiceManager) ReadForIdentity(id string, identityId string, co } func (self *EdgeServiceManager) ReadForIdentityInTx(tx *bbolt.Tx, id string, identityId string, configTypes map[string]struct{}) (*ServiceDetail, error) { + edgeServiceStore := self.env.GetStores().EdgeService identity, err := self.GetEnv().GetManagers().Identity.readInTx(tx, identityId) if err != nil { return nil, err } - - var service *ServiceDetail - - if identity.IsAdmin { - service, err = self.readInTx(tx, id) - if err == nil && service != nil { - service.Permissions = []string{db.PolicyTypeBindName, db.PolicyTypeDialName} - } - } else { - service, err = self.ReadForNonAdminIdentityInTx(tx, id, identityId) - } - if err == nil && len(configTypes) > 0 { - identityServiceConfigs := self.env.GetStores().Identity.LoadServiceConfigsByServiceAndType(tx, identityId, configTypes) - self.mergeConfigs(tx, configTypes, service, identityServiceConfigs) - } - return service, err -} - -func (self *EdgeServiceManager) ReadForNonAdminIdentityInTx(tx *bbolt.Tx, id string, identityId string) (*ServiceDetail, error) { - edgeServiceStore := self.env.GetStores().EdgeService isBindable := edgeServiceStore.IsBindableByIdentity(tx, id, identityId) isDialable := edgeServiceStore.IsDialableByIdentity(tx, id, identityId) - if !isBindable && !isDialable { + if !isBindable && !isDialable && !identity.IsAdmin { // admin can view services even if policies don't permit bind/dial { return nil, boltz.NewNotFoundError(self.GetStore().GetSingularEntityType(), "id", id) } @@ -163,7 +144,17 @@ func (self *EdgeServiceManager) ReadForNonAdminIdentityInTx(tx *bbolt.Tx, id str if isDialable { result.Permissions = append(result.Permissions, db.PolicyTypeDialName) } - return result, nil + if result.Permissions == nil { + // don't return results with no permissions, since some SDKs assume non-nil permissions + result.Permissions = []string{db.PolicyTypeInvalidName} + } + + if len(configTypes) > 0 { + identityServiceConfigs := self.env.GetStores().Identity.LoadServiceConfigsByServiceAndType(tx, identityId, configTypes) + self.mergeConfigs(tx, configTypes, result, identityServiceConfigs) + } + + return result, err } func (self *EdgeServiceManager) PublicQueryForIdentity(sessionIdentity *Identity, configTypes map[string]struct{}, query ast.Query) (*ServiceListResult, error) { @@ -259,14 +250,8 @@ func (result *ServiceListResult) collect(tx *bbolt.Tx, ids []string, queryMetaDa identityServiceConfigs := result.manager.env.GetStores().Identity.LoadServiceConfigsByServiceAndType(tx, result.identityId, result.configTypes) for _, key := range ids { - if !result.isAdmin && result.identityId != "" { - service, err = result.manager.ReadForNonAdminIdentityInTx(tx, key, result.identityId) - } else { - service, err = result.manager.readInTx(tx, key) - if service != nil && result.isAdmin { - service.Permissions = []string{db.PolicyTypeBindName, db.PolicyTypeDialName} - } - } + // service permissions for admin & non-admin identities will be set according to policies + service, err = result.manager.ReadForIdentityInTx(tx, key, result.identityId, result.configTypes) if err != nil { return err } diff --git a/controller/model/testing.go b/controller/model/testing.go index 7ac47f2ea..d2dcf95e5 100644 --- a/controller/model/testing.go +++ b/controller/model/testing.go @@ -383,29 +383,3 @@ func (self *testConfig) GetVersionProvider() versions.VersionProvider { func (self *testConfig) GetCloseNotify() <-chan struct{} { return self.closeNotify } - -// -//type testDbProvider struct { -// ctx *TestContext -//} -// -//func (p *testDbProvider) GetDb() boltz.Db { -// return p.ctx.GetDb() -//} -// -//func (p *testDbProvider) GetStores() *Stores { -// return p.ctx.n.GetStores() -//} -// -//func (p *testDbProvider) GetServiceCache() network.Cache { -// return p -//} -// -//func (p *testDbProvider) NotifyRouterRenamed(_, _ string) {} -// -//func (p *testDbProvider) RemoveFromCache(_ string) { -//} -// -//func (p *testDbProvider) GetManagers() *network.Managers { -// return p.ctx.n.Managers -//} diff --git a/controller/network/assembly.go b/controller/network/assembly.go index 0a8bf5213..6093bfe9e 100644 --- a/controller/network/assembly.go +++ b/controller/network/assembly.go @@ -129,10 +129,10 @@ func (network *Network) clean() { failedLinks = append(failedLinks, duplicateLinks...) now := info.NowInMilliseconds() - lRemove := make(map[string]*Link) + var lRemove []*Link for _, l := range failedLinks { if now-l.CurrentState().Timestamp >= 30000 { - lRemove[l.Id] = l + lRemove = append(lRemove, l) } } for _, lr := range lRemove { diff --git a/controller/network/circuit.go b/controller/network/circuit.go index b87eebdea..9362b29dd 100644 --- a/controller/network/circuit.go +++ b/controller/network/circuit.go @@ -143,10 +143,10 @@ func (self *circuitController) get(id string) (*Circuit, bool) { } func (self *circuitController) all() []*Circuit { - circuits := make([]*Circuit, 0) - for tuple := range self.circuits.IterBuffered() { - circuits = append(circuits, tuple.Val) - } + var circuits []*Circuit + self.circuits.IterCb(func(_ string, circuit *Circuit) { + circuits = append(circuits, circuit) + }) return circuits } diff --git a/controller/network/link.go b/controller/network/link.go index db56db3dd..b6c5becb9 100644 --- a/controller/network/link.go +++ b/controller/network/link.go @@ -18,7 +18,6 @@ package network import ( "github.com/openziti/foundation/v2/concurrenz" - "github.com/openziti/foundation/v2/info" "sync" "sync/atomic" "time" @@ -35,7 +34,7 @@ type Link struct { Dst concurrenz.AtomicValue[*Router] Protocol string DialAddress string - state []*LinkState + state LinkState down bool StaticCost int32 usable atomic.Bool @@ -47,14 +46,17 @@ func newLink(id string, linkProtocol string, dialAddress string, initialLatency Id: id, Protocol: linkProtocol, DialAddress: dialAddress, - state: make([]*LinkState, 0), - down: false, - StaticCost: 1, - SrcLatency: initialLatency.Nanoseconds(), - DstLatency: initialLatency.Nanoseconds(), + state: LinkState{ + Mode: Pending, + Timestamp: time.Now().UnixMilli(), + }, + down: false, + StaticCost: 1, + SrcLatency: initialLatency.Nanoseconds(), + DstLatency: initialLatency.Nanoseconds(), } l.recalculateCost() - l.addState(&LinkState{Mode: Pending, Timestamp: info.NowInMilliseconds()}) + l.recalculateUsable() return l } @@ -66,23 +68,18 @@ func (link *Link) GetDest() *Router { return link.Dst.Load() } -func (link *Link) CurrentState() *LinkState { +func (link *Link) CurrentState() LinkState { link.lock.Lock() defer link.lock.Unlock() - if link.state == nil || len(link.state) < 1 { - return nil - } - return link.state[0] + return link.state } -func (link *Link) addState(s *LinkState) { +func (link *Link) SetState(m LinkMode) { link.lock.Lock() defer link.lock.Unlock() - if link.state == nil { - link.state = make([]*LinkState, 0) - } - link.state = append([]*LinkState{s}, link.state...) + link.state.Mode = m + link.state.Timestamp = time.Now().UnixMilli() link.recalculateUsable() } @@ -102,7 +99,7 @@ func (link *Link) IsDown() bool { func (link *Link) recalculateUsable() { if link.down { link.usable.Store(false) - } else if len(link.state) < 1 || link.state[0].Mode != Connected { + } else if link.state.Mode != Connected { link.usable.Store(false) } else { link.usable.Store(true) @@ -176,10 +173,3 @@ type LinkState struct { Mode LinkMode Timestamp int64 } - -func newLinkState(mode LinkMode) *LinkState { - return &LinkState{ - Mode: mode, - Timestamp: info.NowInMilliseconds(), - } -} diff --git a/controller/network/link_controller.go b/controller/network/link_controller.go index 500bb1a21..4d52d76ba 100644 --- a/controller/network/link_controller.go +++ b/controller/network/link_controller.go @@ -90,11 +90,8 @@ func newLinkController(options *Options) *linkController { return &val }) result.store.AddStringSymbol("state", func(entity *Link) *string { - if state := entity.CurrentState(); state != nil { - val := state.Mode.String() - return &val - } - return nil + val := entity.CurrentState().Mode.String() + return &val }) result.store.AddInt64Symbol("iteration", func(entity *Link) *int64 { val := int64(entity.Iteration) @@ -105,13 +102,12 @@ func newLinkController(options *Options) *linkController { } func (linkController *linkController) buildRouterLinks(router *Router) { - for entry := range linkController.linkTable.links.IterBuffered() { - link := entry.Val + linkController.linkTable.links.IterCb(func(_ string, link *Link) { if link.DstId == router.Id { router.routerLinks.Add(link, link.Src.Id) link.Dst.Store(router) } - } + }) } func (linkController *linkController) add(link *Link) { @@ -127,11 +123,15 @@ func (linkController *linkController) has(link *Link) bool { } func (linkController *linkController) scanForDeadLinks() { - for entry := range linkController.linkTable.links.IterBuffered() { - link := entry.Val + var toRemove []*Link + linkController.linkTable.links.IterCb(func(_ string, link *Link) { if !link.Src.Connected.Load() { - linkController.remove(link) + toRemove = append(toRemove, link) } + }) + + for _, link := range toRemove { + linkController.remove(link) } } @@ -161,7 +161,7 @@ func (linkController *linkController) routerReportedLink(linkId string, iteratio link.Src = src link.Dst.Store(dst) link.DstId = dstId - link.addState(newLinkState(Connected)) + link.SetState(Connected) linkController.add(link) return link, true } @@ -245,7 +245,7 @@ func (linkController *linkController) missingLinks(routers []*Router, pendingTim missingLinks := make([]*Link, 0) for _, srcR := range routers { - if srcR.HasCapability(ctrl_pb.RouterCapability_LinkManagement) { + if srcR.SupportsRouterLinkMgmt() { continue } @@ -273,7 +273,7 @@ func (linkController *linkController) clearExpiredPending(pendingTimeout time.Du toRemove := linkController.linkTable.matching(func(link *Link) bool { state := link.CurrentState() - return state != nil && state.Mode == Pending && state.Timestamp < pendingLimit + return state.Mode == Pending && state.Timestamp < pendingLimit }) for _, link := range toRemove { @@ -289,7 +289,7 @@ func (linkController *linkController) hasDirectedLink(a, b *Router, linkProtocol links := a.routerLinks.GetLinks() for _, link := range links { state := link.CurrentState() - if link.Src == a && link.DstId == b.Id && state != nil && link.Protocol == linkProtocol { + if link.Src == a && link.DstId == b.Id && link.Protocol == linkProtocol { if state.Mode == Connected || (state.Mode == Pending && state.Timestamp > pendingLimit) { return true } @@ -333,9 +333,9 @@ func (self *linkController) ValidateRouterLinks(n *Network, router *Router, cb L linkMap := map[string]*Link{} - for entry := range self.linkTable.links.IterBuffered() { - linkMap[entry.Key] = entry.Val - } + self.linkTable.links.IterCb(func(key string, link *Link) { + linkMap[key] = link + }) result := &mgmt_pb.RouterLinkDetails{ RouterId: router.Id, @@ -429,30 +429,29 @@ func (lt *linkTable) has(link *Link) bool { func (lt *linkTable) all() []*Link { links := make([]*Link, 0, lt.links.Count()) - for tuple := range lt.links.IterBuffered() { - links = append(links, tuple.Val) - } + lt.links.IterCb(func(_ string, link *Link) { + links = append(links, link) + }) return links } func (lt *linkTable) allInMode(mode LinkMode) []*Link { links := make([]*Link, 0) - for tuple := range lt.links.IterBuffered() { - link := tuple.Val + lt.links.IterCb(func(_ string, link *Link) { if link.CurrentState().Mode == mode { links = append(links, link) } - } + }) return links } func (lt *linkTable) matching(f func(*Link) bool) []*Link { var links []*Link - for tuple := range lt.links.IterBuffered() { - if f(tuple.Val) { - links = append(links, tuple.Val) + lt.links.IterCb(func(key string, link *Link) { + if f(link) { + links = append(links, link) } - } + }) return links } diff --git a/controller/network/link_controller_test.go b/controller/network/link_controller_test.go index 2a91e91a5..b257af6b6 100644 --- a/controller/network/link_controller_test.go +++ b/controller/network/link_controller_test.go @@ -77,7 +77,7 @@ func TestNeighbors(t *testing.T) { r0 := newRouterForTest("r0", "", nil, nil, 0, true) r1 := newRouterForTest("r1", "", nil, nil, 0, true) l0 := newTestLink("l0", r0, r1) - l0.addState(newLinkState(Connected)) + l0.SetState(Connected) linkController.add(l0) neighbors := linkController.connectedNeighborsOfRouter(r0) diff --git a/controller/network/network.go b/controller/network/network.go index f6755a2ce..942b09605 100644 --- a/controller/network/network.go +++ b/controller/network/network.go @@ -77,9 +77,8 @@ type Network struct { *Managers nodeId string options *Options - routerChanged chan *Router + assembleAndCleanC chan struct{} linkController *linkController - linkChanged chan *Link forwardingFaults chan struct{} circuitController *circuitController routeSenderController *routeSenderController @@ -125,9 +124,8 @@ func NewNetwork(config Config) (*Network, error) { network := &Network{ nodeId: config.GetId().Token, options: config.GetOptions(), - routerChanged: make(chan *Router, 16), + assembleAndCleanC: make(chan struct{}, 1), linkController: newLinkController(config.GetOptions()), - linkChanged: make(chan *Link, 16), forwardingFaults: make(chan struct{}, 1), circuitController: newCircuitController(), routeSenderController: newRouteSenderController(), @@ -325,7 +323,7 @@ func (network *Network) ConnectRouter(r *Router) { network.linkController.buildRouterLinks(r) network.Routers.markConnected(r) - time.AfterFunc(250*time.Millisecond, func() { network.routerChanged <- r }) + time.AfterFunc(250*time.Millisecond, network.notifyAssembleAndClean) for _, h := range network.routerPresenceHandlers { go h.RouterConnected(r) @@ -408,10 +406,13 @@ func (n *Network) ValidateRouterSdkTerminators(filter string, cb SdkTerminatorVa func (network *Network) DisconnectRouter(r *Router) { // 1: remove Links for Router for _, l := range r.routerLinks.GetLinks() { + wasConnected := l.CurrentState().Mode == Connected if l.Src.Id == r.Id { network.linkController.remove(l) } - network.LinkChanged(l) + if wasConnected { + network.RerouteLink(l) + } } // 2: remove Router network.Routers.markDisconnected(r) @@ -420,7 +421,14 @@ func (network *Network) DisconnectRouter(r *Router) { h.RouterDisconnected(r) } - network.routerChanged <- r + network.notifyAssembleAndClean() +} + +func (network *Network) notifyAssembleAndClean() { + select { + case network.assembleAndCleanC <- struct{}{}: + default: + } } func (network *Network) NotifyExistingLink(id string, iteration uint32, linkProtocol, dialAddress string, srcRouter *Router, dstRouterId string) { @@ -458,11 +466,11 @@ func (network *Network) NotifyExistingLink(id string, iteration uint32, linkProt func (network *Network) LinkConnected(msg *ctrl_pb.LinkConnected) error { if l, found := network.linkController.get(msg.Id); found { - if state := l.CurrentState(); state != nil && state.Mode != Pending { + if state := l.CurrentState(); state.Mode != Pending { return errors.Errorf("link [l/%v] state is %v, not pending, cannot mark connected", msg.Id, state.Mode) } - l.addState(newLinkState(Connected)) + l.SetState(Connected) network.NotifyLinkConnected(l, msg) return nil } @@ -470,7 +478,7 @@ func (network *Network) LinkConnected(msg *ctrl_pb.LinkConnected) error { } func (network *Network) LinkFaulted(l *Link, dupe bool) error { - l.addState(newLinkState(Failed)) + l.SetState(Failed) if dupe { network.NotifyLinkEvent(l, event.LinkDuplicate) } else { @@ -501,10 +509,10 @@ func (network *Network) VerifyRouter(routerId string, fingerprints []string) err return errors.Errorf("could not verify fingerprint for router %v", routerId) } -func (network *Network) LinkChanged(l *Link) { +func (network *Network) RerouteLink(l *Link) { // This is called from Channel.rxer() and thus may not block go func() { - network.linkChanged <- l + network.handleRerouteLink(l) }() } @@ -916,20 +924,19 @@ func (network *Network) Run() { go network.watchdog() + ticker := time.NewTicker(time.Duration(network.options.CycleSeconds) * time.Second) + defer ticker.Stop() + for { select { - case r := <-network.routerChanged: - logrus.WithField("routerId", r.Id).Info("changed router") + case <-network.assembleAndCleanC: network.assemble() network.clean() - case l := <-network.linkChanged: - go network.handleLinkChanged(l) - case <-network.forwardingFaults: network.clean() - case <-time.After(time.Duration(network.options.CycleSeconds) * time.Second): + case <-ticker.C: network.assemble() network.clean() network.smart() @@ -976,7 +983,7 @@ func (network *Network) watchdog() { } } -func (network *Network) handleLinkChanged(l *Link) { +func (network *Network) handleRerouteLink(l *Link) { log := logrus.WithField("linkId", l.Id) log.Info("changed link") if err := network.rerouteLink(l, time.Now().Add(DefaultOptionsRouteTimeout)); err != nil { @@ -1042,7 +1049,7 @@ func (network *Network) RemoveLink(linkId string) { if link != nil { network.linkController.remove(link) - network.linkChanged <- link + network.RerouteLink(link) } } diff --git a/controller/network/path_test.go b/controller/network/path_test.go index 678428c22..144e1b419 100644 --- a/controller/network/path_test.go +++ b/controller/network/path_test.go @@ -51,7 +51,7 @@ func TestSimplePath2(t *testing.T) { network.Routers.markConnected(r1) l0 := newTestLink("l0", r0, r1) - l0.addState(newLinkState(Connected)) + l0.SetState(Connected) network.linkController.add(l0) path, err := network.CreatePath(r0, r1) @@ -115,11 +115,11 @@ func TestTransitPath2(t *testing.T) { network.Routers.markConnected(r2) l0 := newTestLink("l0", r0, r1) - l0.addState(newLinkState(Connected)) + l0.SetState(Connected) network.linkController.add(l0) l1 := newTestLink("l1", r1, r2) - l1.addState(newLinkState(Connected)) + l1.SetState(Connected) network.linkController.add(l1) path, err := network.CreatePath(r0, r2) @@ -218,28 +218,28 @@ func TestShortestPath(t *testing.T) { link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l1", r0, r2) link.SetStaticCost(5) link.SetDstLatency(15 * 1_000_000) link.SetSrcLatency(16 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l2", r1, r3) link.SetStaticCost(9) link.SetDstLatency(20 * 1_000_000) link.SetSrcLatency(21 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l3", r2, r3) link.SetStaticCost(13) link.SetDstLatency(25 * 1_000_000) link.SetSrcLatency(26 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) path, cost, err := network.shortestPath(r0, r3) @@ -286,28 +286,28 @@ func TestShortestPathWithUntraversableRouter(t *testing.T) { link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l1", r0, r2) link.SetStaticCost(5) link.SetDstLatency(15 * 1_000_000) link.SetSrcLatency(16 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l2", r1, r3) link.SetStaticCost(9) link.SetDstLatency(20 * 1_000_000) link.SetSrcLatency(21 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l3", r2, r3) link.SetStaticCost(13) link.SetDstLatency(25 * 1_000_000) link.SetSrcLatency(26 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) path, cost, err := network.shortestPath(r0, r3) @@ -348,7 +348,7 @@ func TestShortestPathWithOnlyUntraversableRouter(t *testing.T) { link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) path, cost, err := network.shortestPath(r0, r1) @@ -388,7 +388,7 @@ func TestShortestPathWithUntraversableEdgeRouters(t *testing.T) { link.SetStaticCost(3) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) path, cost, err := network.shortestPath(r0, r1) @@ -431,14 +431,14 @@ func TestShortestPathWithUntraversableEdgeRoutersAndTraversableMiddle(t *testing link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l1", r1, r2) link.SetStaticCost(3) link.SetDstLatency(12 * 1_000_000) link.SetSrcLatency(15 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) path, cost, err := network.shortestPath(r0, r2) @@ -483,14 +483,14 @@ func TestShortestPathWithUntraversableEdgeRoutersAndUntraversableMiddle(t *testi link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) link = newTestLink("l2", r1, r2) link.SetStaticCost(2) link.SetDstLatency(10 * 1_000_000) link.SetSrcLatency(11 * 1_000_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) path, cost, err := network.shortestPath(r0, r2) @@ -619,7 +619,7 @@ func newPathTestLink(network *Network, id string, srcR, destR *Router) *Link { l.SrcLatency = 0 l.DstLatency = 0 l.recalculateCost() - l.addState(newLinkState(Connected)) + l.SetState(Connected) network.linkController.add(l) return l } diff --git a/controller/network/route_perf_test.go b/controller/network/route_perf_test.go index c28daf0ef..7f24b2eeb 100644 --- a/controller/network/route_perf_test.go +++ b/controller/network/route_perf_test.go @@ -62,7 +62,7 @@ func TestShortestPathAgainstEstablished(t *testing.T) { link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) linkIdx++ } @@ -186,7 +186,7 @@ func BenchmarkShortestPathPerfWithRouterChanges(b *testing.B) { link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) linkIdx++ } @@ -278,7 +278,7 @@ func BenchmarkShortestPathPerf(b *testing.B) { link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) linkIdx++ } @@ -350,7 +350,7 @@ func BenchmarkMoreRealisticShortestPathPerf(b *testing.B) { link.SetStaticCost(int32(nextCost())) link.SetDstLatency(nextCost() * 100_000) link.SetSrcLatency(nextCost() * 100_000) - link.addState(newLinkState(Connected)) + link.SetState(Connected) network.linkController.add(link) linkIdx++ } diff --git a/controller/network/router.go b/controller/network/router.go index b6e9cd62e..e0d71695f 100644 --- a/controller/network/router.go +++ b/controller/network/router.go @@ -31,6 +31,7 @@ import ( "github.com/openziti/ziti/controller/fields" "github.com/openziti/ziti/controller/xt" "google.golang.org/protobuf/proto" + "maps" "reflect" "strings" "sync" @@ -106,6 +107,14 @@ func (entity *Router) HasCapability(capability ctrl_pb.RouterCapability) bool { return entity.Metadata != nil && genext.Contains(entity.Metadata.Capabilities, capability) } +func (entity *Router) SupportsRouterLinkMgmt() bool { + if entity.VersionInfo == nil { + return true + } + supportsLinkMgmt, err := entity.VersionInfo.HasMinimumVersion("0.32.1") + return err != nil || supportsLinkMgmt +} + func NewRouter(id, name, fingerprint string, cost uint16, noTraversal bool) *Router { if name == "" { name = id @@ -184,9 +193,9 @@ func (self *RouterManager) getConnected(id string) *Router { func (self *RouterManager) allConnected() []*Router { var routers []*Router - for v := range self.connected.IterBuffered() { - routers = append(routers, v.Val) - } + self.connected.IterCb(func(_ string, router *Router) { + routers = append(routers, router) + }) return routers } @@ -593,9 +602,7 @@ func (self *RouterLinks) Add(link *Link, otherRouterId string) { byRouter := self.GetLinksByRouter() newLinksByRouter := map[string][]*Link{} - for k, v := range byRouter { - newLinksByRouter[k] = v - } + maps.Copy(newLinksByRouter, byRouter) forRouterList := newLinksByRouter[otherRouterId] newForRouterList := append([]*Link{link}, forRouterList...) newLinksByRouter[otherRouterId] = newForRouterList @@ -606,7 +613,7 @@ func (self *RouterLinks) Remove(link *Link, otherRouterId string) { self.Lock() defer self.Unlock() links := self.GetLinks() - newLinks := make([]*Link, 0, len(links)+1) + newLinks := make([]*Link, 0, len(links)) for _, l := range links { if link != l { newLinks = append(newLinks, l) @@ -616,9 +623,7 @@ func (self *RouterLinks) Remove(link *Link, otherRouterId string) { byRouter := self.GetLinksByRouter() newLinksByRouter := map[string][]*Link{} - for k, v := range byRouter { - newLinksByRouter[k] = v - } + maps.Copy(newLinksByRouter, byRouter) forRouterList := newLinksByRouter[otherRouterId] var newForRouterList []*Link for _, l := range forRouterList { @@ -633,7 +638,6 @@ func (self *RouterLinks) Remove(link *Link, otherRouterId string) { } self.linkByRouter.Store(newLinksByRouter) - } func (self *RouterLinks) Clear() { diff --git a/controller/network/router_messaging.go b/controller/network/router_messaging.go index 3cfd75966..f6c362940 100644 --- a/controller/network/router_messaging.go +++ b/controller/network/router_messaging.go @@ -188,7 +188,12 @@ func (self *RouterMessaging) syncStates() { } if v.sendInProgress.Load() { + continue + } + if !notifyRouter.SupportsRouterLinkMgmt() { + // if the router doesn't support router based link mgmt, don't send these messages + delete(self.routerUpdates, k) continue } @@ -277,6 +282,12 @@ func (self *RouterMessaging) sendTerminatorValidationRequest(routerId string, up } } + supportsVerifyV2, err := notifyRouter.VersionInfo.HasMinimumVersion("0.31.1") + if err != nil { + pfxlog.Logger().WithError(err).Error(`version check of "0.31.1" failed`) + supportsVerifyV2 = true + } + var terminators []*ctrl_pb.Terminator for _, terminator := range updates.terminators { @@ -295,9 +306,16 @@ func (self *RouterMessaging) sendTerminatorValidationRequest(routerId string, up return } - req := &ctrl_pb.ValidateTerminatorsV2Request{ - Terminators: terminators, - FixInvalid: true, + var req protobufs.TypedMessage + if !supportsVerifyV2 { + req = &ctrl_pb.ValidateTerminatorsRequest{ + Terminators: terminators, + } + } else { + req = &ctrl_pb.ValidateTerminatorsV2Request{ + Terminators: terminators, + FixInvalid: true, + } } queueErr := self.routerCommPool.QueueOrError(func() { @@ -309,6 +327,11 @@ func (self *RouterMessaging) sendTerminatorValidationRequest(routerId string, up if self.managers.Dispatcher.IsLeaderOrLeaderless() { if err := protobufs.MarshalTyped(req).WithTimeout(time.Second * 1).SendAndWaitForWire(ch); err != nil { pfxlog.Logger().WithError(err).WithField("routerId", notifyRouter.Id).Error("failed to send validate terminators request to router") + } else if !supportsVerifyV2 { + // V1 doesn't send responses, it will just send deletes if the terminator is invalid. + // we're going to mark these ok. If they're not, we should get a delete message. Older + // routers can still fail to delete, if the delete gets lost for some reason. + self.generateMockResponseForV1(notifyRouter, updates) } } }) @@ -320,6 +343,25 @@ func (self *RouterMessaging) sendTerminatorValidationRequest(routerId string, up } } +func (self *RouterMessaging) generateMockResponseForV1(r *Router, validations *terminatorValidations) { + handler := &terminatorValidationRespReceived{ + router: r, + changeCtx: change.New(), // won't be used since we're marking things valid + resp: &ctrl_pb.ValidateTerminatorsV2Response{ + States: map[string]*ctrl_pb.RouterTerminatorState{}, + }, + } + + for id, t := range validations.terminators { + handler.resp.States[id] = &ctrl_pb.RouterTerminatorState{ + Valid: true, + Marker: t.marker, + } + } + + self.queueEvent(handler) +} + func (self *RouterMessaging) NewValidationResponseHandler(n *Network, r *Router) channel.ReceiveHandlerF { return func(m *channel.Message, ch channel.Channel) { log := pfxlog.Logger().WithField("routerId", r.Id) diff --git a/controller/network/util_test.go b/controller/network/util_test.go index c7cf493c5..d0c63c28d 100644 --- a/controller/network/util_test.go +++ b/controller/network/util_test.go @@ -22,9 +22,9 @@ import ( "github.com/openziti/ziti/controller/models" "github.com/openziti/ziti/controller/xt_smartrouting" - "github.com/openziti/ziti/controller/db" "github.com/openziti/transport/v2" "github.com/openziti/transport/v2/tcp" + "github.com/openziti/ziti/controller/db" ) func newTestEntityHelper(ctx *db.TestContext, network *Network) *testEntityHelper { @@ -88,8 +88,7 @@ func (self *testEntityHelper) addTestService(serviceName string) *Service { func (self *testEntityHelper) discardControllerEvents() { for { select { - case <-self.network.routerChanged: - case <-self.network.linkChanged: + case <-self.network.assembleAndCleanC: default: return } diff --git a/controller/oidc_auth/storage.go b/controller/oidc_auth/storage.go index 7494e617d..9844fab82 100644 --- a/controller/oidc_auth/storage.go +++ b/controller/oidc_auth/storage.go @@ -359,12 +359,17 @@ func (s *HybridStorage) SaveAuthCode(_ context.Context, id string, code string) // DeleteAuthRequest implements the op.Storage interface func (s *HybridStorage) DeleteAuthRequest(_ context.Context, id string) error { - s.authRequests.Remove(id) - for entry := range s.codes.IterBuffered() { - if entry.Val == id { - s.codes.Remove(entry.Key) + + var toRemove []string + s.codes.IterCb(func(key string, v string) { + if v == id { + toRemove = append(toRemove, key) } + }) + + for _, mapKey := range toRemove { + s.codes.Remove(mapKey) } return nil diff --git a/controller/raft/raft.go b/controller/raft/raft.go index 6a07beb8a..edafcbaa9 100644 --- a/controller/raft/raft.go +++ b/controller/raft/raft.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/hashicorp/go-hclog" "github.com/openziti/foundation/v2/concurrenz" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/foundation/v2/versions" "github.com/openziti/transport/v2" "github.com/openziti/ziti/common/pb/cmd_pb" @@ -228,7 +229,7 @@ type Controller struct { clusterStateChangeHandlers concurrenz.CopyOnWriteSlice[func(event ClusterEvent, state ClusterState)] isLeader atomic.Bool clusterEvents chan raft.Observation - commandRateLimiter command.RateLimiter + commandRateLimiter rate.RateLimiter } func (self *Controller) RegisterClusterEventHandler(f func(event ClusterEvent, state ClusterState)) { @@ -248,7 +249,7 @@ func (self *Controller) GetMesh() mesh.Mesh { return self.Mesh } -func (self *Controller) GetRateLimiter() command.RateLimiter { +func (self *Controller) GetRateLimiter() rate.RateLimiter { return self.commandRateLimiter } diff --git a/controller/sync_strats/rtx.go b/controller/sync_strats/rtx.go index 4fe0be8da..9074bf887 100644 --- a/controller/sync_strats/rtx.go +++ b/controller/sync_strats/rtx.go @@ -160,7 +160,11 @@ func (m *routerTxMap) Remove(r *network.Router) { // Range creates a snapshot of the rtx's to loop over and will execute callback // with each rtx. func (m *routerTxMap) Range(callback func(entries *RouterSender)) { - for entry := range m.internalMap.IterBuffered() { - callback(entry.Val) + var routerSenders []*RouterSender + m.internalMap.IterCb(func(_ string, v *RouterSender) { + routerSenders = append(routerSenders, v) + }) + for _, routerSender := range routerSenders { + callback(routerSender) } } diff --git a/controller/sync_strats/sync_instant.go b/controller/sync_strats/sync_instant.go index a9eae870b..8620cff4e 100644 --- a/controller/sync_strats/sync_instant.go +++ b/controller/sync_strats/sync_instant.go @@ -716,12 +716,17 @@ func (strategy *InstantStrategy) synchronize(rtx *RouterSender) { } } - for tuple := range strategy.RouterDataModel.PublicKeys.IterBuffered() { + var pks []*edge_ctrl_pb.DataState_PublicKey + strategy.RouterDataModel.PublicKeys.IterCb(func(_ string, v *edge_ctrl_pb.DataState_PublicKey) { + pks = append(pks, v) + }) + + for _, pk := range pks { peerEvent := &edge_ctrl_pb.DataState_Event{ IsSynthetic: true, Action: edge_ctrl_pb.DataState_Create, Model: &edge_ctrl_pb.DataState_Event_PublicKey{ - PublicKey: newPublicKey(tuple.Val.Data, tuple.Val.Format, tuple.Val.Usages), + PublicKey: newPublicKey(pk.Data, pk.Format, pk.Usages), }, } diff --git a/controller/xt/failure.go b/controller/xt/failure.go index f1c26503f..8e31a6dc2 100644 --- a/controller/xt/failure.go +++ b/controller/xt/failure.go @@ -27,9 +27,14 @@ func (self *failureCosts) CreditOverTime(credit uint8, period time.Duration) *ti ticker := time.NewTicker(period) go func() { for range ticker.C { - for val := range self.costMap.IterBuffered() { - actualCredit := self.successWithCredit(val.Key, uint16(credit)) - GlobalCosts().UpdateDynamicCost(val.Key, func(u uint16) uint16 { + var keys []string + self.costMap.IterCb(func(key string, _ uint16) { + keys = append(keys, key) + }) + + for _, key := range keys { + actualCredit := self.successWithCredit(key, uint16(credit)) + GlobalCosts().UpdateDynamicCost(key, func(u uint16) uint16 { if u < actualCredit { return 0 } diff --git a/doc/ha/ctrl1.yml b/doc/ha/ctrl1.yml index e51cf3cbd..939c5b79f 100644 --- a/doc/ha/ctrl1.yml +++ b/doc/ha/ctrl1.yml @@ -51,8 +51,3 @@ web: - binding: edge-management - binding: edge-client - binding: edge-oidc - options: - redirectURIs: - - "http://localhost:*/auth/callback" - - "http://127.0.0.1:*/auth/callback" - - "https://oauth.pstmn.io/v1/callback" \ No newline at end of file diff --git a/doc/ha/ctrl2.yml b/doc/ha/ctrl2.yml index 173e29948..ffce4c134 100644 --- a/doc/ha/ctrl2.yml +++ b/doc/ha/ctrl2.yml @@ -40,8 +40,3 @@ web: - binding: edge-management - binding: edge-client - binding: edge-oidc - options: - redirectURIs: - - "http://localhost:*/auth/callback" - - "http://127.0.0.1:*/auth/callback" - - "https://oauth.pstmn.io/v1/callback" \ No newline at end of file diff --git a/doc/ha/ctrl3.yml b/doc/ha/ctrl3.yml index e6248e583..6077096cf 100644 --- a/doc/ha/ctrl3.yml +++ b/doc/ha/ctrl3.yml @@ -40,8 +40,3 @@ web: - binding: edge-management - binding: edge-client - binding: edge-oidc - options: - redirectURIs: - - "http://localhost:*/auth/callback" - - "http://127.0.0.1:*/auth/callback" - - "https://oauth.pstmn.io/v1/callback" \ No newline at end of file diff --git a/go.mod b/go.mod index ef340ddd9..d698639e1 100644 --- a/go.mod +++ b/go.mod @@ -33,11 +33,11 @@ require ( github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.1 - github.com/hashicorp/go-hclog v1.6.2 + github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hashicorp/raft v1.6.1 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 - github.com/jedib0t/go-pretty/v6 v6.5.6 + github.com/jedib0t/go-pretty/v6 v6.5.8 github.com/jessevdk/go-flags v1.5.0 github.com/jinzhu/copier v0.4.0 github.com/kataras/go-events v0.0.3 @@ -48,17 +48,17 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/openziti/agent v1.0.16 - github.com/openziti/channel/v2 v2.0.122 - github.com/openziti/edge-api v0.26.14 - github.com/openziti/foundation/v2 v2.0.40 - github.com/openziti/identity v1.0.73 + github.com/openziti/channel/v2 v2.0.128 + github.com/openziti/edge-api v0.26.16 + github.com/openziti/foundation/v2 v2.0.42 + github.com/openziti/identity v1.0.75 github.com/openziti/jwks v1.0.3 - github.com/openziti/metrics v1.2.48 - github.com/openziti/runzmd v1.0.41 - github.com/openziti/sdk-golang v0.23.15 - github.com/openziti/secretstream v0.1.18 - github.com/openziti/storage v0.2.33 - github.com/openziti/transport/v2 v2.0.125 + github.com/openziti/metrics v1.2.51 + github.com/openziti/runzmd v1.0.43 + github.com/openziti/sdk-golang v0.23.19 + github.com/openziti/secretstream v0.1.19 + github.com/openziti/storage v0.2.36 + github.com/openziti/transport/v2 v2.0.131 github.com/openziti/x509-claims v1.0.3 github.com/openziti/xweb/v2 v2.1.0 github.com/openziti/ziti-db-explorer v1.1.3 @@ -79,13 +79,13 @@ require ( github.com/zitadel/oidc/v2 v2.12.0 go.etcd.io/bbolt v1.3.9 go4.org v0.0.0-20180809161055-417644f6feb5 - golang.org/x/crypto v0.21.0 - golang.org/x/net v0.22.0 - golang.org/x/sync v0.6.0 + golang.org/x/crypto v0.22.0 + golang.org/x/net v0.24.0 + golang.org/x/sync v0.7.0 golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 google.golang.org/protobuf v1.33.0 - gopkg.in/AlecAivazis/survey.v1 v1.8.7 + gopkg.in/AlecAivazis/survey.v1 v1.8.8 gopkg.in/resty.v1 v1.12.0 gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v2 v2.4.0 @@ -185,9 +185,9 @@ require ( golang.org/x/image v0.13.0 // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/tools v0.18.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - nhooyr.io/websocket v1.8.10 // indirect + nhooyr.io/websocket v1.8.11 // indirect ) diff --git a/go.sum b/go.sum index 5e76b131c..5f9a0fbd7 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,7 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/AlecAivazis/survey/v2 v2.0.5/go.mod h1:WYBhg6f0y/fNYUuesWQc0PKbJcEliGcYHB9sNT3Bg74= github.com/AppsFlyer/go-sundheit v0.5.0 h1:/VxpyigCfJrq1r97mn9HPiAB2qrhcTFHwNIIDr15CZM= github.com/AppsFlyer/go-sundheit v0.5.0/go.mod h1:2ZM0BnfqT/mljBQO224VbL5XH06TgWuQ6Cn+cTtCpTY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -357,8 +358,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= -github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -403,8 +404,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb-client-go/v2 v2.2.2/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/jedib0t/go-pretty/v6 v6.5.6 h1:nKXVLqPfAwY7sWcYXdNZZZ2fjqDpAtj9UeWupgfUxSg= -github.com/jedib0t/go-pretty/v6 v6.5.6/go.mod h1:5LQIxa52oJ/DlDSLv0HEkWOFMDGoWkJb9ss5KqPpJBg= +github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ= +github.com/jedib0t/go-pretty/v6 v6.5.8/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jeremija/gosubmit v0.2.7 h1:At0OhGCFGPXyjPYAsCchoBUhE099pcBXmsb4iZqROIc= github.com/jeremija/gosubmit v0.2.7/go.mod h1:Ui+HS073lCFREXBbdfrJzMB57OI/bdxTiLtrDHHhFPI= @@ -441,6 +442,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -566,30 +568,30 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openziti/agent v1.0.16 h1:9Saji+8hFE1NpzP2XzDhsVJbCrDlhixoLHfOpFt5Z+U= github.com/openziti/agent v1.0.16/go.mod h1:zfm53+PVWoGFzjGGgQdKby5749G6VRYHe+eQJmoVKy4= -github.com/openziti/channel/v2 v2.0.122 h1:MPALuLUuTsW2QJzaoRkrP/wGqhIeWtSsrnxt2Ept5SU= -github.com/openziti/channel/v2 v2.0.122/go.mod h1:n4r/pWTFYJy98hEtf4ATYVHU70OePjoM8JJzrIvXvZg= +github.com/openziti/channel/v2 v2.0.128 h1:kRigyUmrN1q56ydRmBWzruV0DvVntYx/f2DT2P0qSh4= +github.com/openziti/channel/v2 v2.0.128/go.mod h1:ugebO2eO2CdaVQE45o20iE4R9DK0VRoJsGBZt0x3eLc= github.com/openziti/dilithium v0.3.3 h1:PLgQ6PMNLSTzCFbX/h98cmudgz/cU6TmjdSv5NAPD8k= github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo= -github.com/openziti/edge-api v0.26.14 h1:BpxKavDVdykOY7GSdKXi7klv9Im4vw7bTMujz3wFE1Q= -github.com/openziti/edge-api v0.26.14/go.mod h1:aTRqB9J/6ISbw9+YWjzD0tvWAXY98m8yWcbiGNqAQTU= -github.com/openziti/foundation/v2 v2.0.40 h1:8oXL77egZjnQuKuxPTHiBOh+X/I2mSJefIbnLSv8rFI= -github.com/openziti/foundation/v2 v2.0.40/go.mod h1:38RikdtjvzDUALm3jT3PSOrU0bHCleRty6bHGdfFyUI= -github.com/openziti/identity v1.0.73 h1:F1iF/rDQYfDIv4ktqBYwK/0C5Ra/7y92Z5Umrw4c4fk= -github.com/openziti/identity v1.0.73/go.mod h1:SKcvyjTlSIpxJuhYLjayJgB800EJ/nO4Y/Lhh77NE9A= +github.com/openziti/edge-api v0.26.16 h1:oU4t2Q4f0+K/ypwdQIecgA836PJkCnDMe1glc56ovaA= +github.com/openziti/edge-api v0.26.16/go.mod h1:HTp6bC3cBKAoIwyA06aKKrrBwxFtJALfpEinmvCRgBc= +github.com/openziti/foundation/v2 v2.0.42 h1:cXZFql3xY92AHPCk/wqIoSUX9EHgegwPrs5bhFE2jNg= +github.com/openziti/foundation/v2 v2.0.42/go.mod h1:a+REYnK9bZ2cvmOiuBLp57MjQqn3U5cHhJcDMLMO8rE= +github.com/openziti/identity v1.0.75 h1:FDm3ZojmmGP9eoOxMvmF52RS6JzToTTeWB0pthIA+ks= +github.com/openziti/identity v1.0.75/go.mod h1:ouF+CYh5ywvvMr0Uy1+tbkILIPTdob5WZEnm3v0bPMQ= github.com/openziti/jwks v1.0.3 h1:hf8wkb+Cg4nH/HM0KROFd7u+C3DkRVcFZJ7tDV+4icc= github.com/openziti/jwks v1.0.3/go.mod h1:t4xxq8vlXGsPn29kiQVnZBBDDnEoOFqtJoHibkJunQQ= -github.com/openziti/metrics v1.2.48 h1:bDUcWSTWM4vXh4YU8z953Q/vtlUjtH+pSaQviKzB578= -github.com/openziti/metrics v1.2.48/go.mod h1:rLFoXigAFb5clCDqe/RlvFFjlaTr9Li7iEn3OxInmb0= -github.com/openziti/runzmd v1.0.41 h1:Iuwe91Lzq63ux2R6+VIWEvcEKhpKkPb1QVNwrJHOuDA= -github.com/openziti/runzmd v1.0.41/go.mod h1:T79oGLLJNnAhHzcfnlrOR39S2XGkzwYZLZh62xee0wk= -github.com/openziti/sdk-golang v0.23.15 h1:+h+zin2CB6nP8e2Sqi2Jxo4Sbo3wTp8PIfh7QNm4lPA= -github.com/openziti/sdk-golang v0.23.15/go.mod h1:aYvN+PCUV2AHPcbgcG1Khgso5TbmNaLMke0ByshU4Hs= -github.com/openziti/secretstream v0.1.18 h1:bMHI4ez6pVSGFzHCHHn6imkaCYqd3o9Oa8gDObf8icI= -github.com/openziti/secretstream v0.1.18/go.mod h1:gosi8ohW+dAHhz1h/MjZEcEVWO0jisWqWjEjmx7oAEs= -github.com/openziti/storage v0.2.33 h1:15WKxJ0J1GCNOMmOkoosjoxKN2Hrm4p3g8p9WAiMS9Y= -github.com/openziti/storage v0.2.33/go.mod h1:AbE5KkTVlXVFHRqpc7Ug3eFpNOc1HFBFwNIYn8Rtp+s= -github.com/openziti/transport/v2 v2.0.125 h1:6kDfabGFIE2+UPyvJVtfI43CNKcMwNOrCRX602CbDU4= -github.com/openziti/transport/v2 v2.0.125/go.mod h1:meVOsDPZSO/hNWSA+NGK5sAoeSgWoqfJgKUZlM3iX1E= +github.com/openziti/metrics v1.2.51 h1:+RfVdEGMOmhXBcC/gFIqXeNVnpWt9+cfrbCacFV/e3w= +github.com/openziti/metrics v1.2.51/go.mod h1:qvX8EKF8I42tA4KlNFX+RBx+ywNiVRvJTS4QofDU8KU= +github.com/openziti/runzmd v1.0.43 h1:jQpDQ91I+6kbAM/2QV5s2900/CWgs3HyPhrpCNRQE6E= +github.com/openziti/runzmd v1.0.43/go.mod h1:sTSSi4f3dcgO5w8PgSK4fGAlwR2ajexAlMcEtduuFaU= +github.com/openziti/sdk-golang v0.23.19 h1:jl0PWdsTIKY41Kzku6qAo/KEsIMjyrdKqCcIg+qAR8I= +github.com/openziti/sdk-golang v0.23.19/go.mod h1:mmxfoFAs6hMR1B76rnRSOCG35th6/YY9/Gp8zUeL9PI= +github.com/openziti/secretstream v0.1.19 h1:qSX23lyOI5ROvYPTjy7Lo6eFsq9qBBPzaG5hvgLlw48= +github.com/openziti/secretstream v0.1.19/go.mod h1:1n/Uk8PQEREwB7iQXhUzNqt+8jYEpUZOJDGcetMI08o= +github.com/openziti/storage v0.2.36 h1:EN1PlA0SnGB8mm4QvHgGtVj39OL6me5sgcSl7v2J7Mg= +github.com/openziti/storage v0.2.36/go.mod h1:1fIRgTNyzqoVuC/wAXqpL+QQra+t3MVHp//2XIeG6HM= +github.com/openziti/transport/v2 v2.0.131 h1:K1h3sl04QWQdYGSPSI1aZLufrL44Azw4WmdprzkIMxo= +github.com/openziti/transport/v2 v2.0.131/go.mod h1:N+OPeoqIuZwS72q0XnM4DLHVdKZ3ZOR37rqjOjWA0b8= github.com/openziti/x509-claims v1.0.3 h1:HNdQ8Nf1agB3lBs1gahcO6zfkeS4S5xoQ2/PkY4HRX0= github.com/openziti/x509-claims v1.0.3/go.mod h1:Z0WIpBm6c4ecrpRKrou6Gk2wrLWxJO/+tuUwKh8VewE= github.com/openziti/xweb/v2 v2.1.0 h1:Xhh3C2pZkq/Prr65V+SfFSibLDYteoc4f62KQCcTZF4= @@ -846,11 +848,11 @@ golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -862,8 +864,9 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -962,8 +965,9 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -994,9 +998,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1014,6 +1017,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1084,8 +1088,9 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1284,8 +1289,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/AlecAivazis/survey.v1 v1.8.7 h1:oBJqtgsyBLg9K5FK9twNUbcPnbCPoh+R9a+7nag3qJM= -gopkg.in/AlecAivazis/survey.v1 v1.8.7/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= +gopkg.in/AlecAivazis/survey.v1 v1.8.8 h1:5UtTowJZTz1j7NxVzDGKTz6Lm9IWm8DDF6b7a2wq9VY= +gopkg.in/AlecAivazis/survey.v1 v1.8.8/go.mod h1:CaHjv79TCgAvXMSFJSVgonHXYWxnhzI3eoHtnX5UgUo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1333,8 +1338,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q= -nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w= rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= diff --git a/router/env/env.go b/router/env/env.go index 0fbe75533..0e801b83e 100644 --- a/router/env/env.go +++ b/router/env/env.go @@ -19,10 +19,10 @@ package env import ( "github.com/openziti/channel/v2" "github.com/openziti/foundation/v2/goroutines" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/foundation/v2/versions" "github.com/openziti/identity" "github.com/openziti/metrics" - "github.com/openziti/ziti/controller/command" "github.com/openziti/ziti/router/xgress" "github.com/openziti/ziti/router/xlink" ) @@ -40,6 +40,6 @@ type RouterEnv interface { RenderJsonConfig() (string, error) GetHeartbeatOptions() HeartbeatOptions GetRateLimiterPool() goroutines.Pool - GetCtrlRateLimiter() command.AdaptiveRateLimitTracker + GetCtrlRateLimiter() rate.AdaptiveRateLimitTracker GetVersionInfo() versions.VersionProvider } diff --git a/router/handler_ctrl/validate_terminators_v2.go b/router/handler_ctrl/validate_terminators_v2.go index 4a0cd9021..4d71f5b43 100644 --- a/router/handler_ctrl/validate_terminators_v2.go +++ b/router/handler_ctrl/validate_terminators_v2.go @@ -131,8 +131,15 @@ func (handler *validateTerminatorsV2Handler) validateTerminators(msg *channel.Me func (handler *validateTerminatorsV2Handler) validateTerminator(dialer xgress.Dialer, terminator *ctrl_pb.Terminator, fixInvalid bool) *ctrl_pb.RouterTerminatorState { if inspectable, ok := dialer.(xgress.InspectableDialer); ok { valid, state := inspectable.InspectTerminator(terminator.Id, terminator.Address, fixInvalid) + if valid { + return &ctrl_pb.RouterTerminatorState{ + Valid: true, + Detail: state, + Marker: terminator.Marker, + } + } return &ctrl_pb.RouterTerminatorState{ - Valid: valid, + Valid: false, Detail: state, Reason: ctrl_pb.TerminatorInvalidReason_UnknownTerminator, Marker: terminator.Marker, diff --git a/router/router.go b/router/router.go index 6c9ebcaa4..059b0c0af 100644 --- a/router/router.go +++ b/router/router.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/ziti/controller/command" "io/fs" "os" @@ -83,7 +84,7 @@ type Router struct { xgressListeners []xgress.Listener linkDialerPool goroutines.Pool rateLimiterPool goroutines.Pool - ctrlRateLimiter command.AdaptiveRateLimitTracker + ctrlRateLimiter rate.AdaptiveRateLimitTracker metricsRegistry metrics.UsageRegistry shutdownC chan struct{} shutdownDoneC chan struct{} @@ -382,7 +383,7 @@ func (self *Router) GetRateLimiterPool() goroutines.Pool { return self.rateLimiterPool } -func (self *Router) GetCtrlRateLimiter() command.AdaptiveRateLimitTracker { +func (self *Router) GetCtrlRateLimiter() rate.AdaptiveRateLimitTracker { return self.ctrlRateLimiter } diff --git a/router/xgress_edge/accept.go b/router/xgress_edge/accept.go index d15ef96e7..5570f280e 100644 --- a/router/xgress_edge/accept.go +++ b/router/xgress_edge/accept.go @@ -30,7 +30,7 @@ type Acceptor struct { uListener channel.UnderlayListener listener *listener options *channel.Options - sessionBindHandler channel.BindHandler + sessionBindHandler *sessionConnectionHandler } func (self *Acceptor) BindChannel(binding channel.Binding) error { @@ -95,7 +95,7 @@ func (self *Acceptor) BindChannel(binding channel.Binding) error { binding.AddTypedReceiveHandler(proxy.msgMux) binding.AddCloseHandler(proxy) - return self.sessionBindHandler.BindChannel(binding) + return self.sessionBindHandler.BindChannel(binding, proxy) } func NewAcceptor(listener *listener, uListener channel.UnderlayListener, options *channel.Options) *Acceptor { diff --git a/router/xgress_edge/connections.go b/router/xgress_edge/connections.go index 2ff0ce46c..88684b808 100644 --- a/router/xgress_edge/connections.go +++ b/router/xgress_edge/connections.go @@ -45,7 +45,7 @@ func newSessionConnectHandler(stateManager state.Manager, options *Options, metr } } -func (handler *sessionConnectionHandler) BindChannel(binding channel.Binding) error { +func (handler *sessionConnectionHandler) BindChannel(binding channel.Binding, edgeConn *edgeClientConn) error { ch := binding.GetChannel() binding.AddCloseHandler(handler) @@ -86,6 +86,8 @@ func (handler *sessionConnectionHandler) BindChannel(binding channel.Binding) er return fmt.Errorf("no api session found for token [%s], fingerprints: [%v], subjects [%v]", token, fingerprints, subjects) } + edgeConn.apiSession = apiSession + if apiSession.Claims != nil { token = apiSession.Claims.ApiSessionId } diff --git a/router/xgress_edge/fabric.go b/router/xgress_edge/fabric.go index 861e6d657..d1a7362b1 100644 --- a/router/xgress_edge/fabric.go +++ b/router/xgress_edge/fabric.go @@ -21,9 +21,9 @@ import ( "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/v2" "github.com/openziti/foundation/v2/concurrenz" + "github.com/openziti/foundation/v2/rate" "github.com/openziti/sdk-golang/ziti/edge" "github.com/openziti/ziti/common/pb/edge_ctrl_pb" - "github.com/openziti/ziti/controller/command" "github.com/openziti/ziti/router/xgress" "github.com/openziti/ziti/router/xgress_common" "github.com/pkg/errors" @@ -98,7 +98,7 @@ type edgeTerminator struct { lastAttempt time.Time establishCallback func(result edge_ctrl_pb.CreateTerminatorResult) lock sync.Mutex - rateLimitCallback command.RateLimitControl + rateLimitCallback rate.RateLimitControl } func (self *edgeTerminator) replace(other *edgeTerminator) { @@ -273,13 +273,13 @@ func (self *edgeTerminator) newConnection(connId uint32) (*edgeXgressConn, error return result, nil } -func (self *edgeTerminator) SetRateLimitCallback(control command.RateLimitControl) { +func (self *edgeTerminator) SetRateLimitCallback(control rate.RateLimitControl) { self.lock.Lock() defer self.lock.Unlock() self.rateLimitCallback = control } -func (self *edgeTerminator) GetAndClearRateLimitCallback() command.RateLimitControl { +func (self *edgeTerminator) GetAndClearRateLimitCallback() rate.RateLimitControl { self.lock.Lock() defer self.lock.Unlock() result := self.rateLimitCallback diff --git a/router/xgress_edge/listener.go b/router/xgress_edge/listener.go index db0c71b0e..ea78bca23 100644 --- a/router/xgress_edge/listener.go +++ b/router/xgress_edge/listener.go @@ -109,6 +109,7 @@ type edgeClientConn struct { fingerprints cert.Fingerprints ch channel.Channel idSeq uint32 + apiSession *state.ApiSession } func (self *edgeClientConn) HandleClose(ch channel.Channel) { @@ -170,6 +171,7 @@ func (self *edgeClientConn) processConnect(manager state.Manager, req *channel.M terminatorIdentity, _ := req.GetStringHeader(edge.TerminatorIdentityHeader) request := &ctrl_msg.CreateCircuitRequest{ + ApiSessionToken: self.apiSession.Token, SessionToken: sessionToken, Fingerprints: self.fingerprints.Prints(), TerminatorInstanceId: terminatorIdentity, @@ -213,6 +215,7 @@ func (self *edgeClientConn) sendCreateCircuitRequest(req *ctrl_msg.CreateCircuit func (self *edgeClientConn) sendCreateCircuitRequestV1(req *ctrl_msg.CreateCircuitRequest, ctrlCh channel.Channel) (*ctrl_msg.CreateCircuitResponse, error) { request := &edge_ctrl_pb.CreateCircuitRequest{ SessionToken: req.SessionToken, + ApiSessionToken: req.ApiSessionToken, Fingerprints: req.Fingerprints, TerminatorInstanceId: req.TerminatorInstanceId, PeerData: req.PeerData, diff --git a/tests/service_test.go b/tests/service_test.go index c7e5c84b8..e84efdd5a 100644 --- a/tests/service_test.go +++ b/tests/service_test.go @@ -20,9 +20,9 @@ package tests import ( "fmt" - "github.com/openziti/ziti/common/eid" "github.com/openziti/foundation/v2/errorz" "github.com/openziti/foundation/v2/stringz" + "github.com/openziti/ziti/common/eid" "net/url" "sort" "testing" @@ -50,7 +50,7 @@ func Test_Services(t *testing.T) { ctx.testContextChanged(t) now := time.Now() service := ctx.AdminManagementSession.requireNewService(nil, nil) - service.permissions = []string{"Dial", "Bind"} + service.permissions = []string{"Invalid"} entityJson := ctx.AdminManagementSession.validateEntityWithQuery(service) ctx.validateDateFieldsForCreate(now, entityJson) }) @@ -58,11 +58,11 @@ func Test_Services(t *testing.T) { t.Run("list as admin should return 3 services", func(t *testing.T) { ctx.testContextChanged(t) service1 := ctx.AdminManagementSession.requireNewService(nil, nil) - service1.permissions = []string{"Dial", "Bind"} + service1.permissions = []string{"Invalid"} service2 := ctx.AdminManagementSession.requireNewService(nil, nil) - service2.permissions = []string{"Dial", "Bind"} + service2.permissions = []string{"Invalid"} service3 := ctx.AdminManagementSession.requireNewService(nil, nil) - service3.permissions = []string{"Dial", "Bind"} + service3.permissions = []string{"Invalid"} ctx.AdminManagementSession.validateEntityWithLookup(service1) ctx.AdminManagementSession.validateEntityWithQuery(service1) @@ -108,7 +108,7 @@ func Test_Services(t *testing.T) { t.Run("lookup as admin should pass", func(t *testing.T) { ctx.testContextChanged(t) service := ctx.AdminManagementSession.requireNewService(nil, nil) - service.permissions = []string{"Dial", "Bind"} + service.permissions = []string{"Invalid"} ctx.AdminManagementSession.validateEntityWithLookup(service) }) @@ -163,7 +163,7 @@ func Test_Services(t *testing.T) { ctx.testContextChanged(t) now := time.Now() service := ctx.AdminManagementSession.requireNewService(nil, nil) - service.permissions = []string{"Bind", "Dial"} + service.permissions = []string{"Invalid"} entityJson := ctx.AdminManagementSession.validateEntityWithQuery(service) createdAt := ctx.validateDateFieldsForCreate(now, entityJson) @@ -364,7 +364,7 @@ func Test_ServiceRoleAttributes(t *testing.T) { role1 := eid.New() role2 := eid.New() service := ctx.AdminManagementSession.requireNewService(s(role1, role2), nil) - service.permissions = []string{"Dial", "Bind"} + service.permissions = []string{"Invalid"} ctx.AdminManagementSession.validateEntityWithQuery(service) ctx.AdminManagementSession.validateEntityWithLookup(service) @@ -375,7 +375,7 @@ func Test_ServiceRoleAttributes(t *testing.T) { role1 := eid.New() role2 := eid.New() service := ctx.AdminManagementSession.requireNewService(s(role1, role2), nil) - service.permissions = []string{"Dial", "Bind"} + service.permissions = []string{"Invalid"} role3 := eid.New() service.roleAttributes = []string{role2, role3} @@ -423,7 +423,7 @@ func Test_ServiceRoleAttributes(t *testing.T) { now := time.Now() newService := ctx.AdminManagementSession.requireNewService(nil, nil) - newService.permissions = []string{"Dial", "Bind"} + newService.permissions = []string{"Invalid"} entityJson := ctx.AdminManagementSession.validateEntityWithQuery(newService) diff --git a/version b/version index 61d2f3576..d3827e75a 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.34 +1.0 diff --git a/ziti/cmd/cmd.go b/ziti/cmd/cmd.go index 64fd9fa1e..21a809753 100644 --- a/ziti/cmd/cmd.go +++ b/ziti/cmd/cmd.go @@ -31,7 +31,6 @@ import ( "github.com/openziti/ziti/ziti/cmd/demo" "github.com/openziti/ziti/ziti/cmd/edge" "github.com/openziti/ziti/ziti/cmd/fabric" - "github.com/openziti/ziti/ziti/cmd/install" "github.com/openziti/ziti/ziti/cmd/pki" "github.com/openziti/ziti/ziti/cmd/templates" c "github.com/openziti/ziti/ziti/constants" @@ -143,18 +142,7 @@ func NewCmdRoot(in io.Reader, out, err io.Writer, cmd *cobra.Command) *cobra.Com opsCommands.AddCommand(NewCmdLogFormat(out, err)) opsCommands.AddCommand(NewUnwrapIdentityFileCommand(out, err)) - installCommands := []*cobra.Command{ - install.NewCmdInstall(out, err), - install.NewCmdUpgrade(out, err), - install.NewCmdUse(out, err), - install.NewCmdVersion(out, err), - } - groups := templates.CommandGroups{ - { - Message: "Installing Ziti components:", - Commands: installCommands, - }, { Message: "Working with Ziti resources:", Commands: []*cobra.Command{ @@ -197,6 +185,7 @@ func NewCmdRoot(in io.Reader, out, err io.Writer, cmd *cobra.Command) *cobra.Com cmd.Version = version.GetVersion() cmd.SetVersionTemplate("{{printf .Version}}\n") cmd.AddCommand(NewCmdArt(out, err)) + cmd.AddCommand(common.NewVersionCmd()) return cmd } diff --git a/ziti/tunnel/version.go b/ziti/cmd/common/version.go similarity index 98% rename from ziti/tunnel/version.go rename to ziti/cmd/common/version.go index 947ac932a..a4abc8c55 100644 --- a/ziti/tunnel/version.go +++ b/ziti/cmd/common/version.go @@ -14,7 +14,7 @@ limitations under the License. */ -package tunnel +package common import ( "fmt" diff --git a/ziti/cmd/create/config_templates/controller.yml b/ziti/cmd/create/config_templates/controller.yml index 49f40f11f..8971d279a 100644 --- a/ziti/cmd/create/config_templates/controller.yml +++ b/ziti/cmd/create/config_templates/controller.yml @@ -223,8 +223,6 @@ web: # These settings are per API. The example below is for the 'edge-api' and contains both optional values and # required values. options: { } - - binding: edge-oidc - options: { } - binding: edge-client options: { } - binding: fabric diff --git a/ziti/cmd/fabric/add_identity.go b/ziti/cmd/fabric/add_identity.go deleted file mode 100644 index 90007f08e..000000000 --- a/ziti/cmd/fabric/add_identity.go +++ /dev/null @@ -1,80 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package fabric - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - "github.com/openziti/ziti/ziti/util" - "github.com/spf13/cobra" -) - -type addIdentityOptions struct { - common.CommonOptions - caCert string - clientCert string - clientKey string - readOnly bool -} - -func newAddIdentityCmd(p common.OptionsProvider) *cobra.Command { - options := &addIdentityOptions{ - CommonOptions: p(), - } - - cmd := &cobra.Command{ - Use: "add-identity ", - Short: "adds a fabric identity for connecting to the Ziti Controller", - Args: cobra.ExactArgs(1), - RunE: options.run, - } - - cmd.Flags().StringVarP(&common.CliIdentity, "cli-identity", "i", "", "Specify the saved identity you want the CLI to use when connect to the controller with") - cmd.Flags().StringVar(&options.caCert, "ca-cert", "", "additional root certificates used by the Ziti Controller") - cmd.Flags().StringVar(&options.clientCert, "client-cert", "", "client certificate used to authenticate to the Ziti Controller") - cmd.Flags().StringVar(&options.clientKey, "client-key", "", "client certificate key used to authenticate to the Ziti Controller") - cmd.Flags().BoolVar(&options.readOnly, "read-only", false, "marks this login as read-only. Note: this is not a guarantee that nothing can be changed on the server. Care should still be taken!") - - for _, required := range []string{"ca-cert", "client-cert", "client-key"} { - if err := cmd.MarkFlagRequired(required); err != nil { - panic(err) - } - } - - return cmd -} - -func (o *addIdentityOptions) run(_ *cobra.Command, args []string) error { - config, configFile, err := util.LoadRestClientConfig() - if err != nil { - return err - } - - id := config.GetIdentity() - - loginIdentity := &util.RestClientFabricIdentity{ - Url: args[0], - CaCert: o.caCert, - ClientCert: o.clientCert, - ClientKey: o.clientKey, - ReadOnly: o.readOnly, - } - - o.Printf("Saving identity '%v' to %v\n", id, configFile) - config.FabricIdentities[id] = loginIdentity - - return util.PersistRestClientConfig(config) -} diff --git a/ziti/cmd/fabric/remove_identity.go b/ziti/cmd/fabric/remove_identity.go deleted file mode 100644 index 9d4895ce4..000000000 --- a/ziti/cmd/fabric/remove_identity.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package fabric - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/util" - "github.com/spf13/cobra" -) - -// removeIdentityOptions are the flags for removeIdentity commands -type removeIdentityOptions struct { - common.CommonOptions -} - -// newRemoveIdentityCmd creates the command -func newRemoveIdentityCmd(p common.OptionsProvider) *cobra.Command { - options := &removeIdentityOptions{ - CommonOptions: p(), - } - - cmd := &cobra.Command{ - Use: "remove-identity", - Short: "remove an identity for a Ziti Controller instance", - Args: cobra.ExactArgs(0), - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - SuggestFor: []string{}, - } - - // allow interspersing positional args and flags - cmd.Flags().SetInterspersed(true) - cmd.Flags().StringVarP(&common.CliIdentity, "cli-identity", "i", "", "Specify the saved identity you want the CLI to use when connect to the controller with") - - return cmd -} - -// Run implements this command -func (o *removeIdentityOptions) Run() error { - config, configFile, err := util.LoadRestClientConfig() - if err != nil { - return err - } - - id := config.GetIdentity() - o.Printf("Removing fabric identity '%v' from %v\n", id, configFile) - delete(config.FabricIdentities, id) - return util.PersistRestClientConfig(config) -} diff --git a/ziti/cmd/fabric/root.go b/ziti/cmd/fabric/root.go index 8078ad537..14f32e159 100644 --- a/ziti/cmd/fabric/root.go +++ b/ziti/cmd/fabric/root.go @@ -31,7 +31,6 @@ import ( func NewFabricCmd(p common.OptionsProvider) *cobra.Command { fabricCmd := util.NewEmptyParentCmd("fabric", "Manage the Fabric components of a Ziti network using the Ziti Fabric REST and WebSocket APIs") - fabricCmd.AddCommand(newAddIdentityCmd(p), newRemoveIdentityCmd(p)) fabricCmd.AddCommand(newCreateCommand(p), newListCmd(p), newUpdateCommand(p), newDeleteCmd(p)) fabricCmd.AddCommand(newInspectCmd(p)) fabricCmd.AddCommand(newDbCmd(p)) diff --git a/ziti/cmd/install/common_install.go b/ziti/cmd/install/common_install.go deleted file mode 100644 index c67e91f3e..000000000 --- a/ziti/cmd/install/common_install.go +++ /dev/null @@ -1,240 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/blang/semver" - "github.com/openziti/ziti/common/getziti" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "github.com/openziti/ziti/ziti/util" - "gopkg.in/resty.v1" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" -) - -func isBinaryInstalled(d string) bool { - binDir, err := util.BinaryLocation() - if err == nil { - exists, err := util.FileExists(filepath.Join(binDir, d)) - if err == nil && exists { - return true - } - } - // log.Warnf("%s not found\n", d) - return false -} - -func (o *InstallOptions) deleteInstalledBinary(d string) bool { - binDir, err := util.BinaryLocation() - if err == nil { - exists, err := util.FileExists(filepath.Join(binDir, d)) - if err == nil && exists { - err = os.Remove(filepath.Join(binDir, d)) - if err != nil { - log.Warnf("Error attempting to delete %s: %s \n", d, err) - return false - } - return true - } - } - return false -} - -func (o *InstallOptions) shouldInstallBinary(binDir string, name string) (fileName string, download bool, err error) { - fileName = name - download = false - if runtime.GOOS == "windows" { - fileName += ".exe" - } - pgmPath, err := exec.LookPath(fileName) - if err == nil { - log.Warnf("%s is already available on your PATH at %s\n", util.ColorInfo(fileName), util.ColorInfo(pgmPath)) - return - } - - // lets see if its been installed but just is not on the PATH - exists, err := util.FileExists(filepath.Join(binDir, fileName)) - if err != nil { - return - } - if exists { - log.Warnf("Please add %s to your PATH\n", util.ColorInfo(binDir)) - return - } - download = true - return -} - -func (o *InstallOptions) getLatestZitiAppVersion(branch string, zitiApp string) (semver.Version, error) { - return util.GetLatestVersionFromArtifactory(o.Verbose, o.Staging, branch, zitiApp) -} - -func (o *InstallOptions) getLatestZitiAppVersionForBranch(branch string, zitiApp string) (semver.Version, error) { - return util.GetLatestVersionFromArtifactory(o.Verbose, o.Staging, branch, zitiApp) -} - -func (o *InstallOptions) getLatestTerraformProviderVersion(branch string, provider string) (semver.Version, error) { - return util.GetLatestTerraformProviderVersionFromArtifactory(branch, provider) -} - -func (o *InstallOptions) getLatestGitHubReleaseVersion(zitiApp string) (semver.Version, error) { - var result semver.Version - release, err := getziti.GetHighestVersionGitHubReleaseInfo(zitiApp, o.Verbose) - if release != nil { - result = release.SemVer - } - return result, err -} - -func (o *InstallOptions) getCurrentZitiSnapshotList() ([]string, error) { - children, err := util.GetCurrentSnapshotListFromArtifactory(o.Verbose) - - list := make([]string, 0) - list = append(list, "main") - - for _, v := range children { - str := strings.Replace(v.URI, "/", "", -1) - list = append(list, str) - } - - return list, err -} - -func (o *InstallOptions) installZitiApp(branch string, zitiApp string, upgrade bool, version string) error { - binDir, err := util.BinaryLocation() - if err != nil { - return err - } - binary := zitiApp - fileName := binary - if !upgrade { - f, flag, err := o.shouldInstallBinary(binDir, binary) - if err != nil || !flag { - return err - } - fileName = f - } - var latestVersion semver.Version - - if version != "" { - - if strings.Contains(version, "*") { - latestVersion, err = util.GetLatestSemanticVersionFromArtifactory(o.Verbose, o.Staging, branch, binary, version) - if err != nil { - return err - } - version = latestVersion.String() - } else { - latestVersion, err = semver.Make(version) - if err != nil { - return err - } - } - } - - fullPath := filepath.Join(binDir, fileName) - ext := ".tar.gz" - tarFile := fullPath + ext - - repoUrl := util.GetArtifactoryPath(o.Staging, binary, branch) + "/" + version + "/" + zitiApp + ext - - log.Infof("Attempting to download %s to %s", repoUrl, tarFile) - - err = util.DownloadArtifactoryFile(repoUrl, tarFile) - if err != nil { - return err - } - - log.Infof("Attempting to extract %s to %s", tarFile, fileName) - err = util.UnTargz(tarFile, binDir, []string{binary, fileName}) - if err != nil { - return err - } - err = os.Remove(tarFile) - if err != nil { - return err - } - log.Infof("Successfully installed '%s' version '%s' from branch '%s'\n", zitiApp, latestVersion, branch) - return os.Chmod(fullPath, 0755) -} - -func (o *InstallOptions) installTerraformProvider(branch string, provider string, upgrade bool, version string) error { - resty.SetDebug(o.Verbose) - binDir, err := util.TerraformProviderBinaryLocation() - if err != nil { - return err - } - latestVersion, err := util.GetLatestTerraformProviderVersionFromArtifactory(branch, provider) - if err != nil { - return err - } - if version != "" { - latestVersion, err = semver.Make(version) - if err != nil { - return err - } - } - - fullPath := filepath.Join(binDir, c.TERRAFORM_PROVIDER_PREFIX+provider) - ext := ".tar.gz" - tarFile := fullPath + ext - - repoUrl := util.GetTerraformProviderArtifactoryPath(provider, branch) + "/" + version + "/" + c.TERRAFORM_PROVIDER_PREFIX + provider + "_v" + version + ext - - log.Infof("Attempting to download %s to %s", repoUrl, tarFile) - - err = util.DownloadArtifactoryFile(repoUrl, tarFile) - if err != nil { - return err - } - fileToExtract := c.TERRAFORM_PROVIDER_PREFIX + provider + "_v" + version - if runtime.GOOS == "windows" { - fileToExtract += ".exe" - } - log.Infof("Attempting to extract file: '%s'\n", fileToExtract) - err = util.UnTargz(tarFile, binDir, []string{fileToExtract}) - if err != nil { - return err - } - err = os.Remove(tarFile) - if err != nil { - return err - } - log.Infof("Successfully installed Terraform Provider '%s' version '%s' from branch '%s'\n", provider, latestVersion, branch) - fileToChmod := fullPath + "_v" + version - if runtime.GOOS == "windows" { - fileToChmod += ".exe" - } - return os.Chmod(fileToChmod, 0755) -} - -func (o *InstallOptions) FindVersionAndInstallGitHubRelease(upgrade bool, zitiApp string, zitiAppGitHub string, version string) error { - binDir, err := util.BinaryLocation() - if err != nil { - return err - } - if !upgrade { - if _, download, err := o.shouldInstallBinary(binDir, zitiApp); err != nil || !download { - return err - } - } - return getziti.FindVersionAndInstallGitHubRelease(zitiApp, zitiAppGitHub, runtime.GOOS, runtime.GOARCH, binDir, version, o.Verbose) -} diff --git a/ziti/cmd/install/install.go b/ziti/cmd/install/install.go deleted file mode 100644 index 1873aa1b6..000000000 --- a/ziti/cmd/install/install.go +++ /dev/null @@ -1,108 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "fmt" - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - "github.com/openziti/ziti/ziti/util" - "io" - "os" - "os/exec" - "strings" - - "github.com/spf13/cobra" -) - -// InstallOptions are the flags for delete commands -type InstallOptions struct { - common.CommonOptions -} - -var ( - install_long = templates.LongDesc(` - Install the Ziti platform binaries. -`) - - install_example = templates.Examples(` - # install the Ziti router - ziti install ziti-router - `) -) - -// GetCommandOutput evaluates the given command and returns the trimmed output -func (options *InstallOptions) GetCommandOutput(dir string, name string, args ...string) (string, error) { - os.Setenv("PATH", util.PathWithBinary()) - e := exec.Command(name, args...) - if dir != "" { - e.Dir = dir - } - data, err := e.CombinedOutput() - text := string(data) - text = strings.TrimSpace(text) - if err != nil { - return "", fmt.Errorf("command failed '%s %s': %s %s", name, strings.Join(args, " "), text, err) - } - return text, err -} - -// NewCmdInstall creates the command -func NewCmdInstall(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallOptions{ - common.CommonOptions{ - Out: out, - Err: errOut, - }, - } - - cmd := &cobra.Command{ - Use: "install [flags]", - Short: "Installs a Ziti component/app", - Long: install_long, - Example: install_example, - Aliases: []string{"install"}, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - SuggestFor: []string{"up"}, - Hidden: true, - } - - cmd.AddCommand(NewCmdInstallZitiALL(out, errOut)) - - cmd.AddCommand(NewCmdInstallZitiController(out, errOut)) - cmd.AddCommand(NewCmdInstallZitiRouter(out, errOut)) - cmd.AddCommand(NewCmdInstallZitiTunnel(out, errOut)) - cmd.AddCommand(NewCmdInstallZitiEdgeTunnel(out, errOut)) - cmd.AddCommand(NewCmdInstallZitiProxC(out, errOut)) - - cmd.AddCommand(NewCmdInstallTerraformProviderEdgeController(out, errOut)) - - options.AddCommonFlags(cmd) - - return cmd -} - -// Run implements this command -func (o *InstallOptions) Run() error { - return o.Cmd.Help() -} diff --git a/ziti/cmd/install/install_terraform_provider_edge_controller.go b/ziti/cmd/install/install_terraform_provider_edge_controller.go deleted file mode 100644 index 46e80b165..000000000 --- a/ziti/cmd/install/install_terraform_provider_edge_controller.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "io" - - "github.com/blang/semver" - "github.com/spf13/cobra" -) - -var ( - installTerraformProviderEdgeControllerLong = templates.LongDesc(` - Installs the Terraform Provider for the Ziti Edge Controller -`) - - installTerraformProviderEdgeControllerExample = templates.Examples(` - # Install the Terraform Provider for the Ziti Edge Controller - ziti install terraform-provider-edgecontroller - `) -) - -// InstallTerraformProviderEdgeControllerOptions the options for the upgrade ziti-tunnel command -type InstallTerraformProviderEdgeControllerOptions struct { - InstallOptions - - Version string - Branch string -} - -// NewCmdInstallTerraformProviderEdgeController defines the command -func NewCmdInstallTerraformProviderEdgeController(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallTerraformProviderEdgeControllerOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "terraform-provider-edgecontroller", - Short: "Installs the Terraform Provider for the Ziti Edge Controller", - Aliases: []string{"tpec"}, - Long: installTerraformProviderEdgeControllerLong, - Example: installTerraformProviderEdgeControllerExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to install") - cmd.Flags().StringVarP(&options.Branch, "branch", "b", "master", "The specific version to install") - cmd.Flags().BoolVarP(&options.Verbose, "verbose", "", false, "Enable verbose logging") - return cmd -} - -// Run implements the command -func (o *InstallTerraformProviderEdgeControllerOptions) Run() error { - newVersion, err := o.getLatestTerraformProviderVersion(o.Branch, c.TERRAFORM_PROVIDER_EDGE_CONTROLLER) - if err != nil { - return err - } - - if o.Version != "" { - newVersion, err = semver.Make(o.Version) - if err != nil { - return err - } - } - - log.Infoln("Attempting to install Terraform Provider '" + c.TERRAFORM_PROVIDER_EDGE_CONTROLLER + "' version: " + newVersion.String()) - - return o.installTerraformProvider(o.Branch, c.TERRAFORM_PROVIDER_EDGE_CONTROLLER, false, newVersion.String()) -} diff --git a/ziti/cmd/install/install_ziti_ALL.go b/ziti/cmd/install/install_ziti_ALL.go deleted file mode 100644 index 54b4c4f83..000000000 --- a/ziti/cmd/install/install_ziti_ALL.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "io" - - "github.com/blang/semver" - "github.com/openziti/ziti/common/version" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "github.com/spf13/cobra" -) - -var ( - installZitiALLLong = templates.LongDesc(` - Installs all Ziti apps that have not been installed already -`) - - installZitiALLExample = templates.Examples(` - # Install all the Ziti apps - ziti install all - `) -) - -// InstallZitiALLOptions the options for the upgrade ziti-channel command -type InstallZitiALLOptions struct { - InstallOptions - - Version string -} - -// NewCmdInstallZitiALL defines the command -func NewCmdInstallZitiALL(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallZitiALLOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "all", - Short: "Installs all the Ziti apps that have not been installed already", - Aliases: []string{"*"}, - Long: installZitiALLLong, - Example: installZitiALLExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to install") - options.AddCommonFlags(cmd) - return cmd -} - -func (o *InstallZitiALLOptions) install(zitiApp string) error { - - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), zitiApp) - if err != nil { - return err - } - - if o.Version != "" { - newVersion, err = semver.Make(o.Version) - if err != nil { - return err - } - } - - log.Infoln("Attempting to install '" + zitiApp + "' version: " + newVersion.String()) - - return o.installZitiApp(version.GetBranch(), zitiApp, false, newVersion.String()) -} - -// Run implements the command -func (o *InstallZitiALLOptions) Run() error { - err := o.install(c.ZITI_CONTROLLER) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - err = o.install(c.ZITI_PROX_C) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - err = o.install(c.ZITI_ROUTER) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - err = o.install(c.ZITI_TUNNEL) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - err = o.install(c.ZITI_EDGE_TUNNEL) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - return nil -} diff --git a/ziti/cmd/install/install_ziti_controller.go b/ziti/cmd/install/install_ziti_controller.go deleted file mode 100644 index f7d82b6a5..000000000 --- a/ziti/cmd/install/install_ziti_controller.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - "io" - - "github.com/blang/semver" - "github.com/openziti/ziti/common/version" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "github.com/spf13/cobra" -) - -var ( - installZitiControllerLong = templates.LongDesc(` - Installs the Ziti Controller app if it has not been installed already -`) - - installZitiControllerExample = templates.Examples(` - # Install the Ziti Controller app - ziti install ziti-controller - `) -) - -// InstallZitiControllerOptions the options for the upgrade ziti-controller command -type InstallZitiControllerOptions struct { - InstallOptions - - Version string -} - -// NewCmdInstallZitiController defines the command -func NewCmdInstallZitiController(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallZitiControllerOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "ziti-controller", - Short: "Installs the Ziti Controller app - if it has not been installed already", - Aliases: []string{"controller"}, - Long: installZitiControllerLong, - Example: installZitiControllerExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to install") - options.AddCommonFlags(cmd) - return cmd -} - -// Run implements the command -func (o *InstallZitiControllerOptions) Run() error { - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_CONTROLLER) - if err != nil { - return err - } - - if o.Version != "" { - newVersion, err = semver.Make(o.Version) - if err != nil { - return err - } - } - - log.Infoln("Attempting to install '" + c.ZITI_CONTROLLER + "' version: " + newVersion.String()) - - return o.installZitiApp(version.GetBranch(), c.ZITI_CONTROLLER, false, newVersion.String()) -} diff --git a/ziti/cmd/install/install_ziti_edge_tunnel.go b/ziti/cmd/install/install_ziti_edge_tunnel.go deleted file mode 100644 index 74725f3b3..000000000 --- a/ziti/cmd/install/install_ziti_edge_tunnel.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/blang/semver" - "github.com/openziti/ziti/common/getziti" - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "github.com/spf13/cobra" - "io" - "strings" -) - -var ( - installZitiEdgeTunnelLong = templates.LongDesc(` - Installs the Ziti Edge Tunnel app if it has not been installed already -`) - - installZitiEdgeTunnelExample = templates.Examples(` - # Install the Ziti Edge Tunnel app - ziti install ziti-edge-tunnel - `) -) - -// InstallZitiEdgeTunnelOptions the options for the upgrade ziti-edge-tunnel command -type InstallZitiEdgeTunnelOptions struct { - InstallOptions - - Version string -} - -// NewCmdInstallZitiEdgeTunnel defines the command -func NewCmdInstallZitiEdgeTunnel(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallZitiEdgeTunnelOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "ziti-edge-tunnel", - Short: "Installs the Ziti Edge Tunnel app - if it has not been installed already", - Aliases: []string{"edge-tunnel"}, - Long: installZitiEdgeTunnelLong, - Example: installZitiEdgeTunnelExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to install") - options.AddCommonFlags(cmd) - return cmd -} - -// Run implements the command -func (o *InstallOptions) installZitiEdgeTunnel(targetVersion string) error { - var newVersion semver.Version - - if targetVersion != "" { - newVersion = semver.MustParse(strings.TrimPrefix(targetVersion, "v")) - } else { - v, err := getziti.GetLatestGitHubReleaseVersion(c.ZITI_EDGE_TUNNEL_GITHUB, o.Verbose) - if err != nil { - return err - } - newVersion = v - } - - log.Infoln("Attempting to install '" + c.ZITI_EDGE_TUNNEL + "' version: " + newVersion.String()) - return o.FindVersionAndInstallGitHubRelease(false, c.ZITI_EDGE_TUNNEL, c.ZITI_EDGE_TUNNEL_GITHUB, newVersion.String()) -} - -// Run implements the command -func (o *InstallZitiEdgeTunnelOptions) Run() error { - return o.installZitiEdgeTunnel(o.Version) -} diff --git a/ziti/cmd/install/install_ziti_prox_c.go b/ziti/cmd/install/install_ziti_prox_c.go deleted file mode 100644 index ec79a4bb2..000000000 --- a/ziti/cmd/install/install_ziti_prox_c.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/blang/semver" - "github.com/openziti/ziti/common/getziti" - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "github.com/spf13/cobra" - "io" - "strings" -) - -var ( - installZitiProxCLong = templates.LongDesc(` - Installs the Ziti ProxC app if it has not been installed already -`) - - installZitiProxCExample = templates.Examples(` - # Install the Ziti ProxC app - ziti install ziti-prox-c - `) -) - -// InstallZitiProxCOptions the options for the upgrade ziti-prox-c command -type InstallZitiProxCOptions struct { - InstallOptions - - Version string -} - -// NewCmdInstallZitiProxC defines the command -func NewCmdInstallZitiProxC(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallZitiProxCOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "ziti-prox-c", - Short: "Installs the Ziti ProxC app - if it has not been installed already", - Aliases: []string{"proxc"}, - Long: installZitiProxCLong, - Example: installZitiProxCExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to install") - options.AddCommonFlags(cmd) - return cmd -} - -func (o *InstallOptions) installZitiProxC(targetVersion string) error { - var newVersion semver.Version - - if targetVersion != "" { - newVersion = semver.MustParse(strings.TrimPrefix(targetVersion, "v")) - } else { - v, err := getziti.GetLatestGitHubReleaseVersion(c.ZITI_SDK_C_GITHUB, o.Verbose) - if err != nil { - return err - } - newVersion = v - } - - log.Infoln("Attempting to install '" + c.ZITI_PROX_C + "' version: " + newVersion.String()) - return o.FindVersionAndInstallGitHubRelease(false, c.ZITI_PROX_C, c.ZITI_SDK_C_GITHUB, newVersion.String()) -} - -// Run implements the command -func (o *InstallZitiProxCOptions) Run() error { - return o.installZitiProxC(o.Version) -} diff --git a/ziti/cmd/install/install_ziti_router.go b/ziti/cmd/install/install_ziti_router.go deleted file mode 100644 index bb18bdfcd..000000000 --- a/ziti/cmd/install/install_ziti_router.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "io" - - "github.com/blang/semver" - "github.com/openziti/ziti/common/version" - "github.com/spf13/cobra" -) - -var ( - installZitiRouterLong = templates.LongDesc(` - Installs the Ziti Router app if it has not been installed already -`) - - installZitiRouterExample = templates.Examples(` - # Install the Ziti Router app - ziti install ziti-router - `) -) - -// InstallZitiRouterOptions the options for the upgrade ziti-router command -type InstallZitiRouterOptions struct { - InstallOptions - - Version string -} - -// NewCmdInstallZitiRouter defines the command -func NewCmdInstallZitiRouter(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallZitiRouterOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "ziti-router", - Short: "Installs the Ziti Router app - if it has not been installed already", - Aliases: []string{"router"}, - Long: installZitiRouterLong, - Example: installZitiRouterExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to install") - options.AddCommonFlags(cmd) - return cmd -} - -// Run implements the command -func (o *InstallZitiRouterOptions) Run() error { - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_ROUTER) - if err != nil { - return err - } - - if o.Version != "" { - newVersion, err = semver.Make(o.Version) - if err != nil { - return err - } - } - - log.Infoln("Attempting to install '" + c.ZITI_ROUTER + "' version: " + newVersion.String()) - - return o.installZitiApp(version.GetBranch(), c.ZITI_ROUTER, false, newVersion.String()) -} diff --git a/ziti/cmd/install/install_ziti_tunnel.go b/ziti/cmd/install/install_ziti_tunnel.go deleted file mode 100644 index f411ae8ad..000000000 --- a/ziti/cmd/install/install_ziti_tunnel.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "io" - - "github.com/blang/semver" - "github.com/openziti/ziti/common/version" - "github.com/spf13/cobra" -) - -var ( - installZitiTunnelLong = templates.LongDesc(` - Installs the Ziti Tunnel app if it has not been installed already -`) - - installZitiTunnelExample = templates.Examples(` - # Install the Ziti Tunnel app - ziti install ziti-tunnel - `) -) - -// InstallZitiTunnelOptions the options for the upgrade ziti-tunnel command -type InstallZitiTunnelOptions struct { - InstallOptions - - Version string -} - -// NewCmdInstallZitiTunnel defines the command -func NewCmdInstallZitiTunnel(out io.Writer, errOut io.Writer) *cobra.Command { - options := &InstallZitiTunnelOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "ziti-tunnel", - Short: "Installs the Ziti Tunnel app - if it has not been installed already", - Aliases: []string{"tunnel"}, - Long: installZitiTunnelLong, - Example: installZitiTunnelExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to install") - options.AddCommonFlags(cmd) - return cmd -} - -// Run implements the command -func (o *InstallZitiTunnelOptions) Run() error { - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_TUNNEL) - if err != nil { - return err - } - - if o.Version != "" { - newVersion, err = semver.Make(o.Version) - if err != nil { - return err - } - } - - log.Infoln("Attempting to install '" + c.ZITI_TUNNEL + "' version: " + newVersion.String()) - - return o.installZitiApp(version.GetBranch(), c.ZITI_TUNNEL, false, newVersion.String()) -} diff --git a/ziti/cmd/install/upgrade.go b/ziti/cmd/install/upgrade.go deleted file mode 100644 index f681b93ad..000000000 --- a/ziti/cmd/install/upgrade.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - "io" - - "github.com/spf13/cobra" -) - -// UpgradeOptions are the flags for delete commands -type UpgradeOptions struct { - common.CommonOptions -} - -var ( - upgrade_long = templates.LongDesc(` - Upgrade the Ziti platform binaries. -`) - - upgrade_example = templates.Examples(` - # upgrade the command line tool - ziti upgrade cli - `) -) - -// NewCmdUpgrade creates the command -func NewCmdUpgrade(out io.Writer, errOut io.Writer) *cobra.Command { - options := &UpgradeOptions{ - common.CommonOptions{ - Out: out, - Err: errOut, - }, - } - - cmd := &cobra.Command{ - Use: "upgrade [flags]", - Short: "Upgrades a Ziti component/app", - Long: upgrade_long, - Example: upgrade_example, - Aliases: []string{"bump"}, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - SuggestFor: []string{"up"}, - Hidden: true, - } - - cmd.AddCommand(NewCmdUpgradeZiti(out, errOut)) - - return cmd -} - -// Run implements this command -func (o *UpgradeOptions) Run() error { - return o.Cmd.Help() -} diff --git a/ziti/cmd/install/upgrade_ziti.go b/ziti/cmd/install/upgrade_ziti.go deleted file mode 100644 index 916c835ea..000000000 --- a/ziti/cmd/install/upgrade_ziti.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - "github.com/openziti/ziti/ziti/cmd/templates" - c "github.com/openziti/ziti/ziti/constants" - "io" - - "github.com/openziti/ziti/common/version" - "github.com/spf13/cobra" -) - -var ( - upgradeCLILong = templates.LongDesc(` - Upgrades the Ziti command line tools if there is a newer release -`) - - upgradeCLIExample = templates.Examples(` - # Upgrades the Ziti CLI tools - ziti upgrade cli - `) -) - -// UpgradeZitiOptions the options for the create spring command -type UpgradeZitiOptions struct { - InstallOptions - - Version string -} - -// NewCmdUpgradeZiti defines the command -func NewCmdUpgradeZiti(out io.Writer, errOut io.Writer) *cobra.Command { - options := &UpgradeZitiOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "ziti", - Short: "Upgrades the ziti CLI - if there is a new version available", - Aliases: []string{"z", "cli"}, - Long: upgradeCLILong, - Example: upgradeCLIExample, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - } - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to upgrade to") - options.AddCommonFlags(cmd) - return cmd -} - -// Run implements the command -func (o *UpgradeZitiOptions) Run() error { - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI) - if err != nil { - return err - } - - newVersionStr := newVersion.String() - - if o.Version != "" { - newVersionStr = o.Version - } - - o.deleteInstalledBinary(c.ZITI) - - return o.installZitiApp(version.GetBranch(), c.ZITI, true, newVersionStr) -} diff --git a/ziti/cmd/install/upgrade_ziti_controller.go b/ziti/cmd/install/upgrade_ziti_controller.go deleted file mode 100644 index ed2a3720b..000000000 --- a/ziti/cmd/install/upgrade_ziti_controller.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/common/version" - c "github.com/openziti/ziti/ziti/constants" -) - -// UpgradeZitiControllerOptions the options for the upgrade ziti-controller command -type UpgradeZitiControllerOptions struct { - InstallOptions - - Version string -} - -// Run implements the command -func (o *UpgradeZitiControllerOptions) Run() error { - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_CONTROLLER) - if err != nil { - return err - } - - newVersionStr := newVersion.String() - - if o.Version != "" { - newVersionStr = o.Version - } - - o.deleteInstalledBinary(c.ZITI_CONTROLLER) - - return o.installZitiApp(version.GetBranch(), c.ZITI_CONTROLLER, true, newVersionStr) -} diff --git a/ziti/cmd/install/upgrade_ziti_edge_tunnel.go b/ziti/cmd/install/upgrade_ziti_edge_tunnel.go deleted file mode 100644 index 1794669a1..000000000 --- a/ziti/cmd/install/upgrade_ziti_edge_tunnel.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - c "github.com/openziti/ziti/ziti/constants" -) - -// UpgradeZitiEdgeTunnelOptions the options for the upgrade ziti-edge-tunnel command -type UpgradeZitiEdgeTunnelOptions struct { - InstallOptions - - Version string -} - -// Run implements the command -func (o *UpgradeZitiEdgeTunnelOptions) Run() error { - newVersion, err := o.getLatestGitHubReleaseVersion(c.ZITI_EDGE_TUNNEL_GITHUB) - if err != nil { - return err - } - - newVersionStr := newVersion.String() - - if o.Version != "" { - newVersionStr = o.Version - } - - o.deleteInstalledBinary(c.ZITI_EDGE_TUNNEL) - - return o.FindVersionAndInstallGitHubRelease(true, c.ZITI_EDGE_TUNNEL, c.ZITI_EDGE_TUNNEL_GITHUB, newVersionStr) -} diff --git a/ziti/cmd/install/upgrade_ziti_prox_c.go b/ziti/cmd/install/upgrade_ziti_prox_c.go deleted file mode 100644 index 620ac1cf3..000000000 --- a/ziti/cmd/install/upgrade_ziti_prox_c.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - c "github.com/openziti/ziti/ziti/constants" -) - -// UpgradeZitiProxCOptions the options for the upgrade ziti-prox-c command -type UpgradeZitiProxCOptions struct { - InstallOptions - - Version string -} - -// Run implements the command -func (o *UpgradeZitiProxCOptions) Run() error { - newVersion, err := o.getLatestGitHubReleaseVersion(c.ZITI_SDK_C_GITHUB) - if err != nil { - return err - } - - newVersionStr := newVersion.String() - - if o.Version != "" { - newVersionStr = o.Version - } - - o.deleteInstalledBinary(c.ZITI_PROX_C) - - return o.FindVersionAndInstallGitHubRelease(true, c.ZITI_PROX_C, c.ZITI_SDK_C_GITHUB, newVersionStr) -} diff --git a/ziti/cmd/install/upgrade_ziti_router.go b/ziti/cmd/install/upgrade_ziti_router.go deleted file mode 100644 index e584f694c..000000000 --- a/ziti/cmd/install/upgrade_ziti_router.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/common/version" - c "github.com/openziti/ziti/ziti/constants" -) - -// UpgradeZitiRouterOptions the options for the upgrade ziti-router command -type UpgradeZitiRouterOptions struct { - InstallOptions - - Version string -} - -// Run implements the command -func (o *UpgradeZitiRouterOptions) Run() error { - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_ROUTER) - if err != nil { - return err - } - - newVersionStr := newVersion.String() - - if o.Version != "" { - newVersionStr = o.Version - } - - o.deleteInstalledBinary(c.ZITI_ROUTER) - - return o.installZitiApp(version.GetBranch(), c.ZITI_ROUTER, true, newVersionStr) -} diff --git a/ziti/cmd/install/upgrade_ziti_tunnel.go b/ziti/cmd/install/upgrade_ziti_tunnel.go deleted file mode 100644 index 254a2d27c..000000000 --- a/ziti/cmd/install/upgrade_ziti_tunnel.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "github.com/openziti/ziti/common/version" - c "github.com/openziti/ziti/ziti/constants" -) - -// UpgradeZitiTunnelOptions the options for the upgrade ziti-tunnel command -type UpgradeZitiTunnelOptions struct { - InstallOptions - - Version string -} - -// Run implements the command -func (o *UpgradeZitiTunnelOptions) Run() error { - newVersion, err := o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_TUNNEL) - if err != nil { - return err - } - - newVersionStr := newVersion.String() - - if o.Version != "" { - newVersionStr = o.Version - } - - o.deleteInstalledBinary(c.ZITI_TUNNEL) - - return o.installZitiApp(version.GetBranch(), c.ZITI_TUNNEL, true, newVersionStr) -} diff --git a/ziti/cmd/install/use.go b/ziti/cmd/install/use.go deleted file mode 100644 index 6226049ed..000000000 --- a/ziti/cmd/install/use.go +++ /dev/null @@ -1,161 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "fmt" - "github.com/openziti/ziti/ziti/cmd/common" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/internal/log" - "github.com/openziti/ziti/ziti/util" - "io" - - "github.com/blang/semver" - "github.com/openziti/ziti/common/version" - "github.com/spf13/cobra" -) - -// UseOptions are the flags for delete commands -type UseOptions struct { - InstallOptions - - Version string - Branch string -} - -// NewCmdUse creates the command -func NewCmdUse(out io.Writer, errOut io.Writer) *cobra.Command { - options := &UseOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "use [branch]", - Short: "switch between branches of Ziti", - Long: ` -'ziti use' fetches the list of currently available branch-build names from artifactory, -presents them in a chooser-list, and once one is selected, will switch the current 'ziti' binary for the -chosen one. This is useful for swapping between different feature branches, or back and forth from a release -build to a feature-branch build. -`, - Aliases: []string{}, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - SuggestFor: []string{}, - Hidden: true, - } - - cmd.Flags().StringVarP(&options.Version, "version", "v", "", "The specific version to use") - cmd.Flags().BoolVarP(&options.Verbose, "verbose", "", false, "Enable verbose logging") - cmd.Flags().StringVarP(&options.Branch, "branch", "b", "", "Name of branch to switch to") - cmd.Flags().BoolVarP(&options.Staging, "staging", "", false, "Install/Upgrade components from the ziti-staging repo") - - return cmd -} - -func (o *UseOptions) install(branch string, zitiApp string) error { - newVersion, err := o.getLatestZitiAppVersionForBranch(branch, zitiApp) - if err != nil { - log.Infoln("Attempt to fetch latest version of '" + zitiApp + "' for branch '" + branch + "' failed: " + err.Error()) - - // Special-case branch fallback (to master) when dealing with ziti-prox-c - if zitiApp == c.ZITI_PROX_C || zitiApp == c.ZITI_EDGE_TUNNEL { - branch = "master" - newVersion, err = o.getLatestZitiAppVersionForBranch(branch, zitiApp) - if err != nil { - return err - } - } else { - return err - } - } - - if o.Version != "" { - newVersion, err = semver.Make(o.Version) - if err != nil { - return err - } - } - - log.Infoln("Attempting to install '" + zitiApp + "' version: " + newVersion.String() + " from branch '" + branch + "'") - - return o.installZitiApp(branch, zitiApp, true, newVersion.String()) -} - -// Run implements the command -func (o *UseOptions) Run() error { - - fmt.Println("Current source branch is: ", version.GetBranch()) - - branch := o.Branch - if branch == "" { - list, err := o.getCurrentZitiSnapshotList() - if err != nil { - return err - } - branch, err = util.PickName(list, "Which Branch would you like to switch to: ") - if err != nil { - return err - } - } - - if o.Staging { - if o.Branch != "main" { - log.Errorf("Error: --staging can only be used with --branch of 'main'. You specified '%s'", branch) - return nil - } - } - - err := o.install(branch, c.ZITI) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - err = o.install(branch, c.ZITI_CONTROLLER) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - err = o.install(branch, c.ZITI_ROUTER) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - err = o.install(branch, c.ZITI_TUNNEL) - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - err = o.installZitiProxC("") - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - err = o.installZitiEdgeTunnel("") - if err != nil { - log.Errorf("Error: install failed %s \n", err.Error()) - } - - return nil -} diff --git a/ziti/cmd/install/version.go b/ziti/cmd/install/version.go deleted file mode 100644 index f398866fe..000000000 --- a/ziti/cmd/install/version.go +++ /dev/null @@ -1,277 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package install - -import ( - "fmt" - "github.com/blang/semver" - "github.com/openziti/ziti/ziti/cmd/common" - "github.com/openziti/ziti/ziti/cmd/table" - "github.com/openziti/ziti/ziti/internal/log" - "io" - "os" - - "github.com/openziti/ziti/common/version" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - c "github.com/openziti/ziti/ziti/constants" - "github.com/openziti/ziti/ziti/util" - - "github.com/spf13/cobra" -) - -type VersionOptions struct { - InstallOptions - - Container string - NoVersionCheck bool -} - -func NewCmdVersion(out io.Writer, errOut io.Writer) *cobra.Command { - options := &VersionOptions{ - InstallOptions: InstallOptions{ - CommonOptions: common.CommonOptions{ - Out: out, - Err: errOut, - }, - }, - } - - cmd := &cobra.Command{ - Use: "version", - Short: "Print the version information", - Aliases: []string{"ver", "v"}, - Run: func(cmd *cobra.Command, args []string) { - options.Cmd = cmd - options.Args = args - err := options.Run() - cmdhelper.CheckErr(err) - }, - Hidden: true, - } - options.AddCommonFlags(cmd) - - cmd.Flags().BoolVarP(&options.NoVersionCheck, "no-update", "n", false, - "disable update check") - - return cmd -} - -// Run ... -func (o *VersionOptions) Run() error { - _, err := util.ConfigDir() - if err != nil { - return err - } - - info := util.ColorInfo - - t := table.CreateTable(os.Stdout) - t.AddRow("NAME", "VERSION") - - t.AddRow(c.ZITI, info(version.GetBuildMetadata(o.Verbose))) - - o.versionPrintZitiApp(c.ZITI_CONTROLLER, &t) - o.versionPrintZitiApp(c.ZITI_PROX_C, &t) - o.versionPrintZitiApp(c.ZITI_ROUTER, &t) - o.versionPrintZitiApp(c.ZITI_TUNNEL, &t) - o.versionPrintZitiApp(c.ZITI_EDGE_TUNNEL, &t) - - t.Render() - - if !o.NoVersionCheck && !o.Verbose { - return o.versionCheck() - } - - return nil -} - -func (o *VersionOptions) getVersionFromZitiApp(zitiApp string, versionArg string) (string, error) { - if o.Verbose { - return o.GetCommandOutput("", zitiApp, versionArg, "--verbose") - } - return o.GetCommandOutput("", zitiApp, versionArg) -} - -func (o *VersionOptions) getVersionFromZitiAppMultiArg(zitiApp string) (string, error) { - var varsionArg string - if zitiApp == c.ZITI { - varsionArg = "--version" - } else { - varsionArg = "version" - } - return o.getVersionFromZitiApp(zitiApp, varsionArg) -} - -func (o *VersionOptions) versionPrintZitiApp(zitiApp string, table *table.Table) { - output, err := o.getVersionFromZitiAppMultiArg(zitiApp) - if err == nil { - table.AddRow(zitiApp, util.ColorInfo(output)) - } else { - if !isBinaryInstalled(zitiApp) { - table.AddRow(zitiApp, util.ColorWarning("not installed")) - } - } -} - -func (o *VersionOptions) versionCheck() error { - if err := o.versionCheckZitiApp(c.ZITI); err != nil { - return err - } - if err := o.versionCheckZitiApp(c.ZITI_CONTROLLER); err != nil { - return err - } - if err := o.versionCheckZitiApp(c.ZITI_PROX_C); err != nil { - return err - } - if err := o.versionCheckZitiApp(c.ZITI_ROUTER); err != nil { - return err - } - if err := o.versionCheckZitiApp(c.ZITI_TUNNEL); err != nil { - return err - } - if err := o.versionCheckZitiApp(c.ZITI_EDGE_TUNNEL); err != nil { - return err - } - return nil -} - -func (o *VersionOptions) versionCheckZitiApp(zitiApp string) error { - var currentVersion semver.Version - var newVersion semver.Version - var err error - - if !isBinaryInstalled(zitiApp) { - return nil - } - - switch zitiApp { - case c.ZITI: - newVersion, err = o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI) - case c.ZITI_CONTROLLER: - newVersion, err = o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_CONTROLLER) - case c.ZITI_PROX_C: - newVersion, err = o.getLatestGitHubReleaseVersion(c.ZITI_SDK_C_GITHUB) - case c.ZITI_ROUTER: - newVersion, err = o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_ROUTER) - case c.ZITI_TUNNEL: - newVersion, err = o.getLatestZitiAppVersion(version.GetBranch(), c.ZITI_TUNNEL) - case c.ZITI_EDGE_TUNNEL: - newVersion, err = o.getLatestGitHubReleaseVersion(c.ZITI_EDGE_TUNNEL_GITHUB) - default: - return nil - } - if err != nil { - return err - } - - app := util.ColorInfo(zitiApp) - output, err := o.getVersionFromZitiAppMultiArg(zitiApp) - if err != nil { - log.Warnf("\nAn err occurred: %s", err) - } else { - currentVersion, err = semver.ParseTolerant(output) - if err != nil { - log.Warnf("Failed to get %s version: %s\n", zitiApp, err) - return err - } - if newVersion.GT(currentVersion) { - log.Warnf("\nA new %s version is available: %s\n", app, util.ColorInfo(newVersion.String())) - - if o.BatchMode { - log.Warnf("To upgrade to this new version use: %s\n", util.ColorInfo("ziti upgrade "+zitiApp)) - } else { - message := fmt.Sprintf("Would you like to upgrade to the new %s version?", app) - if util.Confirm(message, true, "Please indicate if you would like to upgrade the binary version.") { - switch zitiApp { - case c.ZITI: - err = o.upgradeZiti() - case c.ZITI_CONTROLLER: - err = o.upgradeZitiController() - case c.ZITI_PROX_C: - err = o.upgradeZitiProxC() - case c.ZITI_ROUTER: - err = o.upgradeZitiRouter() - case c.ZITI_TUNNEL: - err = o.upgradeZitiTunnel() - case c.ZITI_EDGE_TUNNEL: - err = o.upgradeZitiEdgeTunnel() - default: - } - if err != nil { - return err - } - } - } - } - } - - return nil -} - -func (o *VersionOptions) upgradeZiti() error { - options := &UpgradeZitiOptions{ - InstallOptions: InstallOptions{ - CommonOptions: o.CommonOptions, - }, - } - return options.Run() -} - -func (o *VersionOptions) upgradeZitiController() error { - options := &UpgradeZitiControllerOptions{ - InstallOptions: InstallOptions{ - CommonOptions: o.CommonOptions, - }, - } - return options.Run() -} - -func (o *VersionOptions) upgradeZitiProxC() error { - options := &UpgradeZitiProxCOptions{ - InstallOptions: InstallOptions{ - CommonOptions: o.CommonOptions, - }, - } - return options.Run() -} - -func (o *VersionOptions) upgradeZitiRouter() error { - options := &UpgradeZitiRouterOptions{ - InstallOptions: InstallOptions{ - CommonOptions: o.CommonOptions, - }, - } - return options.Run() -} - -func (o *VersionOptions) upgradeZitiTunnel() error { - options := &UpgradeZitiTunnelOptions{ - InstallOptions: InstallOptions{ - CommonOptions: o.CommonOptions, - }, - } - return options.Run() -} - -func (o *VersionOptions) upgradeZitiEdgeTunnel() error { - options := &UpgradeZitiEdgeTunnelOptions{ - InstallOptions: InstallOptions{ - CommonOptions: o.CommonOptions, - }, - } - return options.Run() -} diff --git a/ziti/cmd/table/padding.go b/ziti/cmd/table/padding.go deleted file mode 100644 index e8d5a9745..000000000 --- a/ziti/cmd/table/padding.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package table - -import ( - "math" - "strings" -) - -const ( - ALIGN_LEFT = 0 - ALIGN_CENTER = 1 - ALIGN_RIGHT = 2 -) - -func Pad(s, pad string, width int, align int) string { - switch align { - case ALIGN_CENTER: - return PadCenter(s, pad, width) - case ALIGN_RIGHT: - return PadLeft(s, pad, width) - default: - return PadRight(s, pad, width) - } -} - -func PadRight(s, pad string, width int) string { - gap := width - len(s) - if gap > 0 { - return s + strings.Repeat(pad, gap) - } - return s -} - -func PadLeft(s, pad string, width int) string { - gap := width - len(s) - if gap > 0 { - return strings.Repeat(pad, gap) + s - } - return s -} - -func PadCenter(s, pad string, width int) string { - gap := width - len(s) - if gap > 0 { - gapLeft := int(math.Ceil(float64(gap) / 2)) - gapRight := gap - gapLeft - return strings.Repeat(pad, gapLeft) + s + strings.Repeat(pad, gapRight) - } - return s -} diff --git a/ziti/cmd/table/table.go b/ziti/cmd/table/table.go deleted file mode 100644 index 52528a5cc..000000000 --- a/ziti/cmd/table/table.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package table - -import ( - "fmt" - "io" -) - -type Table struct { - Out io.Writer - Rows [][]string - ColumnWidths []int - ColumnAlign []int -} - -func CreateTable(out io.Writer) Table { - return Table{ - Out: out, - } -} - -// Clear removes all rows while preserving the layout -func (t *Table) Clear() { - t.Rows = [][]string{} -} - -// AddRow adds a new row to the table -func (t *Table) AddRow(col ...string) { - t.Rows = append(t.Rows, col) -} - -func (t *Table) Render() { - // lets figure out the max widths of each column - for _, row := range t.Rows { - for ci, col := range row { - l := len(col) - t.ColumnWidths = ensureArrayCanContain(t.ColumnWidths, ci) - if l > t.ColumnWidths[ci] { - t.ColumnWidths[ci] = l - } - } - } - - out := t.Out - for _, row := range t.Rows { - lastColumn := len(row) - 1 - for ci, col := range row { - if ci > 0 { - fmt.Fprint(out, " ") - } - l := t.ColumnWidths[ci] - align := t.GetColumnAlign(ci) - if ci >= lastColumn && align != ALIGN_CENTER && align != ALIGN_RIGHT { - fmt.Fprint(out, col) - } else { - fmt.Fprint(out, Pad(col, " ", l, align)) - } - } - fmt.Fprint(out, "\n") - } -} - -// SetColumnsAligns sets the alignment of the columns -func (t *Table) SetColumnsAligns(colAligns []int) { - t.ColumnAlign = colAligns -} - -// GetColumnAlign return the column alignment -func (t *Table) GetColumnAlign(i int) int { - t.ColumnAlign = ensureArrayCanContain(t.ColumnAlign, i) - return t.ColumnAlign[i] -} - -// SetColumnAlign sets the column alignment for the given column index -func (t *Table) SetColumnAlign(i int, align int) { - t.ColumnAlign = ensureArrayCanContain(t.ColumnAlign, i) - t.ColumnAlign[i] = align -} - -func ensureArrayCanContain(array []int, idx int) []int { - diff := idx + 1 - len(array) - for i := 0; i < diff; i++ { - array = append(array, 0) - } - return array -} diff --git a/ziti/cmd/templates/templater.go b/ziti/cmd/templates/templater.go index e60a141b2..9c57dea47 100644 --- a/ziti/cmd/templates/templater.go +++ b/ziti/cmd/templates/templater.go @@ -19,7 +19,6 @@ package templates import ( "bytes" "fmt" - "github.com/openziti/ziti/ziti/cmd/table" "io" "strings" "text/template" @@ -168,7 +167,7 @@ func (t *templater) cmdGroupsString(c *cobra.Command) string { if cmd.Runnable() { path := t.groupPath(cmd) //cmds = append(cmds, " "+rpad(path, maxLen - len(path))+" "+cmd.Short) - cmds = append(cmds, " "+table.PadRight(path, " ", maxLen)+" "+cmd.Short) + cmds = append(cmds, " "+padRight(path, " ", maxLen)+" "+cmd.Short) } } groups = append(groups, strings.Join(cmds, "\n")) @@ -176,6 +175,14 @@ func (t *templater) cmdGroupsString(c *cobra.Command) string { return strings.Join(groups, "\n\n") } +func padRight(s, pad string, width int) string { + gap := width - len(s) + if gap > 0 { + return s + strings.Repeat(pad, gap) + } + return s +} + func (t *templater) rootCmdName(c *cobra.Command) string { return t.rootCmd(c).CommandPath() } diff --git a/ziti/controller/root.go b/ziti/controller/root.go index 610e82963..c59827cb5 100644 --- a/ziti/controller/root.go +++ b/ziti/controller/root.go @@ -19,8 +19,9 @@ package controller import ( "fmt" "github.com/michaelquigley/pfxlog" - edgeSubCmd "github.com/openziti/ziti/controller/subcmd" "github.com/openziti/ziti/common/version" + edgeSubCmd "github.com/openziti/ziti/controller/subcmd" + "github.com/openziti/ziti/ziti/cmd/common" "github.com/openziti/ziti/ziti/constants" "github.com/openziti/ziti/ziti/util" "github.com/sirupsen/logrus" @@ -60,7 +61,11 @@ func NewControllerCmd() *cobra.Command { cmd.AddCommand(NewRunCmd()) cmd.AddCommand(NewDeleteSessionsFromConfigCmd()) cmd.AddCommand(NewDeleteSessionsFromDbCmd()) - cmd.AddCommand(NewVersionCmd()) + + versionCmd := common.NewVersionCmd() + versionCmd.Hidden = true + versionCmd.Deprecated = "use 'ziti version' instead of 'ziti controller version'" + cmd.AddCommand(versionCmd) edgeSubCmd.AddCommands(cmd, version.GetCmdBuildInfo()) diff --git a/ziti/controller/version.go b/ziti/controller/version.go deleted file mode 100644 index 99128008e..000000000 --- a/ziti/controller/version.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controller - -import ( - "fmt" - "github.com/openziti/ziti/common/version" - "github.com/spf13/cobra" -) - -func NewVersionCmd() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Show component version", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println(version.GetBuildMetadata(verbose)) - }, - } -} diff --git a/ziti/internal/log/log.go b/ziti/internal/log/log.go index 48182ec06..f94374dbf 100644 --- a/ziti/internal/log/log.go +++ b/ziti/internal/log/log.go @@ -34,10 +34,6 @@ func Infoln(msg string) { fmt.Println(msg) } -func Blank() { - fmt.Println() -} - func Warnf(msg string, args ...interface{}) { Warn(fmt.Sprintf(msg, args...)) } @@ -46,14 +42,6 @@ func Warn(msg string) { color.Yellow(msg) } -func Errorf(msg string, args ...interface{}) { - Error(fmt.Sprintf(msg, args...)) -} - -func Error(msg string) { - color.Red(msg) -} - func Fatalf(msg string, args ...interface{}) { Fatal(fmt.Sprintf(msg, args...)) } diff --git a/ziti/router/root.go b/ziti/router/root.go index 3a9c6f66d..0c4ee9aa6 100644 --- a/ziti/router/root.go +++ b/ziti/router/root.go @@ -19,6 +19,7 @@ package router import ( "fmt" "github.com/michaelquigley/pfxlog" + "github.com/openziti/ziti/ziti/cmd/common" "github.com/openziti/ziti/ziti/constants" "github.com/openziti/ziti/ziti/util" "github.com/sirupsen/logrus" @@ -58,7 +59,11 @@ func NewRouterCmd() *cobra.Command { routerCmd.AddCommand(NewRunCmd()) routerCmd.AddCommand(NewEnrollGwCmd()) - routerCmd.AddCommand(NewVersionCmd()) + + versionCmd := common.NewVersionCmd() + versionCmd.Hidden = true + versionCmd.Deprecated = "use 'ziti version' instead of 'ziti router version'" + routerCmd.AddCommand(versionCmd) return routerCmd } diff --git a/ziti/router/version.go b/ziti/router/version.go deleted file mode 100644 index 8dad7e7e1..000000000 --- a/ziti/router/version.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package router - -import ( - "fmt" - "github.com/openziti/ziti/common/version" - "github.com/spf13/cobra" -) - -func NewVersionCmd() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Show component version", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println(version.GetBuildMetadata(verbose)) - }, - } -} diff --git a/ziti/tunnel/root.go b/ziti/tunnel/root.go index 8a21abf32..3b4389544 100644 --- a/ziti/tunnel/root.go +++ b/ziti/tunnel/root.go @@ -29,7 +29,6 @@ import ( "github.com/michaelquigley/pfxlog" "github.com/openziti/agent" "github.com/openziti/sdk-golang/ziti" - "github.com/openziti/ziti/common/enrollment" "github.com/openziti/ziti/common/version" "github.com/openziti/ziti/tunnel" "github.com/openziti/ziti/tunnel/dns" @@ -70,12 +69,14 @@ func NewTunnelCmd(standalone bool) *cobra.Command { root.PersistentFlags().StringVar(&cliAgentAddr, "cli-agent-addr", "", "Specify where CLI Agent should list (ex: unix:/tmp/myfile.sock or tcp:127.0.0.1:10001)") root.PersistentFlags().StringVar(&cliAgentAlias, "cli-agent-alias", "", "Alias which can be used by ziti agent commands to find this instance") - p := common.NewOptionsProvider(os.Stdout, os.Stderr) - root.AddCommand(enrollment.NewEnrollCommand(p)) root.AddCommand(NewHostCmd()) root.AddCommand(NewProxyCmd()) root.AddCommand(hostSpecificCmds...) - root.AddCommand(NewVersionCmd()) + + versionCmd := common.NewVersionCmd() + versionCmd.Hidden = true + versionCmd.Deprecated = "use 'ziti version' instead of 'ziti router version'" + root.AddCommand(versionCmd) return root } @@ -86,13 +87,6 @@ var cliAgentEnabled bool var cliAgentAddr string var cliAgentAlias string -func Execute() { - if err := NewTunnelCmd(true).Execute(); err != nil { - pfxlog.Logger().Errorf("error: %s", err) - os.Exit(1) - } -} - func rootPreRun(cmd *cobra.Command, _ []string) { verbose, err := cmd.Flags().GetBool("verbose") if err != nil { diff --git a/ziti/util/colors.go b/ziti/util/colors.go deleted file mode 100644 index 1e817ab60..000000000 --- a/ziti/util/colors.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import "github.com/fatih/color" - -var ColorInfo = color.New(color.FgGreen).SprintFunc() -var ColorStatus = color.New(color.FgBlue).SprintFunc() -var ColorWarning = color.New(color.FgYellow).SprintFunc() -var ColorError = color.New(color.FgRed).SprintFunc() diff --git a/ziti/util/commands.go b/ziti/util/commands.go index ac9d1c71e..0baebbf88 100644 --- a/ziti/util/commands.go +++ b/ziti/util/commands.go @@ -18,22 +18,10 @@ package util import ( "github.com/spf13/cobra" - "os" ) -const ( - CONFIGFILENAME = "config" -) - -func PathWithBinary() string { - path := os.Getenv("PATH") - binDir, _ := BinaryLocation() - return binDir + string(os.PathListSeparator) + path -} - // NewEmptyParentCmd creates a new cobra command with no parent func NewEmptyParentCmd(name string, description string) *cobra.Command { - return &cobra.Command{ Use: name, Short: description, diff --git a/ziti/util/dirs.go b/ziti/util/dirs.go index 37d2e341c..42ac2ec26 100644 --- a/ziti/util/dirs.go +++ b/ziti/util/dirs.go @@ -17,8 +17,6 @@ package util import ( - "encoding/json" - "fmt" "os" "path/filepath" "runtime" @@ -54,32 +52,6 @@ func ConfigDir() (string, error) { return path, nil } -func ZitiAppConfigDir(zitiApp string) (string, error) { - h, err := ConfigDir() - if err != nil { - return "", err - } - path := filepath.Join(h, zitiApp) - err = os.MkdirAll(path, DefaultWritePermissions) - if err != nil { - return "", err - } - return path, nil -} - -func CacheDir() (string, error) { - h, err := ConfigDir() - if err != nil { - return "", err - } - path := filepath.Join(h, "cache") - err = os.MkdirAll(path, DefaultWritePermissions) - if err != nil { - return "", err - } - return path, nil -} - func EnvironmentsDir() (string, error) { h, err := ConfigDir() if err != nil { @@ -93,25 +65,6 @@ func EnvironmentsDir() (string, error) { return path, nil } -func NewEnvironmentDir(envName string) (string, error) { - h, err := EnvironmentsDir() - if err != nil { - return "", err - } - path := filepath.Join(h, envName) - - _, err = os.Stat(path) - if err == nil { - return "", fmt.Errorf("Environment dir (%s) already exists", envName) - } - - err = os.MkdirAll(path, DefaultWritePermissions) - if err != nil { - return "", err - } - return path, nil -} - func PKIRootDir() (string, error) { var path string var err error @@ -132,82 +85,3 @@ func PKIRootDir() (string, error) { } return path, nil } - -func BinaryLocation() (string, error) { - h, err := ConfigDir() - if err != nil { - return "", err - } - path := filepath.Join(h, "bin") - err = os.MkdirAll(path, DefaultWritePermissions) - if err != nil { - return "", err - } - return path, nil -} - -// TerraformProviderBinaryLocation provides the proper location to place a Terraform provider based on the currently running OS. -// In Mac/Linux, it's `~/.terraform.d/plugins` and on Windows it's `%APPDATA%\terraform.d\plugins` -func TerraformProviderBinaryLocation() (string, error) { - var path string - h := HomeDir() - if runtime.GOOS == "windows" { - h = os.Getenv("APPDATA") - if h == "" { - return "", fmt.Errorf("APPDATA env var missing; install of Terraform provider cannot proceed") - } - path = filepath.Join(h, "terraform.d/plugins") - } else { - path = filepath.Join(h, ".terraform.d/plugins") - } - err := os.MkdirAll(path, DefaultWritePermissions) - if err != nil { - return "", err - } - return path, nil -} - -// WriteZitiAppConfigFile writes out the config file data for the given Ziti application to the appropriate config file -func WriteZitiAppConfigFile(zitiApp string, configData interface{}) error { - return WriteZitiAppFile(zitiApp, "config", configData) -} - -// WriteZitiAppFile writes application data (config, session, preferences, etc) to an appropriate location -func WriteZitiAppFile(zitiApp string, fileType string, appData interface{}) error { - configDir, err := ZitiAppConfigDir(zitiApp) - if err != nil { - return err - } - filePath := filepath.Join(configDir, fileType+".json") - - data, err := json.MarshalIndent(appData, "", " ") - if err != nil { - return err - } - - return os.WriteFile(filePath, data, 0600) -} - -// ReadZitiAppConfigFile reads in the config file data for the given Ziti application from an appropriate location -func ReadZitiAppConfigFile(zitiApp string, configData interface{}) error { - return ReadZitiAppFile(zitiApp, "config", configData) -} - -// ReadZitiAppFile reads application data (config, session, preferences, etc) for the given Ziti application from an appropriate location -func ReadZitiAppFile(zitiApp string, fileType string, configData interface{}) error { - configDir, err := ZitiAppConfigDir(zitiApp) - if err != nil { - return err - } - - filePath := filepath.Join(configDir, fileType+".json") - - data, err := os.ReadFile(filePath) - if err != nil { - return err - } - - err = json.Unmarshal(data, configData) - - return err -} diff --git a/ziti/util/files.go b/ziti/util/files.go index 305f50c00..e36b4e0db 100644 --- a/ziti/util/files.go +++ b/ziti/util/files.go @@ -17,150 +17,14 @@ package util import ( - "fmt" "io" "os" - "path/filepath" - "strconv" ) const ( DefaultWritePermissions = 0760 ) -func FileExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return true, err -} - -func IsEmpty(name string) (bool, error) { - f, err := os.Open(name) - if err != nil { - return false, err - } - defer f.Close() - - _, err = f.Readdirnames(1) // Or f.Readdir(1) - if err == io.EOF { - return true, nil - } - return false, err // Either not empty or error, suits both cases -} - -// CreateUniqueDirectory creates a new directory but if the combination of dir and name exists -// then append a number until a unique name is found -func CreateUniqueDirectory(dir string, name string, maximumAttempts int) (string, error) { - for i := 0; i < maximumAttempts; i++ { - n := name - if i > 0 { - n += strconv.Itoa(i) - } - p := filepath.Join(dir, n) - exists, err := FileExists(p) - if err != nil { - return p, err - } - if !exists { - err := os.MkdirAll(p, DefaultWritePermissions) - if err != nil { - return "", fmt.Errorf("Failed to create directory %s due to %s", p, err) - } - return p, nil - } - } - return "", fmt.Errorf("Could not create a unique file in %s starting with %s after %d attempts", dir, name, maximumAttempts) -} - -func RenameDir(src string, dst string, force bool) (err error) { - err = CopyDir(src, dst, force) - if err != nil { - return fmt.Errorf("failed to copy source dir %s to %s: %s", src, dst, err) - } - err = os.RemoveAll(src) - if err != nil { - return fmt.Errorf("failed to cleanup source dir %s: %s", src, err) - } - return nil -} - -func RenameFile(src string, dst string) (err error) { - err = CopyFile(src, dst) - if err != nil { - return fmt.Errorf("failed to copy source file %s to %s: %s", src, dst, err) - } - err = os.RemoveAll(src) - if err != nil { - return fmt.Errorf("failed to cleanup source file %s: %s", src, err) - } - return nil -} - -// credit https://gist.github.com/r0l1/92462b38df26839a3ca324697c8cba04 -func CopyDir(src string, dst string, force bool) (err error) { - src = filepath.Clean(src) - dst = filepath.Clean(dst) - - si, err := os.Stat(src) - if err != nil { - return err - } - if !si.IsDir() { - return fmt.Errorf("source is not a directory") - } - - _, err = os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return - } - if err == nil { - if force { - os.RemoveAll(dst) - } else { - return fmt.Errorf("destination already exists") - } - } - - err = os.MkdirAll(dst, si.Mode()) - if err != nil { - return - } - - entries, err := os.ReadDir(src) - if err != nil { - return - } - - for _, entry := range entries { - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - if entry.IsDir() { - err = CopyDir(srcPath, dstPath, force) - if err != nil { - return - } - } else { - // Skip symlinks. - if entry.Type()&os.ModeSymlink != 0 { - continue - } - - err = CopyFile(srcPath, dstPath) - if err != nil { - return - } - } - } - - return -} - // credit https://gist.github.com/r0l1/92462b38df26839a3ca324697c8cba04 func CopyFile(src, dst string) (err error) { in, err := os.Open(src) @@ -200,65 +64,3 @@ func CopyFile(src, dst string) (err error) { return } - -// CopyDirOverwrite copies from the source dir to the destination dir overwriting files along the way -func CopyDirOverwrite(src string, dst string) (err error) { - src = filepath.Clean(src) - dst = filepath.Clean(dst) - - si, err := os.Stat(src) - if err != nil { - return err - } - if !si.IsDir() { - return fmt.Errorf("source is not a directory") - } - - _, err = os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return - } - - err = os.MkdirAll(dst, si.Mode()) - if err != nil { - return - } - - entries, err := os.ReadDir(src) - if err != nil { - return - } - - for _, entry := range entries { - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - if entry.IsDir() { - err = CopyDirOverwrite(srcPath, dstPath) - if err != nil { - return - } - } else { - // Skip symlinks. - if entry.Type()&os.ModeSymlink != 0 { - continue - } - - err = CopyFile(srcPath, dstPath) - if err != nil { - return - } - } - } - return -} - -// loads a file -func LoadBytes(dir, name string) ([]byte, error) { - path := filepath.Join(dir, name) // relative path - bytes, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("error loading file %s in directory %s, %v", name, dir, err) - } - return bytes, nil -} diff --git a/ziti/util/pickers.go b/ziti/util/pickers.go index 2099bfb05..a11610319 100644 --- a/ziti/util/pickers.go +++ b/ziti/util/pickers.go @@ -17,11 +17,6 @@ package util import ( - "fmt" - "github.com/openziti/ziti/ziti/internal/log" - "sort" - "strings" - "gopkg.in/AlecAivazis/survey.v1" ) @@ -42,19 +37,6 @@ func PickValue(message string, defaultValue string, required bool) (string, erro return answer, nil } -func PickPassword(message string) (string, error) { - answer := "" - prompt := &survey.Password{ - Message: message, - } - validator := survey.Required - err := survey.AskOne(prompt, &answer, validator) - if err != nil { - return "", err - } - return strings.TrimSpace(answer), nil -} - func PickNameWithDefault(names []string, message string, defaultValue string) (string, error) { name := "" if len(names) == 0 { @@ -75,91 +57,6 @@ func PickNameWithDefault(names []string, message string, defaultValue string) (s return name, nil } -func PickRequiredNameWithDefault(names []string, message string, defaultValue string) (string, error) { - name := "" - if len(names) == 0 { - return "", nil - } else if len(names) == 1 { - name = names[0] - } else { - prompt := &survey.Select{ - Message: message, - Options: names, - Default: defaultValue, - } - err := survey.AskOne(prompt, &name, survey.Required) - if err != nil { - return "", err - } - } - return name, nil -} - func PickName(names []string, message string) (string, error) { return PickNameWithDefault(names, message, "") } - -func PickNames(names []string, message string) ([]string, error) { - picked := []string{} - if len(names) == 0 { - return picked, nil - } else if len(names) == 1 { - return names, nil - } else { - prompt := &survey.MultiSelect{ - Message: message, - Options: names, - } - err := survey.AskOne(prompt, &picked, nil) - if err != nil { - return picked, err - } - } - return picked, nil -} - -// SelectNamesWithFilter selects from a list of names with a given filter. Optionally selecting them all -func SelectNamesWithFilter(names []string, message string, selectAll bool, filter string) ([]string, error) { - filtered := []string{} - for _, name := range names { - if filter == "" || strings.Contains(name, filter) { - filtered = append(filtered, name) - } - } - if len(filtered) == 0 { - return nil, fmt.Errorf("No names match filter: %s", filter) - } - return SelectNames(filtered, message, selectAll) -} - -// SelectNames select which names from the list should be chosen -func SelectNames(names []string, message string, selectAll bool) ([]string, error) { - answer := []string{} - if len(names) == 0 { - return answer, fmt.Errorf("No names to choose from!") - } - sort.Strings(names) - - prompt := &survey.MultiSelect{ - Message: message, - Options: names, - } - if selectAll { - prompt.Default = names - } - err := survey.AskOne(prompt, &answer, nil) - return answer, err -} - -// Confirm prompts the user to confirm something -func Confirm(message string, defaultValue bool, help string) bool { - answer := defaultValue - prompt := &survey.Confirm{ - Message: message, - Default: defaultValue, - Help: help, - } - _ = survey.AskOne(prompt, &answer, nil) - log.Blank() - return answer -} diff --git a/ziti/util/rest.go b/ziti/util/rest.go index 5919e01d8..d88319aa5 100644 --- a/ziti/util/rest.go +++ b/ziti/util/rest.go @@ -17,66 +17,23 @@ package util import ( - "archive/tar" "bytes" - "compress/gzip" "encoding/json" "fmt" "github.com/Jeffail/gabs" - "github.com/blang/semver" openApiRuntime "github.com/go-openapi/runtime" "github.com/go-openapi/strfmt" "github.com/openziti/edge-api/rest_management_api_client" "github.com/openziti/edge-api/rest_model" - "github.com/openziti/ziti/common/version" fabric_rest_client "github.com/openziti/ziti/controller/rest_client" - cmdhelper "github.com/openziti/ziti/ziti/cmd/helpers" - c "github.com/openziti/ziti/ziti/constants" "gopkg.in/resty.v1" "io" "net/http" "net/url" - "os" "path" - "path/filepath" - "runtime" - "strconv" - "strings" - "text/template" "time" ) -// Download a file from the given URL -func DownloadFile(filepath string, url string) (err error) { - // Create the file - out, err := os.Create(filepath) - if err != nil { - return err - } - defer out.Close() - - // Get the data - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - // Writer the body to file - _, err = io.Copy(out, resp.Body) - if err != nil { - return err - } - - // make it executable - err = os.Chmod(filepath, 0755) - - if err != nil { - return err - } - return nil -} - // Use a 2-second timeout with a retry count of 5 func NewClient() *resty.Client { return resty. @@ -86,316 +43,6 @@ func NewClient() *resty.Client { SetRedirectPolicy(resty.FlexibleRedirectPolicy(15)) } -func getRequest(verbose bool) *resty.Request { - return NewClient(). - SetDebug(verbose). - R() -} - -// DownloadArtifactoryFile will download a file from the given Artifactory URL -func DownloadArtifactoryFile(url string, filepath string) (err error) { - fullUrl := "https://netfoundry.jfrog.io/netfoundry/" + url - resp, err := getRequest(false). - SetHeader("X-JFrog-Art-Api", cmdhelper.JFrogAPIKey()). - SetOutput(filepath). - Get(fullUrl) - - if err != nil { - return fmt.Errorf("unable to download '%s', %s", url, err) - } - - if resp.IsError() { - return fmt.Errorf("unable to download file, error HTTP status code [%d] returned for url [%s]", resp.StatusCode(), fullUrl) - } - - return nil -} - -// Used to parse the 'get-object-tagging' response -type Data struct { - TagSet []struct { - Value string `json:"Value"` - Key string `json:"Key"` - } -} - -// Used to parse the '/api/versions' response from Artifactory -type ArtifactoryVersionsData struct { - Version string `json:"version"` - Artifacts []struct { - } -} - -func GetLatestVersionFromArtifactory(verbose bool, staging bool, branch string, appName string) (semver.Version, error) { - resp, err := getRequest(verbose). - SetQueryParams(map[string]string{ - // "key": "value", - }). - SetHeader("X-JFrog-Art-Api", cmdhelper.JFrogAPIKey()). - SetResult(&ArtifactoryVersionsData{}). - Get("https://netfoundry.jfrog.io/netfoundry/api/versions/" + GetArtifactoryPath(staging, appName, branch)) - - if err != nil { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s'; %s", appName, err) - } - - if resp.StatusCode() == http.StatusNotFound { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s' on branch '%s'; Not Found", appName, branch) - } - if resp.StatusCode() != http.StatusOK { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s' on branch '%s'; %s", appName, branch, resp.Status()) - } - - result := *resp.Result().(*ArtifactoryVersionsData) - - return semver.Make(strings.TrimPrefix(result.Version, "v")) -} - -// Used to parse the '/api/search/aql' response from Artifactory -type AQLResult struct { - Repo string - Path string - Name string - Type string - Size int - Created string - CreatedBy string - Modified string - ModifiedBy string - Updated string - Properties []struct { - Key string - Value string - } -} -type ArtifactoryAQLData struct { - Results []AQLResult -} - -type AQLVars struct { - SemverMajor string - SemverMaxMinor string - SemverMinMinor string - App string - Arch string - OS string -} - -func GetLatestSemanticVersionFromArtifactory(verbose bool, staging bool, branch string, appName string, versionWildcard string) (semver.Version, error) { - sv := strings.Split(versionWildcard, ".") - minor, err := strconv.Atoi(sv[1]) - if err != nil { - panic(err) - } - maxMinor := minor + 1 - aqlVars := AQLVars{sv[0], strconv.Itoa(maxMinor), sv[1], appName, runtime.GOARCH, runtime.GOOS} - tpl, err := template.New("aql").Parse("items.find( { \"@build.number\":{\"$lt\":\"{{ .SemverMajor}}.{{ .SemverMaxMinor}}.*\"}, \"@build.number\":{\"$gt\":\"{{ .SemverMajor}}.{{ .SemverMinMinor}}.*\"}, \"repo\":{\"$match\":\"ziti-release\"}, \"path\":{\"$match\":\"{{ .App}}/{{ .Arch}}/{{ .OS}}/*\"} } ).include(\"@build.number\") ") - if err != nil { - panic(err) - } - var body bytes.Buffer - err = tpl.Execute(&body, aqlVars) - if err != nil { - panic(err) - } - resp, err := getRequest(verbose). - SetHeader("X-JFrog-Art-Api", cmdhelper.JFrogAPIKey()). - SetHeader("Content-Type", "text/plain"). - SetBody(body.String()). - SetResult(&ArtifactoryAQLData{}). - Post("https://netfoundry.jfrog.io/netfoundry/api/search/aql") - - if err != nil { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s'; %s", appName, err) - } - - if resp.StatusCode() == http.StatusNotFound { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s' on branch '%s'; Not Found", appName, branch) - } - if resp.StatusCode() != http.StatusOK { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s' on branch '%s'; %s", appName, branch, resp.Status()) - } - - aqlData := (*resp.Result().(*ArtifactoryAQLData)) - - latestSemVer, _ := semver.Make("0.0.0") - - for _, result := range aqlData.Results { - sv, err := semver.Make(result.Properties[0].Value) - if err != nil { - panic(err) - } - if sv.GT(latestSemVer) { - latestSemVer = sv - } - } - - return latestSemVer, nil -} - -func GetLatestTerraformProviderVersionFromArtifactory(branch string, provider string) (semver.Version, error) { - repoUrl := "https://netfoundry.jfrog.io/netfoundry/api/versions/" + GetTerraformProviderArtifactoryPath(provider, branch) - resp, err := getRequest(false). - SetQueryParams(map[string]string{ - // "key": "value", - }). - SetHeader("X-JFrog-Art-Api", cmdhelper.JFrogAPIKey()). - SetResult(&ArtifactoryVersionsData{}). - Get(repoUrl) - - if err != nil { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s' on repo url %s; %s", provider, repoUrl, err) - } - - if resp.StatusCode() == http.StatusNotFound { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s' on branch '%s' on repo url %s; Not Found", provider, branch, repoUrl) - } - if resp.StatusCode() != http.StatusOK { - return semver.Version{}, fmt.Errorf("unable to get latest version for '%s' on branch '%s' on repo url %s; %s", provider, branch, repoUrl, resp.Status()) - } - - result := (*resp.Result().(*ArtifactoryVersionsData)) - - return semver.Make(strings.TrimPrefix(result.Version, "v")) -} - -// Used to parse the '/api/storage' response from Artifactory -type ArtifactoryStorageChildrenData struct { - URI string `json:"uri"` - Folder bool `json:"folder"` -} -type ArtifactoryStorageData struct { - Repo string `json:"repo"` - Path string `json:"path"` - Created string `json:"created"` - LastModified string `json:"lastModified"` - LastUpdated string `json:"lastUpdated"` - Children []ArtifactoryStorageChildrenData `json:"children"` - URI string `json:"uri"` -} - -func GetCurrentSnapshotListFromArtifactory(verbose bool) ([]ArtifactoryStorageChildrenData, error) { - resp, err := getRequest(verbose). - SetHeader("X-JFrog-Art-Api", cmdhelper.JFrogAPIKey()). - SetResult(&ArtifactoryStorageData{}). - Get("https://netfoundry.jfrog.io/netfoundry/api/storage/ziti-snapshot/") - - if err != nil { - return nil, fmt.Errorf("unable to get list of branches; %s", err) - } - - result := (*resp.Result().(*ArtifactoryStorageData)) - - return result.Children, nil -} - -func GetArtifactoryPath(staging bool, appName string, branch string) string { - if branch == "" { - branch = version.GetBranch() - } - - arch := runtime.GOARCH - os := runtime.GOOS - - var path string - if staging { - path = "ziti-staging/" - } else if branch == "main" { - path = "ziti-release/" - } else { - path = "ziti-snapshot/" + branch + "/" - } - // Special-case the source-repo when dealing with ziti-prox-c - if branch == "main" && appName == c.ZITI_PROX_C { - path = "ziti-staging/" - } - - path += appName + "/" + arch + "/" + os - - return path -} - -func GetTerraformProviderArtifactoryPath(provider string, branch string) string { - if branch == "" { - branch = "master" - } - var path string - if branch == "master" { - path = c.TERRAFORM_PROVIDER_PREFIX + provider + "-release/" - } else { - path = c.TERRAFORM_PROVIDER_PREFIX + provider + "-snapshot/" + branch + "/" - } - path += c.TERRAFORM_PROVIDER_PREFIX + provider + "/" + version.GetArchitecture() + "/" + version.GetOS() - - return path -} - -// untargz a tarball to a target, from -// http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip -func UnTargz(tarball, target string, onlyFiles []string) error { - zreader, err := os.Open(tarball) - if err != nil { - return err - } - defer zreader.Close() - - reader, err := gzip.NewReader(zreader) - defer func() { - _ = reader.Close() - }() - - if err != nil { - panic(err) - } - - tarReader := tar.NewReader(reader) - - for { - inkey := false - header, err := tarReader.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - - for _, value := range onlyFiles { - if value == "*" || value == path.Base(header.Name) { - inkey = true - break - } - } - - if !inkey { - continue - } - - path := filepath.Join(target, path.Base(header.Name)) - info := header.FileInfo() - if info.IsDir() { - if err = os.MkdirAll(path, info.Mode()); err != nil { - return err - } - continue - } else { - if err = os.Remove(path); err != nil { - fmt.Printf("error removing [%s] (%v)", path, err) - } - } - - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) - if err != nil { - return err - } - defer file.Close() - _, err = io.Copy(file, tarReader) - if err != nil { - return err - } - } - return nil -} - func PrettyPrintResponse(resp *resty.Response) string { out := resp.String() var prettyJSON bytes.Buffer diff --git a/ziti/util/session.go b/ziti/util/session.go deleted file mode 100644 index 3ff612507..000000000 --- a/ziti/util/session.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "fmt" -) - -const ( - appName = "ziti" - sessionType = "edge-controller-session" -) - -// Session stores configuration options for the CLI -type Session struct { - Host string - Token string - Cert string -} - -func (session *Session) GetBaseUrl() string { - return session.Host -} - -func (session *Session) GetCert() string { - return session.Cert -} - -func (session *Session) GetToken() string { - return session.Token -} - -// Persist writes out the Ziti CLI session file -func (session *Session) Persist() error { - return WriteZitiAppFile(appName, sessionType, session) -} - -// Load reads in the Ziti CLI session file -func (session *Session) Load() error { - err := ReadZitiAppFile(appName, sessionType, session) - if err != nil { - return fmt.Errorf("unable to load Ziti CLI configuration. Exiting. Error: %v", err) - } - if session.Host == "" { - return fmt.Errorf("host not specified in cli config file. Exiting") - } - return nil -} - -func (session *Session) String() string { - return fmt.Sprintf("session Host: %v, Token: %s, Cert: %s", session.Host, session.Token, session.Cert) -} diff --git a/ziti/util/strings.go b/ziti/util/strings.go deleted file mode 100644 index 6a2a52716..000000000 --- a/ziti/util/strings.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "regexp" - "sort" - "strings" -) - -// RegexpSplit splits a string into an array using the regexSep as a separator -func RegexpSplit(text string, regexSeperator string) []string { - reg := regexp.MustCompile(regexSeperator) - indexes := reg.FindAllStringIndex(text, -1) - lastIdx := 0 - result := make([]string, len(indexes)+1) - for i, element := range indexes { - result[i] = text[lastIdx:element[0]] - lastIdx = element[1] - } - result[len(indexes)] = text[lastIdx:] - return result -} - -// StringIndexes returns all the indices where the value occurs in the given string -func StringIndexes(text string, value string) []int { - answer := []int{} - t := text - valueLen := len(value) - offset := 0 - for { - idx := strings.Index(t, value) - if idx < 0 { - break - } - answer = append(answer, idx+offset) - offset += valueLen - t = t[idx+valueLen:] - } - return answer -} - -func StringArrayIndex(array []string, value string) int { - for i, v := range array { - if v == value { - return i - } - } - return -1 -} - -// FirstNotEmptyString returns the first non empty string or the empty string if none can be found -func FirstNotEmptyString(values ...string) string { - for _, v := range values { - if v != "" { - return v - } - } - return "" -} - -// SortedMapKeys returns the sorted keys of the given map -func SortedMapKeys(m map[string]string) []string { - answer := make([]string, 0, len(m)) - for k := range m { - answer = append(answer, k) - } - sort.Strings(answer) - return answer -} - -func ReverseStrings(a []string) { - for i, j := 0, len(a)-1; i < j; i, j = i+1, j-1 { - a[i], a[j] = a[j], a[i] - } -} - -// StringArrayToLower returns a string slice with all the values converted to lower case -func StringArrayToLower(values []string) []string { - answer := make([]string, 0, len(values)) - for _, v := range values { - answer = append(answer, strings.ToLower(v)) - } - return answer -} - -// StringMatches returns true if the given text matches the includes/excludes lists -func StringMatchesAny(text string, includes []string, excludes []string) bool { - for _, x := range excludes { - if StringMatchesPattern(text, x) { - return false - } - } - if len(includes) == 0 { - return true - } - for _, inc := range includes { - if StringMatchesPattern(text, inc) { - return true - } - } - return false -} - -// StringMatchesPattern returns true if the given text matches the includes/excludes lists -func StringMatchesPattern(text string, pattern string) bool { - if pattern == "*" { - return true - } - if strings.HasSuffix(pattern, "*") { - prefix := strings.TrimSuffix(pattern, "*") - return strings.HasPrefix(text, prefix) - } - return text == pattern -} diff --git a/ziti/util/suggestions.go b/ziti/util/suggestions.go deleted file mode 100644 index 5d223ff32..000000000 --- a/ziti/util/suggestions.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "fmt" - "sort" - "strings" -) - -const ( - DefaultSuggestionsMinimumDistance = 2 -) - -func InvalidOptionError(option string, value string, err error) error { - return InvalidOptionf(option, value, "%s", err) -} - -func InvalidOptionf(option string, value string, message string, a ...interface{}) error { - text := fmt.Sprintf(message, a...) - return fmt.Errorf("Invalid option: --%s %s\n%s", option, value, text) -} - -func MissingOption(name string) error { - return fmt.Errorf("Missing option: --%s", name) -} - -func InvalidOption(name string, value string, values []string) error { - suggestions := SuggestionsFor(value, values, DefaultSuggestionsMinimumDistance) - if len(suggestions) > 0 { - if len(suggestions) == 1 { - return InvalidOptionf(name, value, "Did you mean: --%s %s", name, suggestions[0]) - } - return InvalidOptionf(name, value, "Did you mean one of: %s", strings.Join(suggestions, ", ")) - } - sort.Strings(values) - return InvalidOptionf(name, value, "Possible values: %s", strings.Join(values, ", ")) -} - -func InvalidArg(value string, values []string) error { - suggestions := SuggestionsFor(value, values, DefaultSuggestionsMinimumDistance) - if len(suggestions) > 0 { - if len(suggestions) == 1 { - return InvalidArgf(value, "Did you mean: %s", suggestions[0]) - } - return InvalidArgf(value, "Did you mean one of: %s", strings.Join(suggestions, ", ")) - } - sort.Strings(values) - return InvalidArgf(value, "Possible values: %s", strings.Join(values, ", ")) -} - -func InvalidArgError(value string, err error) error { - return InvalidArgf(value, "%s", err) -} - -func InvalidArgf(value string, message string, a ...interface{}) error { - text := fmt.Sprintf(message, a...) - return fmt.Errorf("Invalid argument: %s\n%s", value, text) -} - -func SuggestionsFor(typedName string, values []string, suggestionsMinimumDistance int, explicitSuggestions ...string) []string { - suggestions := []string{} - for _, value := range values { - levenshteinDistance := ld(typedName, value, true) - suggestByLevenshtein := levenshteinDistance <= suggestionsMinimumDistance - suggestByPrefix := strings.HasPrefix(strings.ToLower(value), strings.ToLower(typedName)) - if suggestByLevenshtein || suggestByPrefix { - suggestions = append(suggestions, value) - } - for _, explicitSuggestion := range explicitSuggestions { - if strings.EqualFold(typedName, explicitSuggestion) { - suggestions = append(suggestions, value) - } - } - } - return suggestions -} - -// ld compares two strings and returns the levenshtein distance between them. -// -// this was copied from vendor/github.com/spf13/cobra/command.go as its not public -func ld(s, t string, ignoreCase bool) int { - if ignoreCase { - s = strings.ToLower(s) - t = strings.ToLower(t) - } - d := make([][]int, len(s)+1) - for i := range d { - d[i] = make([]int, len(t)+1) - } - for i := range d { - d[i][0] = i - } - for j := range d[0] { - d[0][j] = j - } - for j := 1; j <= len(t); j++ { - for i := 1; i <= len(s); i++ { - if s[i-1] == t[j-1] { - d[i][j] = d[i-1][j-1] - } else { - min := d[i-1][j] - if d[i][j-1] < min { - min = d[i][j-1] - } - if d[i-1][j-1] < min { - min = d[i-1][j-1] - } - d[i][j] = min + 1 - } - } - - } - return d[len(s)][len(t)] -} diff --git a/zititest/go.mod b/zititest/go.mod index cd87b36f4..f72f361cf 100644 --- a/zititest/go.mod +++ b/zititest/go.mod @@ -13,22 +13,23 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/michaelquigley/pfxlog v0.6.10 github.com/openziti/agent v1.0.16 - github.com/openziti/channel/v2 v2.0.122 - github.com/openziti/edge-api v0.26.14 - github.com/openziti/fablab v0.5.42 - github.com/openziti/foundation/v2 v2.0.40 - github.com/openziti/identity v1.0.73 - github.com/openziti/sdk-golang v0.23.15 - github.com/openziti/storage v0.2.33 - github.com/openziti/transport/v2 v2.0.125 + github.com/openziti/channel/v2 v2.0.128 + github.com/openziti/edge-api v0.26.16 + github.com/openziti/fablab v0.5.49 + github.com/openziti/foundation/v2 v2.0.42 + github.com/openziti/identity v1.0.75 + github.com/openziti/sdk-golang v0.23.19 + github.com/openziti/storage v0.2.36 + github.com/openziti/transport/v2 v2.0.131 github.com/openziti/ziti v0.28.3 + github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/pkg/errors v0.9.1 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 go.etcd.io/bbolt v1.3.9 - golang.org/x/net v0.22.0 + golang.org/x/net v0.24.0 google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -87,7 +88,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/schema v1.2.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect - github.com/hashicorp/go-hclog v1.6.2 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect @@ -100,7 +101,7 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.13.0 // indirect github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect - github.com/jedib0t/go-pretty/v6 v6.5.6 // indirect + github.com/jedib0t/go-pretty/v6 v6.5.8 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -137,13 +138,12 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/openziti/dilithium v0.3.3 // indirect github.com/openziti/jwks v1.0.3 // indirect - github.com/openziti/metrics v1.2.48 // indirect - github.com/openziti/runzmd v1.0.41 // indirect - github.com/openziti/secretstream v0.1.18 // indirect + github.com/openziti/metrics v1.2.51 // indirect + github.com/openziti/runzmd v1.0.43 // indirect + github.com/openziti/secretstream v0.1.19 // indirect github.com/openziti/x509-claims v1.0.3 // indirect github.com/openziti/xweb/v2 v2.1.0 // indirect github.com/openziti/ziti-db-explorer v1.1.3 // indirect - github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/parallaxsecond/parsec-client-go v0.0.0-20221025095442-f0a77d263cf9 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/sftp v1.13.6 // indirect @@ -185,21 +185,21 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect go4.org v0.0.0-20180809161055-417644f6feb5 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/image v0.13.0 // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect - golang.org/x/sync v0.6.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.18.0 // indirect - gopkg.in/AlecAivazis/survey.v1 v1.8.7 // indirect + gopkg.in/AlecAivazis/survey.v1 v1.8.8 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/resty.v1 v1.12.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - nhooyr.io/websocket v1.8.10 // indirect + nhooyr.io/websocket v1.8.11 // indirect rsc.io/goversion v1.2.0 // indirect ) diff --git a/zititest/go.sum b/zititest/go.sum index a888528c5..aea03010a 100644 --- a/zititest/go.sum +++ b/zititest/go.sum @@ -44,6 +44,7 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/AlecAivazis/survey/v2 v2.0.5/go.mod h1:WYBhg6f0y/fNYUuesWQc0PKbJcEliGcYHB9sNT3Bg74= github.com/AppsFlyer/go-sundheit v0.5.0 h1:/VxpyigCfJrq1r97mn9HPiAB2qrhcTFHwNIIDr15CZM= github.com/AppsFlyer/go-sundheit v0.5.0/go.mod h1:2ZM0BnfqT/mljBQO224VbL5XH06TgWuQ6Cn+cTtCpTY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -360,8 +361,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= -github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -411,8 +412,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/jedib0t/go-pretty/v6 v6.5.6 h1:nKXVLqPfAwY7sWcYXdNZZZ2fjqDpAtj9UeWupgfUxSg= -github.com/jedib0t/go-pretty/v6 v6.5.6/go.mod h1:5LQIxa52oJ/DlDSLv0HEkWOFMDGoWkJb9ss5KqPpJBg= +github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ= +github.com/jedib0t/go-pretty/v6 v6.5.8/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jeremija/gosubmit v0.2.7 h1:At0OhGCFGPXyjPYAsCchoBUhE099pcBXmsb4iZqROIc= github.com/jeremija/gosubmit v0.2.7/go.mod h1:Ui+HS073lCFREXBbdfrJzMB57OI/bdxTiLtrDHHhFPI= @@ -455,6 +456,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -588,32 +590,32 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openziti/agent v1.0.16 h1:9Saji+8hFE1NpzP2XzDhsVJbCrDlhixoLHfOpFt5Z+U= github.com/openziti/agent v1.0.16/go.mod h1:zfm53+PVWoGFzjGGgQdKby5749G6VRYHe+eQJmoVKy4= -github.com/openziti/channel/v2 v2.0.122 h1:MPALuLUuTsW2QJzaoRkrP/wGqhIeWtSsrnxt2Ept5SU= -github.com/openziti/channel/v2 v2.0.122/go.mod h1:n4r/pWTFYJy98hEtf4ATYVHU70OePjoM8JJzrIvXvZg= +github.com/openziti/channel/v2 v2.0.128 h1:kRigyUmrN1q56ydRmBWzruV0DvVntYx/f2DT2P0qSh4= +github.com/openziti/channel/v2 v2.0.128/go.mod h1:ugebO2eO2CdaVQE45o20iE4R9DK0VRoJsGBZt0x3eLc= github.com/openziti/dilithium v0.3.3 h1:PLgQ6PMNLSTzCFbX/h98cmudgz/cU6TmjdSv5NAPD8k= github.com/openziti/dilithium v0.3.3/go.mod h1:vsCjI2AU/hon9e+dLhUFbCNGesJDj2ASgkySOcpmvjo= -github.com/openziti/edge-api v0.26.14 h1:BpxKavDVdykOY7GSdKXi7klv9Im4vw7bTMujz3wFE1Q= -github.com/openziti/edge-api v0.26.14/go.mod h1:aTRqB9J/6ISbw9+YWjzD0tvWAXY98m8yWcbiGNqAQTU= -github.com/openziti/fablab v0.5.42 h1:vENJKfEba2T4sSLwlKDL/IzBYfY8iHnhc4umf6IESiY= -github.com/openziti/fablab v0.5.42/go.mod h1:HDT06y1QX8kO8ZQrgHvZmJsvc8iRybESGtlDLDII4ks= -github.com/openziti/foundation/v2 v2.0.40 h1:8oXL77egZjnQuKuxPTHiBOh+X/I2mSJefIbnLSv8rFI= -github.com/openziti/foundation/v2 v2.0.40/go.mod h1:38RikdtjvzDUALm3jT3PSOrU0bHCleRty6bHGdfFyUI= -github.com/openziti/identity v1.0.73 h1:F1iF/rDQYfDIv4ktqBYwK/0C5Ra/7y92Z5Umrw4c4fk= -github.com/openziti/identity v1.0.73/go.mod h1:SKcvyjTlSIpxJuhYLjayJgB800EJ/nO4Y/Lhh77NE9A= +github.com/openziti/edge-api v0.26.16 h1:oU4t2Q4f0+K/ypwdQIecgA836PJkCnDMe1glc56ovaA= +github.com/openziti/edge-api v0.26.16/go.mod h1:HTp6bC3cBKAoIwyA06aKKrrBwxFtJALfpEinmvCRgBc= +github.com/openziti/fablab v0.5.49 h1:uYmBQBS7auPeUlQBpnrU+ktDFhUTOklwwftBBqXcaDk= +github.com/openziti/fablab v0.5.49/go.mod h1:TAXN6lhTnmjL4nwkio3Ql96Lf5gnyeSbm7uV+CilsPo= +github.com/openziti/foundation/v2 v2.0.42 h1:cXZFql3xY92AHPCk/wqIoSUX9EHgegwPrs5bhFE2jNg= +github.com/openziti/foundation/v2 v2.0.42/go.mod h1:a+REYnK9bZ2cvmOiuBLp57MjQqn3U5cHhJcDMLMO8rE= +github.com/openziti/identity v1.0.75 h1:FDm3ZojmmGP9eoOxMvmF52RS6JzToTTeWB0pthIA+ks= +github.com/openziti/identity v1.0.75/go.mod h1:ouF+CYh5ywvvMr0Uy1+tbkILIPTdob5WZEnm3v0bPMQ= github.com/openziti/jwks v1.0.3 h1:hf8wkb+Cg4nH/HM0KROFd7u+C3DkRVcFZJ7tDV+4icc= github.com/openziti/jwks v1.0.3/go.mod h1:t4xxq8vlXGsPn29kiQVnZBBDDnEoOFqtJoHibkJunQQ= -github.com/openziti/metrics v1.2.48 h1:bDUcWSTWM4vXh4YU8z953Q/vtlUjtH+pSaQviKzB578= -github.com/openziti/metrics v1.2.48/go.mod h1:rLFoXigAFb5clCDqe/RlvFFjlaTr9Li7iEn3OxInmb0= -github.com/openziti/runzmd v1.0.41 h1:Iuwe91Lzq63ux2R6+VIWEvcEKhpKkPb1QVNwrJHOuDA= -github.com/openziti/runzmd v1.0.41/go.mod h1:T79oGLLJNnAhHzcfnlrOR39S2XGkzwYZLZh62xee0wk= -github.com/openziti/sdk-golang v0.23.15 h1:+h+zin2CB6nP8e2Sqi2Jxo4Sbo3wTp8PIfh7QNm4lPA= -github.com/openziti/sdk-golang v0.23.15/go.mod h1:aYvN+PCUV2AHPcbgcG1Khgso5TbmNaLMke0ByshU4Hs= -github.com/openziti/secretstream v0.1.18 h1:bMHI4ez6pVSGFzHCHHn6imkaCYqd3o9Oa8gDObf8icI= -github.com/openziti/secretstream v0.1.18/go.mod h1:gosi8ohW+dAHhz1h/MjZEcEVWO0jisWqWjEjmx7oAEs= -github.com/openziti/storage v0.2.33 h1:15WKxJ0J1GCNOMmOkoosjoxKN2Hrm4p3g8p9WAiMS9Y= -github.com/openziti/storage v0.2.33/go.mod h1:AbE5KkTVlXVFHRqpc7Ug3eFpNOc1HFBFwNIYn8Rtp+s= -github.com/openziti/transport/v2 v2.0.125 h1:6kDfabGFIE2+UPyvJVtfI43CNKcMwNOrCRX602CbDU4= -github.com/openziti/transport/v2 v2.0.125/go.mod h1:meVOsDPZSO/hNWSA+NGK5sAoeSgWoqfJgKUZlM3iX1E= +github.com/openziti/metrics v1.2.51 h1:+RfVdEGMOmhXBcC/gFIqXeNVnpWt9+cfrbCacFV/e3w= +github.com/openziti/metrics v1.2.51/go.mod h1:qvX8EKF8I42tA4KlNFX+RBx+ywNiVRvJTS4QofDU8KU= +github.com/openziti/runzmd v1.0.43 h1:jQpDQ91I+6kbAM/2QV5s2900/CWgs3HyPhrpCNRQE6E= +github.com/openziti/runzmd v1.0.43/go.mod h1:sTSSi4f3dcgO5w8PgSK4fGAlwR2ajexAlMcEtduuFaU= +github.com/openziti/sdk-golang v0.23.19 h1:jl0PWdsTIKY41Kzku6qAo/KEsIMjyrdKqCcIg+qAR8I= +github.com/openziti/sdk-golang v0.23.19/go.mod h1:mmxfoFAs6hMR1B76rnRSOCG35th6/YY9/Gp8zUeL9PI= +github.com/openziti/secretstream v0.1.19 h1:qSX23lyOI5ROvYPTjy7Lo6eFsq9qBBPzaG5hvgLlw48= +github.com/openziti/secretstream v0.1.19/go.mod h1:1n/Uk8PQEREwB7iQXhUzNqt+8jYEpUZOJDGcetMI08o= +github.com/openziti/storage v0.2.36 h1:EN1PlA0SnGB8mm4QvHgGtVj39OL6me5sgcSl7v2J7Mg= +github.com/openziti/storage v0.2.36/go.mod h1:1fIRgTNyzqoVuC/wAXqpL+QQra+t3MVHp//2XIeG6HM= +github.com/openziti/transport/v2 v2.0.131 h1:K1h3sl04QWQdYGSPSI1aZLufrL44Azw4WmdprzkIMxo= +github.com/openziti/transport/v2 v2.0.131/go.mod h1:N+OPeoqIuZwS72q0XnM4DLHVdKZ3ZOR37rqjOjWA0b8= github.com/openziti/x509-claims v1.0.3 h1:HNdQ8Nf1agB3lBs1gahcO6zfkeS4S5xoQ2/PkY4HRX0= github.com/openziti/x509-claims v1.0.3/go.mod h1:Z0WIpBm6c4ecrpRKrou6Gk2wrLWxJO/+tuUwKh8VewE= github.com/openziti/xweb/v2 v2.1.0 h1:Xhh3C2pZkq/Prr65V+SfFSibLDYteoc4f62KQCcTZF4= @@ -873,11 +875,11 @@ golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -889,8 +891,9 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -989,8 +992,9 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1021,9 +1025,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1041,6 +1044,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1111,8 +1115,9 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1311,8 +1316,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/AlecAivazis/survey.v1 v1.8.7 h1:oBJqtgsyBLg9K5FK9twNUbcPnbCPoh+R9a+7nag3qJM= -gopkg.in/AlecAivazis/survey.v1 v1.8.7/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= +gopkg.in/AlecAivazis/survey.v1 v1.8.8 h1:5UtTowJZTz1j7NxVzDGKTz6Lm9IWm8DDF6b7a2wq9VY= +gopkg.in/AlecAivazis/survey.v1 v1.8.8/go.mod h1:CaHjv79TCgAvXMSFJSVgonHXYWxnhzI3eoHtnX5UgUo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1362,8 +1367,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q= -nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w= rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= diff --git a/zititest/models/db-ctrl-test/configs/ctrl.yml.tmpl b/zititest/models/db-ctrl-test/configs/ctrl.yml.tmpl new file mode 100644 index 000000000..430638ecf --- /dev/null +++ b/zititest/models/db-ctrl-test/configs/ctrl.yml.tmpl @@ -0,0 +1,194 @@ +v: 3 + +db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/ctrl.db + +identity: + cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem + key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key + ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.chain.pem + +network: + enableLegacyLinkMgmt: true + +tls: + handshakeTimeout: 30s + +# the endpoint that routers will connect to the controller over. +ctrl: + listener: tls:0.0.0.0:6262 + options: + advertiseAddress: tls:{{ .Host.PublicIp }}:6262 + # (optional) settings + # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000) + #maxQueuedConnects: 50 + + # the maximum number of connects that have begun hello synchronization (1 to 1000, default 16) + #maxOutstandingConnects: 100 + + # the number of milliseconds to wait before a hello synchronization fails and closes the connection (30ms to 60000ms, default: 1000ms) + #connectTimeoutMs: 3000 + + # Sets the control channel write timeout. A write timeout will close the control channel, so the router will reconnect + #writeTimeout: 15s + + # A listener address which will be sent to connecting routers in order to change their configured controller + # address. If defined, routers will update address configuration to immediately use the new address for future + # connections. The value of newListener must be resolvable both via DNS and validate via certificates + #newListener: tls:localhost:6262 + +events: + jsonLogger: + subscriptions: +# - type: fabric.routers +# - type: fabric.terminators +# - type: metrics +# sourceFilter: .* +# metricFilter: .*egress.*m1_rate* +# - type: fabric.circuits +# include: +# - created +# - type: edge.sessions +# include: +# - created +# - type: edge.apiSessions +# - type: fabric.usage +# - type: services +# - type: fabric.usage + - type: edge.entityCounts + interval: 1m + handler: + type: file + format: json + path: /home/ubuntu/logs/events.json + +healthChecks: + boltCheck: + # How often to try entering a bolt read tx. Defaults to 30 seconds + interval: 30s + # When to timeout the check. Defaults to 15 seconds + timeout: 15s + # How long to wait before starting the check. Defaults to 15 seconds + initialDelay: 15s + +# By having an 'edge' section defined, the ziti-controller will attempt to parse the edge configuration. Removing this +# section, commenting out, or altering the name of the section will cause the edge to not run. +edge: + # This section represents the configuration of the Edge API that is served over HTTPS + api: + #(optional, default 90s) Alters how frequently heartbeat and last activity values are persisted + # activityUpdateInterval: 90s + #(optional, default 250) The number of API Sessions updated for last activity per transaction + # activityUpdateBatchSize: 250 + # sessionTimeout - optional, default 10m + # The number of minutes before an Edge API session will timeout. Timeouts are reset by + # API requests and connections that are maintained to Edge Routers + sessionTimeout: 30m + # address - required + # The default address (host:port) to use for enrollment for the Client API. This value must match one of the addresses + # defined in a bind point's address field for the `edge-client` API in the web section. + address: {{ .Host.PublicIp }}:1280 + # enrollment - required + # A section containing settings pertaining to enrollment. + enrollment: + # signingCert - required + # A Ziti Identity configuration section that specifically makes use of the cert and key fields to define + # a signing certificate from the PKI that the Ziti environment is using to sign certificates. The signingCert.cert + # will be added to the /.well-known CA store that is used to bootstrap trust with the Ziti Controller. + signingCert: + cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert + key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key + + # edgeIdentity - optional + # A section for identity enrollment specific settings + edgeIdentity: + # duration - optional, default 5m + # The length of time that a Ziti Edge Identity enrollment should remain valid. After + # this duration, the enrollment will expire and not longer be usable. + duration: 1h + # edgeRouter - Optional + # A section for edge router enrollment specific settings. + edgeRouter: + # duration - optional, default 5m + # The length of time that a Ziti Edge Router enrollment should remain valid. After + # this duration, the enrollment will expire and not longer be usable. + duration: 1h + + +# web - optional +# Defines webListeners that will be hosted by the controller. Each webListener can host many APIs and be bound to many +# bind points. +web: + # name - required + # Provides a name for this listener, used for logging output. Not required to be unique, but is highly suggested. + - name: all-apis-localhost + # bindPoints - required + # One or more bind points are required. A bind point specifies an interface (interface:port string) that defines + # where on the host machine the webListener will listen and the address (host:port) that should be used to + # publicly address the webListener(i.e. mydomain.com, localhost, 127.0.0.1). This public address may be used for + # incoming address resolution as well as used in responses in the API. + bindPoints: + #interface - required + # A host:port string on which network interface to listen on. 0.0.0.0 will listen on all interfaces + - interface: 0.0.0.0:1280 + + # address - required + # The public address that external incoming requests will be able to resolve. Used in request processing and + # response content that requires full host:port/path addresses. + address: {{ .Host.PublicIp }}:1280 + + # newAddress - optional + # A host:port string which will be sent out as an HTTP header "ziti-new-address" if specified. If the header + # is present, clients should update location configuration to immediately use the new address for future + # connections. The value of newAddress must be resolvable both via DNS and validate via certificates + #newAddress: localhost:1280 + # identity - optional + # Allows the webListener to have a specific identity instead of defaulting to the root `identity` section. + # identity: + # cert: ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ctrl-client.cert.pem + # server_cert: ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ctrl-server.cert.pem + # key: ${ZITI_SOURCE}/ziti/etc/ca/intermediate/private/ctrl.key.pem + # ca: ${ZITI_SOURCE}/ziti/etc/ca/intermediate/certs/ca-chain.cert.pem + # options - optional + # Allows the specification of webListener level options - mainly dealing with HTTP/TLS settings. These options are + # used for all http servers started by the current webListener. + options: + # idleTimeout - optional, default 5000ms + # The maximum amount of idle time in milliseconds allowed for pipelined HTTP requests. Setting this too high + # can cause resources on the host to be consumed as clients remain connected and idle. Lowering this value + # will cause clients to reconnect on subsequent HTTPs requests. + idleTimeout: 5000ms #http timeouts, new + + # readTimeout - optional, default 5000ms + # The maximum amount of time in milliseconds http servers will wait to read the first incoming requests. A higher + # value risks consuming resources on the host with clients that are acting bad faith or suffering from high latency + # or packet loss. A lower value can risk losing connections to high latency/packet loss clients. + + readTimeout: 5000ms + # writeTimeout - optional, default 10000ms + # The total maximum time in milliseconds that the http server will wait for a single requests to be received and + # responded too. A higher value can allow long running requests to consume resources on the host. A lower value + # can risk ending requests before the server has a chance to respond. + + writeTimeout: 100000ms + # minTLSVersion - optional, default TSL1.2 + # The minimum version of TSL to support + + minTLSVersion: TLS1.2 + # maxTLSVersion - optional, default TSL1.3 + # The maximum version of TSL to support + + maxTLSVersion: TLS1.3 + # apis - required + # Allows one or more APIs to be bound to this webListener + apis: + # binding - required + # Specifies an API to bind to this webListener. Built-in APIs are + # - health-checks + # - edge-management + # - edge-client + # - fabric-management + - binding: health-checks + - binding: fabric + - binding: edge-management + - binding: edge-client + - binding: edge-oidc \ No newline at end of file diff --git a/zititest/models/db-ctrl-test/configs/router.yml.tmpl b/zititest/models/db-ctrl-test/configs/router.yml.tmpl new file mode 100644 index 000000000..af00778de --- /dev/null +++ b/zititest/models/db-ctrl-test/configs/router.yml.tmpl @@ -0,0 +1,78 @@ +{{$ssh_username := .Model.MustVariable "credentials.ssh.username"}} +{{$identity := .Component.Id}} +{{$router_ip := .Host.PublicIp}} + +v: 3 + +enableDebugOps: true + +identity: + cert: /home/{{$ssh_username}}/router-pki/{{$identity}}-client.cert + server_cert: /home/{{$ssh_username}}/router-pki/{{$identity}}-server.cert + key: /home/{{$ssh_username}}/router-pki/{{$identity}}.key + ca: /home/{{$ssh_username}}/router-pki/{{$identity}}-server.chain.pem + +tls: + handshakeTimeout: 30s + +ctrl: + endpoints: {{ range $host := .Model.MustSelectHosts "component.ctrl" 1 }} + - tls:{{ $host.PublicIp }}:6262{{end}} + startupTimeout: 5m + +healthChecks: + ctrlPingCheck: + # How often to ping the controller over the control channel. Defaults to 30 seconds + interval: 30s + # When to timeout the ping. Defaults to 15 seconds + timeout: 15s + # How long to wait before pinging the controller. Defaults to 15 seconds + initialDelay: 15s + +metrics: + reportInterval: 15s + messageQueueSize: 10 + +link: +{{if .Component.HasTag "public"}} + listeners: + - binding: transport + bind: tls:0.0.0.0:60{{printf "%02d" .Component.ScaleIndex }} + advertise: tls:{{$router_ip}}:60{{printf "%02d" .Component.ScaleIndex }} +{{end}} + dialers: + - binding: transport + options: + connectTimeout: 30s + +listeners: +{{if .Component.HasTag "tunneler"}} + - binding: tunnel + options: + mode: host +{{end}} + - binding: edge + address: tls:0.0.0.0:62{{printf "%02d" .Component.ScaleIndex }} + options: + # (required) The public hostname and port combination that Ziti SDKs should connect on. Previously this was in the chanIngress section. + advertise: {{ .Host.PublicIp }}:62{{printf "%02d" .Component.ScaleIndex }} + +# By having an 'edge' section defined, the ziti-router will attempt to parse the edge configuration. Removing this +# section, commenting out, or altering the name of the section will cause the router to no longer operate as an Edge +# Router. +edge: + # (required) Information used to generate the initial registration CSR. For documentation on these fields please + # refer to the openssl documentation. These values MUST be supplied and have no defaults. + csr: + country: US + province: NC + locality: Charlotte + organization: NetFoundry + organizationalUnit: Ziti + + # (required) SANs that this Gateways certs should contain. At least one IP or DNS SAN should be defined that matches + # the edge listeners "advertise" value from the "listeners" section. + sans: + ip: + - {{ .Host.PublicIp }} + diff --git a/zititest/models/db-ctrl-test/main.go b/zititest/models/db-ctrl-test/main.go new file mode 100644 index 000000000..9d14e3d25 --- /dev/null +++ b/zititest/models/db-ctrl-test/main.go @@ -0,0 +1,382 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "bufio" + "embed" + _ "embed" + "fmt" + "github.com/openziti/fablab" + "github.com/openziti/fablab/kernel/lib/actions" + "github.com/openziti/fablab/kernel/lib/actions/component" + "github.com/openziti/fablab/kernel/lib/actions/host" + "github.com/openziti/fablab/kernel/lib/actions/semaphore" + "github.com/openziti/fablab/kernel/lib/binding" + "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key" + "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore" + "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform" + distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution" + "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync" + aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key" + "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/fablab/resources" + "github.com/openziti/foundation/v2/stringz" + "github.com/openziti/ziti/zititest/models/test_resources" + "github.com/openziti/ziti/zititest/zitilab" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/models" + "go.etcd.io/bbolt" + "os" + "path" + "strings" + "time" +) + +const ( + EnrollIdentitiesAction = "enrollIdentities" +) + +const ( + useLatestZitiOnRouters = true +) + +//go:embed configs +var configResource embed.FS + +var dbStrategyInstance = &dbStrategy{ + routerMappings: map[string]string{}, +} + +type dbStrategy struct { + routerMappings map[string]string +} + +func (self *dbStrategy) ProcessDbModel(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error { + dbFile := self.GetDbFile(m) + dbDir := path.Dir(dbFile) + mapFile := path.Join(dbDir, "edge-router-mapping.csv") + mapContents, err := os.ReadFile(mapFile) + if err != nil { + return fmt.Errorf("failed to read edge router mapping file (%w)", err) + } + + if err = self.loadErMap(string(mapContents)); err != nil { + return fmt.Errorf("unable to parse router mapping file (%w)", err) + } + + if err = self.ProcessEdgeRouters(tx, m, builder); err != nil { + return err + } + + if err = self.CreateEnrollIdentitiesAction(tx, m, builder); err != nil { + return err + } + + return nil +} + +func (self *dbStrategy) loadErMap(csv string) error { + sc := bufio.NewScanner(strings.NewReader(csv)) + for sc.Scan() { + parts := strings.Split(sc.Text(), ",") + if len(parts) >= 2 { + routerId := parts[0] + nfVersion := parts[1] + zitiVersion := nfZitiVersionMap[nfVersion] + self.routerMappings[routerId] = zitiVersion + } + } + return nil +} + +func (self *dbStrategy) GetDbFile(m *model.Model) string { + return m.MustStringVariable("db_file") +} + +func (self *dbStrategy) ProcessEdgeRouters(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error { + ids, _, err := builder.GetStores().EdgeRouter.QueryIds(tx, "true limit none") + if err != nil { + return err + } + + versions := map[string]int{} + hostsPerRegion := ((len(ids) / len(sites)) / 20) + 1 + + hostIdF := func(regionId string, idx int) string { + return fmt.Sprintf("%s-r%d", regionId[:len(regionId)-2], idx) + } + + echoServerIdF := func(regionId string, idx int) string { + return fmt.Sprintf("%s-echo%d", regionId[:len(regionId)-2], idx) + } + + simCount := 0 + + for _, site := range sites { + regionId := site[:len(site)-1] + region := m.Regions[regionId] + + if region != nil && region.Site != site { + return fmt.Errorf("trying to add region for site %v, but one exists, with different site", site) + } + + if region == nil { + region = &model.Region{ + Id: regionId, + Scope: model.Scope{Tags: model.Tags{}}, + Region: regionId, + Site: site, + Hosts: model.Hosts{}, + } + m.Regions[regionId] = region + } + + for i := 0; i < 10; i++ { + simCount++ + simId := fmt.Sprintf("sim%02d", simCount) + region.Hosts[simId] = &model.Host{ + InstanceType: "t3.small", + Components: model.Components{ + simId: { + Scope: model.Scope{Tags: model.Tags{"sim"}}, + Type: &zitilab.SimpleSimType{ + ConfigPath: "identities", + }, + }, + }, + } + } + + for i := 0; i < hostsPerRegion; i++ { + hostId := hostIdF(regionId, i) + routerHost := &model.Host{ + Id: hostId, + Scope: model.Scope{Tags: model.Tags{}}, + Region: region, + Components: model.Components{ + echoServerIdF(regionId, i): { + Scope: model.Scope{Tags: model.Tags{"server"}}, + Type: &zitilab.EchoServerType{ + Port: 8888, + }, + }, + }, + InstanceType: "t3.large", + ScaleIndex: uint32(i), + } + region.Hosts[hostId] = routerHost + } + } + + siteIdx := 0 + hostIdx := 0 + + for _, id := range ids { + er, err := builder.GetStores().EdgeRouter.LoadById(tx, id) + if err != nil { + return err + } + + if siteIdx >= len(sites) { + siteIdx = 0 + hostIdx++ + if hostIdx >= hostsPerRegion { + hostIdx = 0 + } + } + + site := sites[siteIdx] + regionId := site[:len(site)-1] + region := m.Regions[regionId] + hostId := hostIdF(regionId, hostIdx) + routerHost := region.Hosts[hostId] + + siteIdx++ + + version := "" + if !useLatestZitiOnRouters { + version = self.routerMappings[id] + if replacement, found := versionMapping[version]; found { + version = replacement + } + versions[version] = versions[version] + 1 + } + + routerComponent := &model.Component{ + Scope: model.Scope{Tags: model.Tags{"router", "pre-created"}}, + Type: &zitilab.RouterType{ + Version: version, + }, + Host: routerHost, + ScaleIndex: uint32(len(routerHost.Components)), + } + + routerHost.Components[er.Id] = routerComponent + + if er.IsTunnelerEnabled { + routerComponent.Scope.Tags = append(routerComponent.Scope.Tags, "tunneler") + } + + if stringz.Contains(er.RoleAttributes, "public") { + routerComponent.Scope.Tags = append(routerComponent.Scope.Tags, "public") + } + } + + return nil +} + +func (self *dbStrategy) CreateEnrollIdentitiesAction(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error { + ids, _, err := builder.GetStores().Identity.QueryIds(tx, `type != "Router" limit none`) + if err != nil { + return err + } + + m.Actions[EnrollIdentitiesAction] = func(m *model.Model) model.Action { + return model.ActionFunc(func(run model.Run) error { + return provisionIdentities(ids, run) + }) + } + + return nil +} + +var m = &model.Model{ + Id: "db-ctrl-test", + Scope: model.Scope{ + Defaults: model.Variables{ + "environment": "ctrl-test", + "credentials": model.Variables{ + "aws": model.Variables{ + "managed_key": true, + }, + "ssh": model.Variables{ + "username": "ubuntu", + }, + "edge": model.Variables{ + "username": "admin", + }, + }, + }, + }, + StructureFactories: []model.Factory{ + &models.ZitiDbBuilder{Strategy: dbStrategyInstance}, + }, + Resources: model.Resources{ + resources.Configs: resources.SubFolder(configResource, "configs"), + resources.Binaries: os.DirFS(path.Join(os.Getenv("GOPATH"), "bin")), + resources.Terraform: test_resources.TerraformResources(), + }, + Regions: model.Regions{ + "us-east-1": { + Region: "us-east-1", + Site: "us-east-1c", + Hosts: model.Hosts{ + "ctrl": { + InstanceType: "c5.9xlarge", + InstanceResourceType: "ondemand_iops", + EC2: model.EC2Host{ + Volume: model.EC2Volume{ + Type: "gp3", + SizeGB: 16, + IOPS: 3000, + }, + }, + Components: model.Components{ + "ctrl": { + Scope: model.Scope{Tags: model.Tags{"ctrl"}}, + Type: &zitilab.ControllerType{}, + }, + }, + }, + }, + }, + }, + + Actions: model.ActionBinders{ + "bootstrap": model.ActionBinder(func(m *model.Model) model.Action { + workflow := actions.Workflow() + + workflow.AddAction(component.StopInParallel("*", 200)) + workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*")) + + workflow.AddAction(component.Start("#ctrl")) + workflow.AddAction(semaphore.Sleep(2 * time.Second)) + + workflow.AddAction(edge.Login("#ctrl")) + + workflow.AddAction(model.ActionFunc(provisionRouters)) + workflow.AddAction(m.ExecuteAction("syncRouterJwts")) + workflow.AddAction(model.ActionFunc(enrollRouters)) + + workflow.AddAction(m.ExecuteAction(EnrollIdentitiesAction)) + workflow.AddAction(model.ActionFunc(func(run model.Run) error { + src := model.MakeBuildPath("identities") + return rsync.RsyncSelected("component.sim", src, "identities").Execute(run) + })) + return workflow + }), + "syncRouterJwts": model.Bind(model.ActionFunc(func(run model.Run) error { + src := model.MakeBuildPath("router-jwts") + return rsync.RsyncSelected("component.router", src, "router-jwts").Execute(run) + })), + "stop": model.Bind(component.StopInParallel("*", 200)), + "clean": model.Bind(actions.Workflow( + component.StopInParallel("*", 200), + host.GroupExec("*", 25, "rm -f logs/*"), + )), + "login": model.Bind(edge.Login("#ctrl")), + }, + + Infrastructure: model.Stages{ + aws_ssh_key.Express(), + terraform_0.Express(), + semaphore_0.Ready(90 * time.Second), + }, + + Distribution: model.Stages{ + distribution.DistributeSshKey("*"), + rsync.RsyncStaged(), + model.StageActionF(func(run model.Run) error { + quickRun, _ := run.GetModel().GetBoolVariable("quick_run") + _, targetedSync := run.GetModel().Scope.GetVariable("sync.target") + + if !quickRun && !targetedSync { + dbFile := dbStrategyInstance.GetDbFile(run.GetModel()) + deferred := rsync.NewRsyncHost("#ctrl", dbFile, "/home/ubuntu/ctrl.db") + return deferred.Execute(run) + } + return nil + }), + }, + + Disposal: model.Stages{ + terraform.Dispose(), + aws_ssh_key2.Dispose(), + }, +} + +func main() { + m.AddActivationActions("bootstrap") + + model.AddBootstrapExtension(binding.AwsCredentialsLoader) + model.AddBootstrapExtension(aws_ssh_key.KeyManager) + + fablab.InitModel(m) + fablab.Run() +} diff --git a/zititest/models/db-ctrl-test/model_constants.go b/zititest/models/db-ctrl-test/model_constants.go new file mode 100644 index 000000000..b4b903fa0 --- /dev/null +++ b/zititest/models/db-ctrl-test/model_constants.go @@ -0,0 +1,145 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +var versionMapping = map[string]string{ + "": "0.31.4", + "0.30.4": "0.30.5", + "0.31.0": "0.31.4", + "0.29.0": "0.28.1", + "0.25.2": "0.28.1", + "0.26.10": "0.28.1", +} + +var sites = []string{ + "us-east-1c", + "us-west-2b", + "eu-west-2a", + "eu-central-1a", + "ap-southeast-2a", +} + +var nfZitiVersionMap = map[string]string{ + "7.3.54": "0.24.3", + "7.3.77": "0.26.7", + "7.3.36": "0.20.13", + "7.3.45": "0.22.7", + "7.3.91": "0.28.1", + "7.3.58": "0.24.8", + "8.0.96": "0.31.0", + "7.3.32": "0.20.9", + "8.0.95": "0.30.5", + "7.3.92": "0.28.4", + "7.3.69": "0.25.13", + "7.3.57": "0.24.7", + "7.3.98": "0.31.4", + "7.3.71": "0.26.1", + "7.3.93": "0.29.0", + "7.3.22": "0.19.8", + "7.3.43": "0.22.4", + "7.3.31": "0.20.8", + "7.3.24": "0.19.10", + "7.3.87": "0.27.7", + "7.3.55": "0.24.4", + "7.3.86": "0.27.6", + "7.3.4": "0.17.5", + "7.3.25": "0.19.11", + "7.3.74": "0.26.4", + "7.3.81": "0.27.0", + "7.3.12": "0.18.9", + "8.0.98": "0.31.4", + "7.3.7": "0.18.3", + "7.3.51": "0.23.1", + "7.3.13": "0.18.10", + "7.3.5": "0.18.0", + "7.3.48": "0.22.10", + "7.3.30": "0.20.3", + "7.3.9": "0.18.5", + "7.3.94": "0.30.4", + "7.3.23": "0.19.9", + "7.3.89": "0.27.9", + "7.3.41": "0.22.2", + "8.0.94": "0.30.4", + "7.3.62": "0.25.2", + "7.3.19": "0.19.5", + "7.3.65": "0.25.5", + "7.3.67": "0.25.8", + "7.3.11": "0.18.8", + "7.3.73": "0.26.3", + "7.3.61": "0.24.13", + "7.3.14": "0.19.0", + "7.3.96": "0.31.0", + "7.3.82": "0.27.2", + "7.3.34": "0.20.11", + "7.3.78": "0.26.8", + "7.3.50": "0.23.0", + "7.3.90": "0.28.0", + "7.3.44": "0.22.6", + "7.3.56": "0.24.6", + "7.3.35": "0.20.12", + "7.3.64": "0.25.4", + "8.0.99": "0.32.2", + "7.3.49": "0.22.11", + "7.3.59": "0.24.9", + "7.3.88": "0.27.8", + "7.3.97": "0.31.2", + "7.3.52": "0.24.1", + "7.3.99": "0.32.2", + "7.3.29": "0.20.2", + "7.3.72": "0.26.2", + "7.3.37": "0.20.14", + "7.3.53": "0.24.2", + "7.3.10": "0.18.7", + "7.3.33": "0.20.10", + "7.3.26": "0.19.12", + "7.3.15": "0.19.1", + "7.3.38": "0.21.0", + "7.3.63": "0.25.3", + "7.3.46": "0.22.8", + "7.3.28": "0.20.1", + "7.3.75": "0.26.5", + "7.3.39": "0.22.0", + "8.0.100": "0.33.1", + "7.3.60": "0.24.12", + "7.3.95": "0.30.5", + "7.3.21": "0.19.7", + "8.0.85": "0.27.5", + "7.3.18": "0.19.4", + "7.3.6": "0.18.2", + "7.3.76": "0.26.6", + "7.3.42": "0.22.3", + "7.3.16": "0.19.2", + "7.3.20": "0.19.6", + "8.0.97": "0.31.2", + "7.3.70": "0.26.0", + "7.3.68": "0.25.10", + "7.3.66": "0.25.6", + "7.3.84": "0.27.4", + "7.3.8": "0.18.4", + "7.3.27": "0.19.13", + "8.0.93": "0.29.0", + "7.3.85": "0.27.5", + "7.3.17": "0.19.3", + "7.3.80": "0.26.11", + "7.3.100": "0.33.1", + "7.3.79": "0.26.10", + "7.3.40": "0.22.1", + "8.0.92": "0.28.4", + "7.3.83": "0.27.3", + "8.0.91": "0.28.1", + "7.3.47": "0.22.9", +} diff --git a/zititest/models/db-ctrl-test/provision.go b/zititest/models/db-ctrl-test/provision.go new file mode 100644 index 000000000..a564efac8 --- /dev/null +++ b/zititest/models/db-ctrl-test/provision.go @@ -0,0 +1,99 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib/parallel" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab" + zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" + "github.com/openziti/ziti/zititest/zitilab/cli" + "os" + "path/filepath" +) + +func provisionIdentities(identities []string, run model.Run) error { + var tasks []parallel.Task + + identitiesDir := model.MakeBuildPath("identities") + _ = os.MkdirAll(identitiesDir, 0770) + for _, v := range identities { + id := v + task := func() error { + jwtFileName := filepath.Join(identitiesDir, id+".jwt") + args := []string{"create", "enrollment", "ott", "--jwt-output-file", jwtFileName, "--", id} + + if err := zitilib_actions.EdgeExec(run.GetModel(), args...); err != nil { + return err + } + + if _, err := cli.Exec(m, "edge", "enroll", "--", jwtFileName); err != nil { + return fmt.Errorf("failed to enroll %s (%w)", id, err) + } + return nil + } + tasks = append(tasks, task) + } + + if err := parallel.Execute(tasks, 10); err != nil { + return err + } + + return nil +} + +func provisionRouters(run model.Run) error { + routerPkiDir := model.MakeBuildPath("router-jwts") + _ = os.MkdirAll(routerPkiDir, 0770) + + return run.GetModel().ForEachComponent(".router.pre-created", 15, func(c *model.Component) error { + jwtFileName := filepath.Join(routerPkiDir, c.Id+".jwt") + args := []string{"re-enroll", "edge-router", "-j", "--jwt-output-file", jwtFileName, "--", c.Id} + return zitilib_actions.EdgeExec(c.GetModel(), args...) + }) +} + +func enrollRouters(run model.Run) error { + zitiVersion := "" + ctrls := run.GetModel().SelectComponents(".ctrl") + for _, ctrl := range ctrls { + if ctrl.Type != nil { + zitiVersion = ctrl.Type.GetVersion() + break + } + } + + return run.GetModel().ForEachHost("component.router", 15, func(h *model.Host) error { + var cmds []string + cmds = append(cmds, "mkdir -p /home/ubuntu/router-pki/") + hostUser := h.GetSshUser() + for _, c := range h.Components { + var routerType *zitilab.RouterType + var ok bool + if routerType, ok = c.Type.(*zitilab.RouterType); !ok || !c.HasTag("pre-created") { + continue + } + + remoteJwt := fmt.Sprintf("/home/%s/router-jwts/%s.jwt", hostUser, c.Id) + tmpl := "set -o pipefail; %s router enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log " + cmd := fmt.Sprintf(tmpl, zitilab.GetZitiBinaryPath(c, zitiVersion), routerType.GetConfigName(c), remoteJwt, c.Id) + cmds = append(cmds, cmd) + } + return h.ExecLogOnlyOnError(cmds...) + }) +} diff --git a/zititest/models/db-sdk-hosting-test/main.go b/zititest/models/db-sdk-hosting-test/main.go index 9ff26d6a3..ff6ddfe1d 100644 --- a/zititest/models/db-sdk-hosting-test/main.go +++ b/zititest/models/db-sdk-hosting-test/main.go @@ -74,7 +74,7 @@ func (d dbStrategy) PostProcess(router *db.EdgeRouter, c *model.Component) { } func (d dbStrategy) ProcessDbModel(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error { - if err := builder.CreateEdgeRouterHosts(tx, m); err != nil { + if err := builder.CreateEdgeRouterHosts(tx, m, d); err != nil { return err } return d.CreateIdentityHosts(tx, m, builder) diff --git a/zititest/models/links-test/main.go b/zititest/models/links-test/main.go index 09d9207e0..653ce3869 100644 --- a/zititest/models/links-test/main.go +++ b/zititest/models/links-test/main.go @@ -100,7 +100,7 @@ var m = &model.Model{ }), model.FactoryFunc(func(m *model.Model) error { return m.ForEachHost("component.router", 1, func(host *model.Host) error { - host.InstanceType = "c5.xlarge" + host.InstanceType = "t3.large" return nil }) }), diff --git a/zititest/models/links-test/validation.go b/zititest/models/links-test/validation.go index d3e9ba417..60cfb5915 100644 --- a/zititest/models/links-test/validation.go +++ b/zititest/models/links-test/validation.go @@ -75,12 +75,29 @@ func validateLinksForCtrl(c *model.Component, deadline time.Time) error { password := c.MustStringVariable("credentials.edge.password") edgeApiBaseUrl := c.Host.PublicIp + ":1280" - clients, err := zitirest.NewManagementClients(edgeApiBaseUrl) - if err != nil { - return err - } - if err = clients.Authenticate(username, password); err != nil { - return err + var clients *zitirest.Clients + loginStart := time.Now() + for { + var err error + clients, err = zitirest.NewManagementClients(edgeApiBaseUrl) + if err != nil { + if time.Since(loginStart) > time.Minute { + return err + } + pfxlog.Logger().WithField("ctrlId", c.Id).WithError(err).Info("failed to initialize mgmt client, trying again in 1s") + time.Sleep(time.Second) + continue + } + + if err = clients.Authenticate(username, password); err != nil { + if time.Since(loginStart) > time.Minute { + return err + } + pfxlog.Logger().WithField("ctrlId", c.Id).WithError(err).Info("failed to authenticate, trying again in 1s") + time.Sleep(time.Second) + } else { + break + } } allLinksPresent := false diff --git a/zititest/models/router-test/main.go b/zititest/models/router-test/main.go index d7f01a097..eed67d9d2 100644 --- a/zititest/models/router-test/main.go +++ b/zititest/models/router-test/main.go @@ -58,7 +58,7 @@ func (self scaleStrategy) GetEntityCount(entity model.Entity) uint32 { type dbStrategy struct{} func (d dbStrategy) ProcessDbModel(tx *bbolt.Tx, m *model.Model, builder *models.ZitiDbBuilder) error { - return builder.CreateEdgeRouterHosts(tx, m) + return builder.CreateEdgeRouterHosts(tx, m, d) } func (d dbStrategy) GetDbFile(*model.Model) string { diff --git a/zititest/models/sdk-hosting-test/main.go b/zititest/models/sdk-hosting-test/main.go index b1d173b01..ac5b76454 100644 --- a/zititest/models/sdk-hosting-test/main.go +++ b/zititest/models/sdk-hosting-test/main.go @@ -306,6 +306,11 @@ var m = &model.Model{ return parallel.Execute(tasks, 25) })) + workflow.AddAction(semaphore.Sleep(2 * time.Second)) + workflow.AddAction(component.StartInParallel(".router", 10)) + workflow.AddAction(semaphore.Sleep(2 * time.Second)) + workflow.AddAction(component.StartInParallel(".host", 50)) + return workflow }), "stop": model.Bind(component.StopInParallelHostExclusive("*", 15)), diff --git a/zititest/models/smart/stages.go b/zititest/models/smart/stages.go index f0772641d..a061b2634 100644 --- a/zititest/models/smart/stages.go +++ b/zititest/models/smart/stages.go @@ -68,7 +68,7 @@ func (self *stageFactory) Build(m *model.Model) error { distribution.Locations(models.HasRouterComponent, "logs"), distribution.Locations(models.LoopListenerTag, "logs"), distribution.Locations(models.LoopDialerTag, "logs"), - rsync.Rsync(1), + rsync.RsyncStaged(), } // Run the bootstrap and start actions diff --git a/zititest/models/simple/actions/bootstrap.go b/zititest/models/smoke/actions/bootstrap.go similarity index 100% rename from zititest/models/simple/actions/bootstrap.go rename to zititest/models/smoke/actions/bootstrap.go diff --git a/zititest/models/simple/actions/start.go b/zititest/models/smoke/actions/start.go similarity index 100% rename from zititest/models/simple/actions/start.go rename to zititest/models/smoke/actions/start.go diff --git a/zititest/models/simple/configs/ctrl.yml.tmpl b/zititest/models/smoke/configs/ctrl.yml.tmpl similarity index 100% rename from zititest/models/simple/configs/ctrl.yml.tmpl rename to zititest/models/smoke/configs/ctrl.yml.tmpl diff --git a/zititest/models/simple/configs/router.yml.tmpl b/zititest/models/smoke/configs/router.yml.tmpl similarity index 100% rename from zititest/models/simple/configs/router.yml.tmpl rename to zititest/models/smoke/configs/router.yml.tmpl diff --git a/zititest/models/simple/simple.go b/zititest/models/smoke/simple.go similarity index 98% rename from zititest/models/simple/simple.go rename to zititest/models/smoke/simple.go index 459b911bc..d6853c774 100644 --- a/zititest/models/simple/simple.go +++ b/zititest/models/smoke/simple.go @@ -14,7 +14,7 @@ limitations under the License. */ -package simple +package smoke import ( "embed" @@ -30,7 +30,7 @@ import ( "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform" "github.com/openziti/fablab/kernel/model" "github.com/openziti/fablab/resources" - "github.com/openziti/ziti/zititest/models/simple/actions" + "github.com/openziti/ziti/zititest/models/smoke/actions" "github.com/openziti/ziti/zititest/models/test_resources" "github.com/openziti/ziti/zititest/zitilab" "github.com/openziti/ziti/zititest/zitilab/actions/edge" @@ -208,7 +208,9 @@ var Model = &model.Model{ }, "echo-server": { Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}}, - Type: &zitilab.EchoServerType{}, + Type: &zitilab.EchoServerType{ + BindService: "echo", + }, }, "iperf-server-ert": { Scope: model.Scope{Tags: model.Tags{"iperf", "service"}}, diff --git a/zititest/models/simple-transfer/main.go b/zititest/models/smoketest/main.go similarity index 84% rename from zititest/models/simple-transfer/main.go rename to zititest/models/smoketest/main.go index 658cee65c..756216444 100644 --- a/zititest/models/simple-transfer/main.go +++ b/zititest/models/smoketest/main.go @@ -18,11 +18,11 @@ package main import ( "github.com/openziti/fablab" - "github.com/openziti/ziti/zititest/models/simple" + "github.com/openziti/ziti/zititest/models/smoke" ) func main() { - simple.InitBootstrapExtensions() - fablab.InitModel(simple.Model) + smoke.InitBootstrapExtensions() + fablab.InitModel(smoke.Model) fablab.Run() } diff --git a/zititest/models/test_resources/terraform/main.tf b/zititest/models/test_resources/terraform/main.tf index b101aa159..efde51cad 100644 --- a/zititest/models/test_resources/terraform/main.tf +++ b/zititest/models/test_resources/terraform/main.tf @@ -34,6 +34,11 @@ module "{{ $regionId }}_host_{{ $hostId }}" { spot_price = "{{ $host.SpotPrice }}" spot_type = "{{ $host.SpotType }}" name = "{{ $region.Model.MustVariable "environment" }}.{{ $host.Id }}" + {{ if not (eq $host.EC2.Volume.Type "") }} + volume_type = "{{ $host.EC2.Volume.Type }}" + volume_size = "{{ $host.EC2.Volume.SizeGB }}" + volume_iops = "{{ $host.EC2.Volume.IOPS }}" + {{ end }} } output "{{ $regionId }}_host_{{ $hostId }}_public_ip" { value = module.{{ $regionId }}_host_{{ $hostId }}.public_ip } diff --git a/zititest/models/test_resources/terraform/ondemand_iops_instance/main.tf b/zititest/models/test_resources/terraform/ondemand_iops_instance/main.tf new file mode 100644 index 000000000..e90664e41 --- /dev/null +++ b/zititest/models/test_resources/terraform/ondemand_iops_instance/main.tf @@ -0,0 +1,58 @@ +variable "name" {} +variable "access_key" {} +variable "secret_key" {} +variable "environment_tag" { default = "" } +variable "instance_type" {} +variable "key_name" {} +variable "key_path" {} +variable "region" {} +variable "security_group_id" {} +variable "ssh_user" { default = "ubuntu" } +variable "subnet_id" {} +variable "spot_price" {} +variable "spot_type" {} +variable "volume_type" { default = "gp2" } +variable "volume_size" { default = 4 } +variable "volume_iops" {} + +output "public_ip" { value = aws_instance.fablab.public_ip } +output "private_ip" { value = aws_instance.fablab.private_ip } + +provider "aws" { + access_key = var.access_key + secret_key = var.secret_key + region = var.region +} + +data "aws_ami" "ami" { + most_recent = true + owners = ["self"] + + filter { + name = "name" + values = ["ziti-tests-*"] + } +} + +resource "aws_instance" "fablab" { + ami = data.aws_ami.ami.id + instance_type = var.instance_type + key_name = var.key_name + vpc_security_group_ids = [var.security_group_id] + subnet_id = var.subnet_id + associate_public_ip_address = true + + root_block_device { + volume_type = var.volume_type + volume_size = var.volume_size + iops = var.volume_iops + } + + tags = { + Name = var.name + cost_environment = "non-production" + cost_category = "CICD" + cost_team_owner = "ADVDEV" + source = "fablab" + } +} diff --git a/zititest/simple-sim/simple-sim.go b/zititest/simple-sim/simple-sim.go new file mode 100644 index 000000000..b334cd805 --- /dev/null +++ b/zititest/simple-sim/simple-sim.go @@ -0,0 +1,215 @@ +/* + Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/edge-api/rest_model" + "github.com/openziti/sdk-golang/ziti" + "github.com/openziti/ziti/common/eid" + cmap "github.com/orcaman/concurrent-map/v2" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "math/rand" + "os" + "path/filepath" + "slices" + "time" +) + +type simpleSimAction struct { + verbose bool + logFormatter string + configFile string + + authenticated cmap.ConcurrentMap[string, time.Time] +} + +func newSimpleSimCmd() *cobra.Command { + action := &simpleSimAction{ + authenticated: cmap.New[time.Time](), + } + + cmd := &cobra.Command{ + Use: "simple-sim ", + Short: "Generates traffic to one more ziti identities", + Args: cobra.ExactArgs(1), + Run: action.run, + } + + // allow interspersing positional args and flags + cmd.Flags().SetInterspersed(true) + cmd.Flags().BoolVarP(&action.verbose, "verbose", "v", false, "Enable verbose logging") + cmd.Flags().StringVar(&action.logFormatter, "log-formatter", "", "Specify log formatter [json|pfxlog|text]") + cmd.Flags().StringVarP(&action.configFile, "identity", "i", "", "Specify the Ziti identity to use. If not specified the Ziti listener won't be started") + cmd.Flags().SetInterspersed(true) + + return cmd +} + +func (self *simpleSimAction) initLogging() { + logLevel := logrus.InfoLevel + if self.verbose { + logLevel = logrus.DebugLevel + } + + options := pfxlog.DefaultOptions().SetTrimPrefix("github.com/openziti/").NoColor() + pfxlog.GlobalInit(logLevel, options) + + switch self.logFormatter { + case "pfxlog": + pfxlog.SetFormatter(pfxlog.NewFormatter(pfxlog.DefaultOptions().SetTrimPrefix("github.com/openziti/").StartingToday())) + case "json": + pfxlog.SetFormatter(&logrus.JSONFormatter{TimestampFormat: "2006-01-02T15:04:05.000Z"}) + case "text": + pfxlog.SetFormatter(&logrus.TextFormatter{}) + default: + // let logrus do its own thing + } +} + +func (self *simpleSimAction) run(_ *cobra.Command, args []string) { + self.initLogging() + + log := pfxlog.Logger() + + fsInfo, err := os.Stat(args[0]) + if err != nil { + log.WithError(err).Fatalf("unable to stat config file/dir %s", args[0]) + return + } + + simCount := 0 + if fsInfo.IsDir() { + files, err := os.ReadDir(args[0]) + if err != nil { + log.WithError(err).Fatalf("failed to scan directory %s:", args[0]) + return + } + + for _, file := range files { + if filepath.Ext(file.Name()) == ".json" { + fn, err := filepath.Abs(filepath.Join(args[0], file.Name())) + if err != nil { + log.Fatalf("failed to listing file %s: %v", file.Name(), err) + } + self.startSimForIdentity(fn) + simCount++ + } + } + } + + for { + time.Sleep(time.Second * 5) + pfxlog.Logger().Infof("%d/%d identities authenticated", self.authenticated.Count(), simCount) + } +} + +func (self *simpleSimAction) startSimForIdentity(path string) { + log := pfxlog.Logger().WithField("config", path) + + zitiConfig, err := ziti.NewConfigFromFile(path) + if err != nil { + log.WithError(err).Fatal("unable to load ziti identity") + return + } + + zitiContext, err := ziti.NewContext(zitiConfig) + if err != nil { + log.WithError(err).Fatal("could not create sdk context from config") + } + + go self.runSim(path, zitiContext) +} + +func (self *simpleSimAction) runSim(path string, ctx ziti.Context) { + log := pfxlog.Logger().WithField("config", path) + + var lastLog time.Time + cycle := 0 + + for { + cycle++ + validServices := 0 + dialSuccesses := 0 + services, err := ctx.GetServices() + if err != nil { + self.authenticated.Remove(ctx.GetId()) + log.WithField("cycle", cycle).WithError(err).Error("unable to list services") + time.Sleep(time.Second * 5) + continue + } else { + self.authenticated.Set(ctx.GetId(), time.Now()) + } + + for _, service := range services { + if !slices.Contains(service.Permissions, rest_model.DialBindDial) { + continue + } + validServices++ + + svcLog := log.WithField("svc", *service.Name).WithField("cycle", cycle) + conn, err := ctx.Dial(*service.Name) + if err != nil { + svcLog.WithError(err).Error("unable to dial service") + } else { + randomString := eid.New() + if _, err = conn.Write([]byte(randomString)); err != nil { + svcLog.WithError(err).Error("unable to write") + } else { + result := make([]byte, len(randomString)) + n, err := conn.Read(result) + if err != nil { + svcLog.WithError(err).Error("unable to read") + } else { + result = result[:n] + if string(result) != randomString { + svcLog.WithError(err).Error("%s != %s", randomString, result) + } else { + dialSuccesses++ + } + } + } + } + + if conn != nil { + if err = conn.Close(); err != nil { + svcLog.WithError(err).Error("error closing conn") + } + time.Sleep(time.Millisecond * 250) + } + } + + if time.Since(lastLog) > time.Minute { + log.WithField("serviceCount", len(services)). + WithField("cycle", cycle). + WithField("successfulDials", fmt.Sprintf("%d/%d", dialSuccesses, validServices)). + Info("completed") + lastLog = time.Now() + } + + time.Sleep(time.Duration(500+rand.Intn(1000)) * time.Second) + } +} + +func main() { + cli := newSimpleSimCmd() + if err := cli.Execute(); err != nil { + pfxlog.Logger().Fatal(err.Error()) + } +} diff --git a/zititest/tests/model_test.go b/zititest/tests/model_test.go index 0d22ea134..2b505a409 100644 --- a/zititest/tests/model_test.go +++ b/zititest/tests/model_test.go @@ -19,7 +19,7 @@ package tests import ( "github.com/openziti/fablab" "github.com/openziti/fablab/kernel/model" - "github.com/openziti/ziti/zititest/models/simple" + "github.com/openziti/ziti/zititest/models/smoke" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -33,9 +33,9 @@ func init() { panic(errors.Errorf("no instance found for current instance id %v", cfg.GetSelectedInstanceId())) } - if instance.Model == simple.Model.Id { - simple.InitBootstrapExtensions() - fablab.InitModel(simple.Model) + if instance.Model == smoke.Model.Id { + smoke.InitBootstrapExtensions() + fablab.InitModel(smoke.Model) } else { panic(errors.Errorf("unsupported model for network tests [%v]", instance.Model)) } diff --git a/zititest/ziti-fabric-test/main.go b/zititest/ziti-fabric-test/main.go index c4b1ecefa..34179bc20 100644 --- a/zititest/ziti-fabric-test/main.go +++ b/zititest/ziti-fabric-test/main.go @@ -19,8 +19,6 @@ package main import ( - "math/rand" - "github.com/michaelquigley/pfxlog" "github.com/openziti/transport/v2" "github.com/openziti/transport/v2/tcp" @@ -28,7 +26,6 @@ import ( "github.com/openziti/transport/v2/transwarp" "github.com/openziti/transport/v2/transwarptls" "github.com/openziti/transport/v2/wss" - "github.com/openziti/foundation/v2/info" "github.com/openziti/ziti/zititest/ziti-fabric-test/subcmd" _ "github.com/openziti/ziti/zititest/ziti-fabric-test/subcmd/client" _ "github.com/openziti/ziti/zititest/ziti-fabric-test/subcmd/loop2" @@ -46,6 +43,5 @@ func init() { } func main() { - rand.Seed(info.NowInMilliseconds()) subcmd.Execute() } diff --git a/zititest/zitilab/cli/cli.go b/zititest/zitilab/cli/cli.go index dfd4b1e4f..4b243c7b5 100644 --- a/zititest/zitilab/cli/cli.go +++ b/zititest/zitilab/cli/cli.go @@ -39,7 +39,7 @@ func Exec(m *model.Model, args ...string) (string, error) { ziticli.SetArgs(args) logrus.Infof("executing: %s", strings.Join(args, " ")) if err := ziticli.Execute(); err != nil { - logrus.Errorf("err executing command, err:[%e]", err) + logrus.WithError(err).Error("err executing command") } return cliOut.String(), nil diff --git a/zititest/zitilab/component_common.go b/zititest/zitilab/component_common.go index 31d23922a..6174f7929 100644 --- a/zititest/zitilab/component_common.go +++ b/zititest/zitilab/component_common.go @@ -38,7 +38,7 @@ func getZitiProcessFilter(c *model.Component, zitiType string) func(string) bool func startZitiComponent(c *model.Component, zitiType string, version string, configName string) error { user := c.GetHost().GetSshUser() - binaryPath := getZitiBinaryPath(c, version) + binaryPath := GetZitiBinaryPath(c, version) configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s", user, configName) logsPath := fmt.Sprintf("/home/%s/logs/%s.log", user, c.Id) @@ -74,7 +74,7 @@ func canonicalizeGoAppVersion(version *string) { } } -func getZitiBinaryPath(c *model.Component, version string) string { +func GetZitiBinaryPath(c *model.Component, version string) string { return getBinaryPath(c, "ziti", version) } @@ -114,3 +114,19 @@ func reEnrollIdentity(run model.Run, c *model.Component, zitiBinaryPath string, return c.GetHost().ExecLogOnlyOnError(cmd) } + +func setupDnsForTunneler(c *model.Component) error { + key := "ziti_tunnel.resolve_setup_done" + if _, found := c.Host.Data[key]; !found { + cmds := []string{ + "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", + "sudo systemctl restart systemd-resolved", + } + if err := c.Host.ExecLogOnlyOnError(cmds...); err != nil { + return err + } + c.Host.Data[key] = true + return nil + } + return nil +} diff --git a/zititest/zitilab/component_controller.go b/zititest/zitilab/component_controller.go index d7ded6f76..d641318f7 100644 --- a/zititest/zitilab/component_controller.go +++ b/zititest/zitilab/component_controller.go @@ -45,6 +45,14 @@ type ControllerType struct { DNSNames []string } +func (self *ControllerType) Label() string { + return "ziti-controller" +} + +func (self *ControllerType) GetVersion() string { + return self.Version +} + func (self *ControllerType) InitType(*model.Component) { canonicalizeGoAppVersion(&self.Version) } @@ -104,7 +112,15 @@ func (self *ControllerType) IsRunning(_ model.Run, c *model.Component) (bool, er return len(pids) > 0, nil } -func (self *ControllerType) Start(_ model.Run, c *model.Component) error { +func (self *ControllerType) Start(r model.Run, c *model.Component) error { + isRunninng, err := self.IsRunning(r, c) + if err != nil { + return err + } + if isRunninng { + fmt.Printf("controller %s already started\n", c.Id) + return nil + } return startZitiComponent(c, "controller", self.Version, self.getConfigName(c)) } @@ -113,7 +129,7 @@ func (self *ControllerType) Stop(_ model.Run, c *model.Component) error { } func (self *ControllerType) GetBinaryPath(c *model.Component) string { - return getZitiBinaryPath(c, self.Version) + return GetZitiBinaryPath(c, self.Version) } func (self *ControllerType) InitStandalone(run model.Run, c *model.Component) error { @@ -130,7 +146,7 @@ func (self *ControllerType) InitStandalone(run model.Run, c *model.Component) er factory := c.GetHost().NewSshConfigFactory() - binaryPath := getZitiBinaryPath(c, self.Version) + binaryPath := GetZitiBinaryPath(c, self.Version) configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s", factory.User(), self.getConfigName(c)) tmpl := "rm -f /home/%v/fablab/ctrl.db && set -o pipefail; %s controller --log-formatter pfxlog edge init %s -u %s -p %s 2>&1 | tee logs/controller.edge.init.log" @@ -142,7 +158,7 @@ func (self *ControllerType) InitRaft(run model.Run, c *model.Component) error { count := 0 err := run.GetModel().ForEachComponent("*", 1, func(other *model.Component) error { if _, ok := c.Type.(*ControllerType); ok { - binaryPath := getZitiBinaryPath(c, self.Version) + binaryPath := GetZitiBinaryPath(c, self.Version) tmpl := "%s agent cluster add %v --id %v" cmd := fmt.Sprintf(tmpl, binaryPath, "tls:"+other.Host.PublicIp+":6262", other.Id) count++ diff --git a/zititest/zitilab/component_echo_server.go b/zititest/zitilab/component_echo_server.go index 6ea0e7f98..ba9279e5b 100644 --- a/zititest/zitilab/component_echo_server.go +++ b/zititest/zitilab/component_echo_server.go @@ -1,3 +1,19 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package zitilab import ( @@ -13,8 +29,18 @@ var _ model.ComponentType = (*EchoServerType)(nil) type EchoServerMode int type EchoServerType struct { - Version string - LocalPath string + Version string + LocalPath string + Port uint16 + BindService string +} + +func (self *EchoServerType) Label() string { + return "echo-server" +} + +func (self *EchoServerType) GetVersion() string { + return self.Version } func (self *EchoServerType) InitType(*model.Component) { @@ -48,7 +74,7 @@ func (self *EchoServerType) IsRunning(_ model.Run, c *model.Component) (bool, er func (self *EchoServerType) Start(run model.Run, c *model.Component) error { user := c.GetHost().GetSshUser() - binaryPath := getZitiBinaryPath(c, self.Version) + binaryPath := GetZitiBinaryPath(c, self.Version) configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s.json", user, c.Id) logsPath := fmt.Sprintf("/home/%s/logs/%s.log", user, c.Id) @@ -57,8 +83,18 @@ func (self *EchoServerType) Start(run model.Run, c *model.Component) error { ha = "--ha" } - serviceCmd := fmt.Sprintf("nohup %s demo echo-server --cli-agent-alias %s %s -i %s > %s 2>&1 &", - binaryPath, c.Id, ha, configPath, logsPath) + serviceHostingFlags := "" + if self.BindService != "" { + serviceHostingFlags = fmt.Sprintf("-i %s -s %s", configPath, self.BindService) + } + + portFlag := "" + if self.Port > 0 { + portFlag = fmt.Sprintf("-p %d", self.Port) + } + + serviceCmd := fmt.Sprintf("nohup %s demo echo-server --cli-agent-alias %s %s %s %s > %s 2>&1 &", + binaryPath, c.Id, ha, serviceHostingFlags, portFlag, logsPath) value, err := c.GetHost().ExecLogged(serviceCmd) if err != nil { diff --git a/zititest/zitilab/component_iperf.go b/zititest/zitilab/component_iperf.go index eedac0073..738b1f16d 100644 --- a/zititest/zitilab/component_iperf.go +++ b/zititest/zitilab/component_iperf.go @@ -29,6 +29,14 @@ type IPerfServerType struct { Port uint16 } +func (self *IPerfServerType) Label() string { + return "iperf-server" +} + +func (self *IPerfServerType) GetVersion() string { + return "os-provided" +} + func (self *IPerfServerType) Dump() any { return map[string]string{ "type_id": "iperf-server", diff --git a/zititest/zitilab/component_router.go b/zititest/zitilab/component_router.go index 23dd92de3..89123e0e9 100644 --- a/zititest/zitilab/component_router.go +++ b/zititest/zitilab/component_router.go @@ -19,7 +19,6 @@ package zitilab import ( "fmt" "github.com/openziti/fablab/kernel/lib" - "github.com/openziti/fablab/kernel/lib/actions/host" "github.com/openziti/fablab/kernel/model" zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" "github.com/openziti/ziti/zititest/zitilab/stageziti" @@ -45,6 +44,14 @@ type RouterType struct { LocalPath string } +func (self *RouterType) Label() string { + return "ziti-router" +} + +func (self *RouterType) GetVersion() string { + return self.Version +} + func (self *RouterType) InitType(*model.Component) { canonicalizeGoAppVersion(&self.Version) } @@ -68,12 +75,7 @@ func (self *RouterType) Dump() any { func (self *RouterType) InitializeHost(run model.Run, c *model.Component) error { if self.isTunneler(c) { - cmds := []string{ - "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", - "sudo systemctl restart systemd-resolved", - "mkdir -p /home/ubuntu/logs", - } - return host.Exec(c.GetHost(), cmds...).Execute(run) + return setupDnsForTunneler(c) } return nil } @@ -84,7 +86,7 @@ func (self *RouterType) StageFiles(r model.Run, c *model.Component) error { configSource = "router.yml.tmpl" } - configName := self.getConfigName(c) + configName := self.GetConfigName(c) if err := lib.GenerateConfigForComponent(c, self.ConfigSourceFS, configSource, configName, r); err != nil { return err @@ -97,7 +99,7 @@ func (self *RouterType) isTunneler(c *model.Component) bool { return c.HasLocalOrAncestralTag("tunneler") } -func (self *RouterType) getConfigName(c *model.Component) string { +func (self *RouterType) GetConfigName(c *model.Component) string { configName := self.ConfigName if configName == "" { configName = c.Id + ".yml" @@ -126,7 +128,7 @@ func (self *RouterType) Start(r model.Run, c *model.Component) error { fmt.Printf("router %s already started\n", c.Id) return nil } - return startZitiComponent(c, "router", self.Version, self.getConfigName(c)) + return startZitiComponent(c, "router", self.Version, self.GetConfigName(c)) } func (self *RouterType) Stop(run model.Run, c *model.Component) error { @@ -174,15 +176,15 @@ func (self *RouterType) CreateAndEnroll(run model.Run, c *model.Component) error } tmpl := "set -o pipefail; %s router enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log " - cmd := fmt.Sprintf(tmpl, getZitiBinaryPath(c, self.Version), self.getConfigName(c), remoteJwt, c.Id) + cmd := fmt.Sprintf(tmpl, GetZitiBinaryPath(c, self.Version), self.GetConfigName(c), remoteJwt, c.Id) return c.GetHost().ExecLogOnlyOnError(cmd) } -func (self *RouterType) ReEnroll(_ model.Run, c *model.Component) error { +func (self *RouterType) ReEnroll(run model.Run, c *model.Component) error { jwtFileName := filepath.Join(model.ConfigBuild(), c.Id+".jwt") - args := []string{"re-enroll", "edge-router", c.Id, "-j", "--jwt-output-file", jwtFileName} + args := []string{"re-enroll", "edge-router", "-j", "--jwt-output-file", jwtFileName, "--", c.Id} if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil { return err @@ -193,8 +195,18 @@ func (self *RouterType) ReEnroll(_ model.Run, c *model.Component) error { return err } + zitiVersion := self.Version + + ctrls := run.GetModel().SelectComponents(".ctrl") + for _, ctrl := range ctrls { + if ctrl.Type != nil { + zitiVersion = ctrl.Type.GetVersion() + break + } + } + tmpl := "set -o pipefail; %s router enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log " - cmd := fmt.Sprintf(tmpl, getZitiBinaryPath(c, self.Version), self.getConfigName(c), remoteJwt, c.Id) + cmd := fmt.Sprintf(tmpl, GetZitiBinaryPath(c, zitiVersion), self.GetConfigName(c), remoteJwt, c.Id) return c.GetHost().ExecLogOnlyOnError(cmd) } diff --git a/zititest/zitilab/component_simple_sim.go b/zititest/zitilab/component_simple_sim.go new file mode 100644 index 000000000..319aad6a4 --- /dev/null +++ b/zititest/zitilab/component_simple_sim.go @@ -0,0 +1,94 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zitilab + +import ( + "fmt" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/stageziti" + "github.com/sirupsen/logrus" + "strings" +) + +var _ model.ComponentType = (*SimpleSimType)(nil) + +type SimpleSimType struct { + LocalPath string + ConfigPath string +} + +func (self *SimpleSimType) Label() string { + return "simple-sim" +} + +func (self *SimpleSimType) GetVersion() string { + return "local" +} + +func (self *SimpleSimType) Dump() any { + return map[string]string{ + "type_id": "simple-sim", + "local_path": self.LocalPath, + "config_path": self.ConfigPath, + } +} + +func (self *SimpleSimType) StageFiles(r model.Run, c *model.Component) error { + return stageziti.StageLocalOnce(r, "simple-sim", c, self.LocalPath) +} + +func (self *SimpleSimType) getProcessFilter() func(string) bool { + return func(s string) bool { + return strings.Contains(s, "simple-sim") + } +} + +func (self *SimpleSimType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + pids, err := c.GetHost().FindProcesses(self.getProcessFilter()) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *SimpleSimType) Start(_ model.Run, c *model.Component) error { + user := c.GetHost().GetSshUser() + + binaryPath := getBinaryPath(c, "simple-sim", "") + logsPath := fmt.Sprintf("/home/%s/logs/%s.log", user, c.Id) + configPath := fmt.Sprintf("/home/%s/%s", user, self.ConfigPath) + + serviceCmd := fmt.Sprintf("%s %s --log-formatter pfxlog > %s 2>&1 &", + binaryPath, configPath, logsPath) + + value, err := c.Host.ExecLogged( + "rm -f "+logsPath, + serviceCmd) + if err != nil { + return err + } + + if len(value) > 0 { + logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n")) + } + + return nil +} + +func (self *SimpleSimType) Stop(_ model.Run, c *model.Component) error { + return c.GetHost().KillProcesses("-TERM", self.getProcessFilter()) +} diff --git a/zititest/zitilab/component_zcat.go b/zititest/zitilab/component_zcat.go index 162867850..a4aeefc59 100644 --- a/zititest/zitilab/component_zcat.go +++ b/zititest/zitilab/component_zcat.go @@ -1,3 +1,19 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package zitilab import ( @@ -15,6 +31,14 @@ type ZCatType struct { LocalPath string } +func (self *ZCatType) Label() string { + return "zcat" +} + +func (self *ZCatType) GetVersion() string { + return self.Version +} + func (self *ZCatType) InitType(*model.Component) { canonicalizeGoAppVersion(&self.Version) } diff --git a/zititest/zitilab/component_ziti_edge_tunnel.go b/zititest/zitilab/component_ziti_edge_tunnel.go index a6fd4e147..67d3c8a6f 100644 --- a/zititest/zitilab/component_ziti_edge_tunnel.go +++ b/zititest/zitilab/component_ziti_edge_tunnel.go @@ -34,6 +34,14 @@ type ZitiEdgeTunnelType struct { ConfigPathF func(c *model.Component) string } +func (self *ZitiEdgeTunnelType) Label() string { + return "ziti-edge-tunnel" +} + +func (self *ZitiEdgeTunnelType) GetVersion() string { + return self.Version +} + func (self *ZitiEdgeTunnelType) GetActions() map[string]model.ComponentAction { return map[string]model.ComponentAction{ ZitiTunnelActionsReEnroll: model.ComponentActionF(self.ReEnroll), @@ -125,5 +133,5 @@ func (self *ZitiEdgeTunnelType) Stop(_ model.Run, c *model.Component) error { } func (self *ZitiEdgeTunnelType) ReEnroll(run model.Run, c *model.Component) error { - return reEnrollIdentity(run, c, getZitiBinaryPath(c, self.ZitiVersion), self.GetConfigPath(c)) + return reEnrollIdentity(run, c, GetZitiBinaryPath(c, self.ZitiVersion), self.GetConfigPath(c)) } diff --git a/zititest/zitilab/component_ziti_tunnel.go b/zititest/zitilab/component_ziti_tunnel.go index 9ddab77ac..e8b3d6d05 100644 --- a/zititest/zitilab/component_ziti_tunnel.go +++ b/zititest/zitilab/component_ziti_tunnel.go @@ -56,6 +56,14 @@ type ZitiTunnelType struct { ConfigPathF func(c *model.Component) string } +func (self *ZitiTunnelType) Label() string { + return "ziti-tunnel" +} + +func (self *ZitiTunnelType) GetVersion() string { + return self.Version +} + func (self *ZitiTunnelType) GetActions() map[string]model.ComponentAction { return map[string]model.ComponentAction{ ZitiTunnelActionsReEnroll: model.ComponentActionF(self.ReEnroll), @@ -80,18 +88,7 @@ func (self *ZitiTunnelType) StageFiles(r model.Run, c *model.Component) error { func (self *ZitiTunnelType) InitializeHost(_ model.Run, c *model.Component) error { if self.Mode == ZitiTunnelModeTproxy { - key := "ziti_tunnel.resolve_setup_done" - if _, found := c.Host.Data[key]; !found { - cmds := []string{ - "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", - "sudo systemctl restart systemd-resolved", - } - if err := c.Host.ExecLogOnlyOnError(cmds...); err != nil { - return err - } - c.Host.Data[key] = true - return nil - } + return setupDnsForTunneler(c) } return nil } @@ -120,7 +117,7 @@ func (self *ZitiTunnelType) Start(_ model.Run, c *model.Component) error { user := c.GetHost().GetSshUser() - binaryPath := getZitiBinaryPath(c, self.Version) + binaryPath := GetZitiBinaryPath(c, self.Version) configPath := self.GetConfigPath(c) logsPath := fmt.Sprintf("/home/%s/logs/%s.log", user, c.Id) @@ -151,5 +148,5 @@ func (self *ZitiTunnelType) Stop(_ model.Run, c *model.Component) error { } func (self *ZitiTunnelType) ReEnroll(run model.Run, c *model.Component) error { - return reEnrollIdentity(run, c, getZitiBinaryPath(c, self.Version), self.GetConfigPath(c)) + return reEnrollIdentity(run, c, GetZitiBinaryPath(c, self.Version), self.GetConfigPath(c)) } diff --git a/zititest/zitilab/component_zrok_controller.go b/zititest/zitilab/component_zrok_controller.go index 42a49797c..aa60c6dcb 100644 --- a/zititest/zitilab/component_zrok_controller.go +++ b/zititest/zitilab/component_zrok_controller.go @@ -47,6 +47,14 @@ type ZrokControllerType struct { PreCreateClients string } +func (self *ZrokControllerType) Label() string { + return "zrok-controller" +} + +func (self *ZrokControllerType) GetVersion() string { + return self.Version +} + func (self *ZrokControllerType) InitType(*model.Component) { canonicalizeGoAppVersion(&self.Version) } diff --git a/zititest/zitilab/component_zrok_frontend.go b/zititest/zitilab/component_zrok_frontend.go index e11412042..040825e8b 100644 --- a/zititest/zitilab/component_zrok_frontend.go +++ b/zititest/zitilab/component_zrok_frontend.go @@ -46,6 +46,14 @@ type ZrokFrontendType struct { ZrokCtrlSelector string } +func (self *ZrokFrontendType) Label() string { + return "zrok-frontend" +} + +func (self *ZrokFrontendType) GetVersion() string { + return self.Version +} + func (self *ZrokFrontendType) InitType(*model.Component) { canonicalizeGoAppVersion(&self.Version) if self.ZrokCtrlSelector == "" { diff --git a/zititest/zitilab/component_zrok_looptest.go b/zititest/zitilab/component_zrok_looptest.go index 26c6b971e..9939486ac 100644 --- a/zititest/zitilab/component_zrok_looptest.go +++ b/zititest/zitilab/component_zrok_looptest.go @@ -39,6 +39,14 @@ type ZrokLoopTestType struct { Pacing time.Duration } +func (self *ZrokLoopTestType) Label() string { + return "zrok-loop-test" +} + +func (self *ZrokLoopTestType) GetVersion() string { + return self.Version +} + func (self *ZrokLoopTestType) InitType(*model.Component) { canonicalizeGoAppVersion(&self.Version) if self.Iterations == 0 { diff --git a/zititest/zitilab/models/db_builder.go b/zititest/zitilab/models/db_builder.go index e02efab21..d8c3ca858 100644 --- a/zititest/zitilab/models/db_builder.go +++ b/zititest/zitilab/models/db_builder.go @@ -15,9 +15,12 @@ import ( type ZitiDbBuilderStrategy interface { GetDbFile(m *model.Model) string + ProcessDbModel(tx *bbolt.Tx, m *model.Model, builder *ZitiDbBuilder) error +} + +type ZitiEdgeRouterStrategy interface { GetSite(router *db.EdgeRouter) (string, bool) PostProcess(router *db.EdgeRouter, c *model.Component) - ProcessDbModel(tx *bbolt.Tx, m *model.Model, builder *ZitiDbBuilder) error } type ZitiDbBuilder struct { @@ -63,7 +66,7 @@ func (self *ZitiDbBuilder) Build(m *model.Model) error { }) } -func (self *ZitiDbBuilder) CreateEdgeRouterHosts(tx *bbolt.Tx, m *model.Model) error { +func (self *ZitiDbBuilder) CreateEdgeRouterHosts(tx *bbolt.Tx, m *model.Model, erStrategy ZitiEdgeRouterStrategy) error { ids, _, err := self.stores.EdgeRouter.QueryIds(tx, "true limit none") if err != nil { return err @@ -75,7 +78,7 @@ func (self *ZitiDbBuilder) CreateEdgeRouterHosts(tx *bbolt.Tx, m *model.Model) e return err } - if site, useEdgeRouter := self.Strategy.GetSite(er); useEdgeRouter { + if site, useEdgeRouter := erStrategy.GetSite(er); useEdgeRouter { regionId := site[:len(site)-1] var region *model.Region @@ -114,7 +117,7 @@ func (self *ZitiDbBuilder) CreateEdgeRouterHosts(tx *bbolt.Tx, m *model.Model) e } host.Components[er.Id] = component - self.Strategy.PostProcess(er, component) + erStrategy.PostProcess(er, component) } } return nil diff --git a/zititest/zitilab/stageziti/stageziti.go b/zititest/zitilab/stageziti/stageziti.go index 325d86b7a..a0486239f 100644 --- a/zititest/zitilab/stageziti/stageziti.go +++ b/zititest/zitilab/stageziti/stageziti.go @@ -64,6 +64,15 @@ func StageZrok(run model.Run, component *model.Component, version string, source }) } +func StageLocalOnce(run model.Run, executable string, component *model.Component, source string) error { + op := fmt.Sprintf("install.%s-local", executable) + return run.DoOnce(op, func() error { + return StageExecutable(run, executable, component, "", source, func() error { + return fmt.Errorf("unable to fetch %s, as it a local-only application", executable) + }) + }) +} + func StageExecutable(run model.Run, executable string, component *model.Component, version string, source string, fallbackF func() error) error { fileName := executable if version != "" {