From c9eb26038f8c9067071b79a3f223d9337be1cc7d Mon Sep 17 00:00:00 2001 From: Jesse S Date: Tue, 10 Sep 2024 09:43:33 -0700 Subject: [PATCH] VEC-317 0.10.0 support (#11) * !feat: VEC-317 support AVS 0.9.1 - change --hnsw-max-records to --hnsw-m - change --hnsw-healer-schedule-delay to --hnsw-healer-schedule - change --hnsw-merge-parallelism to --hnsw-merge-index-parallelism - add --hnsw-merge-reindex-parallelism * fix: VEC-333 add symlink in deb and rpm installs * test: Use avs 0.10.0 in tests --- .github/workflows/create-prerelease.yml | 4 +- Makefile | 20 +- README.md | 6 +- bin/asvecrpm/asvec.spec | 15 +- .../usr/local/aerospike/bin/.gitignore | 1 - cmd/flags/constants.go | 7 +- cmd/flags/hnsw.go | 29 +-- cmd/indexCreate.go | 5 +- cmd/indexList.go | 2 +- cmd/indexUpdate.go | 5 +- cmd/query.go | 2 +- cmd/writers/indexList.go | 5 +- docker/auth/docker-compose.yml | 2 +- docker/mtls/docker-compose.yml | 2 +- docker/multi-node-LB/docker-compose.yml | 6 +- .../docker-compose.yml | 6 +- docker/multi-node/docker-compose.yml | 6 +- docker/tls/docker-compose.yml | 2 +- docker/vanilla/docker-compose.yml | 2 +- e2e_multi_node_LB_test.go | 9 +- e2e_multi_node_test.go | 11 +- e2e_test.go | 181 ++++++++++-------- go.mod | 2 +- go.sum | 14 +- tests/base_suite.go | 2 +- tests/indexDef.yaml | 5 +- tests/utils.go | 126 ++++++++---- 27 files changed, 280 insertions(+), 197 deletions(-) delete mode 100644 bin/asvecrpm/usr/local/aerospike/bin/.gitignore diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index 7165633..6a63a57 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -34,11 +34,11 @@ jobs: run: | /usr/local/bin/brew install --overwrite python@3.11 || echo "I1.1" /usr/local/bin/brew link --overwrite python@3.11 || echo "I1.2" - /usr/local/bin/brew install --overwrite dpkg upx zip make wget jq rpm || echo "I2" + /usr/local/bin/brew install --overwrite dpkg zip make wget jq rpm || echo "I2" /usr/local/bin/brew link --overwrite python@3.11 || echo "I1.3" /usr/local/bin/brew install python-gdbm@3.11 || echo "I1.4" /usr/local/bin/brew install python-tk@3.11 || echo "I1.5" - for i in dpkg upx zip make wget jq rpm python3.11; do command -v $i || exit 1; done + for i in dpkg zip make wget jq rpm python3.11; do command -v $i || exit 1; done echo "Dependencies checked" - name: Get go version from go.mod run: | diff --git a/Makefile b/Makefile index b66a1a5..af6447d 100644 --- a/Makefile +++ b/Makefile @@ -329,9 +329,11 @@ pkg-deb-amd64: cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec rm -rf $(BIN_DIR)/deb mkdir -p $(BIN_DIR)/deb/DEBIAN - mkdir -p $(BIN_DIR)/deb/usr/local/aerospike/bin + mkdir -p $(BIN_DIR)/deb/opt/aerospike/bin + mkdir -p $(BIN_DIR)/deb/usr/bin @ eval "$$amddebscript" - mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/local/aerospike/bin/ + mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/opt/aerospike/bin/ + ln -s /opt/aerospike/bin/asvec $(BIN_DIR)/deb/usr/bin/asvec sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb @@ -342,9 +344,11 @@ pkg-deb-arm64: cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec rm -rf $(BIN_DIR)/deb mkdir -p $(BIN_DIR)/deb/DEBIAN - mkdir -p $(BIN_DIR)/deb/usr/local/aerospike/bin + mkdir -p $(BIN_DIR)/deb/opt/aerospike/bin + mkdir -p $(BIN_DIR)/deb/usr/bin @ eval "$$armdebscript" - mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/local/aerospike/bin/ + mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/opt/aerospike/bin/ + ln -s /opt/aerospike/bin/asvec $(BIN_DIR)/deb/usr/bin/asvec sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb @@ -391,8 +395,10 @@ pkg-zip: pkg-zip-amd64 pkg-zip-arm64 pkg-rpm-amd64: rm -rf $(BIN_DIR)/asvec-rpm-centos cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos + mkdir -p $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin + mkdir -p $(BIN_DIR)/asvec-rpm-centos/usr/bin sed -i.bak "s/VERSIONHERE/${rpm_ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec - cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/usr/local/aerospike/bin/asvec + cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin/asvec rm -f $(BIN_DIR)/asvec-linux-x86_64.rpm bash -ce "cd $(BIN_DIR) && rpmbuild --target=x86_64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec" rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${rpm_ver}.rpm @@ -402,8 +408,10 @@ pkg-rpm-amd64: pkg-rpm-arm64: rm -rf $(BIN_DIR)/asvec-rpm-centos cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos + mkdir -p $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin + mkdir -p $(BIN_DIR)/asvec-rpm-centos/usr/bin sed -i.bak "s/VERSIONHERE/${rpm_ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec - cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/usr/local/aerospike/bin/asvec + cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin/asvec rm -f $(BIN_DIR)/asvec-linux-arm64.rpm bash -ce "cd $(BIN_DIR) && rpmbuild --target=arm64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec" rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${rpm_ver}.rpm diff --git a/README.md b/README.md index d79fdb1..19f2592 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,12 @@ asvec --help > [!NOTE] > More features are in the works. Don't worry! +- **Data Browsing**: Easily run queries on an index. - **Index Management**: Listing, creating, and dropping indexes. -- **User Management**: Listing, creating, and dropping users. Revoking and granting user's roles. +- **User Management**: Listing, creating, and dropping users. Revoking and + granting user's roles. +- **Node visibility**: Listing nodes and important metadata i.e. version, peers, + etc. ## Issues diff --git a/bin/asvecrpm/asvec.spec b/bin/asvecrpm/asvec.spec index d8abe63..3bff150 100644 --- a/bin/asvecrpm/asvec.spec +++ b/bin/asvecrpm/asvec.spec @@ -1,7 +1,7 @@ Buildroot: ./ Name: asvec Version: VERSIONHERE -Release: 2 +Release: 1 Summary: Tool for deploying non-prod Aerospike server clusters on docker or in AWS License: see github.com/aerospike/asvec Group: aerospike @@ -12,9 +12,16 @@ Group: aerospike %define _binaries_in_noarch_packages_terminate_build 0 %description - - Tool for deploying non-prod Aerospike server clusters on docker or in AWS %files -"/usr/local/aerospike/bin/asvec" +/opt/aerospike/bin/asvec +/usr/bin/asvec + +%install +# Ensure the buildroot directories exist +mkdir -p %{buildroot}/opt/aerospike/bin +mkdir -p %{buildroot}/usr/bin + +%prep +ln -sf /opt/aerospike/bin/asvec %{buildroot}/usr/bin/asvec diff --git a/bin/asvecrpm/usr/local/aerospike/bin/.gitignore b/bin/asvecrpm/usr/local/aerospike/bin/.gitignore deleted file mode 100644 index 0d50953..0000000 --- a/bin/asvecrpm/usr/local/aerospike/bin/.gitignore +++ /dev/null @@ -1 +0,0 @@ -asvec diff --git a/cmd/flags/constants.go b/cmd/flags/constants.go index deb1101..15422b5 100644 --- a/cmd/flags/constants.go +++ b/cmd/flags/constants.go @@ -37,7 +37,7 @@ const ( StorageNamespace = "storage-namespace" StorageSet = "storage-set" CutoffTime = "cutoff-time" - HnswMaxEdges = "hnsw-max-edges" + HnswMaxEdges = "hnsw-m" HnswConstructionEf = "hnsw-ef-construction" HnswEf = "hnsw-ef" HnswMaxMemQueueSize = "hnsw-max-mem-queue-size" @@ -48,9 +48,10 @@ const ( HnswHealerMaxScanRatePerNode = "hnsw-healer-max-scan-rate-per-node" HnswHealerMaxScanPageSize = "hnsw-healer-max-scan-page-size" HnswHealerReindexPercent = "hnsw-healer-reindex-percent" - HnswHealerScheduleDelay = "hnsw-healer-schedule-delay" + HnswHealerSchedule = "hnsw-healer-schedule" HnswHealerParallelism = "hnsw-healer-parallelism" - HnswMergeParallelism = "hnsw-merge-parallelism" + HnswMergeParallelism = "hnsw-merge-index-parallelism" + HnswMergeReIndexParallelism = "hnsw-merge-reindex-parallelism" TLSProtocols = "tls-protocols" TLSCaFile = "tls-cafile" TLSCaPath = "tls-capath" diff --git a/cmd/flags/hnsw.go b/cmd/flags/hnsw.go index 2a3ae75..6bfa875 100644 --- a/cmd/flags/hnsw.go +++ b/cmd/flags/hnsw.go @@ -64,7 +64,7 @@ type HealerFlags struct { MaxScanRatePerNode Uint32OptionalFlag MaxScanPageSize Uint32OptionalFlag ReindexPercent Float32OptionalFlag - ScheduleDelay DurationOptionalFlag + Schedule StringOptionalFlag Parallelism Uint32OptionalFlag } @@ -73,18 +73,18 @@ func NewHnswHealerFlags() *HealerFlags { MaxScanRatePerNode: Uint32OptionalFlag{}, MaxScanPageSize: Uint32OptionalFlag{}, ReindexPercent: Float32OptionalFlag{}, - ScheduleDelay: DurationOptionalFlag{}, + Schedule: StringOptionalFlag{}, Parallelism: Uint32OptionalFlag{}, } } func (cf *HealerFlags) NewFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.Var(&cf.MaxScanRatePerNode, HnswHealerMaxScanRatePerNode, "Maximum allowed record scan rate per AVS node.") //nolint:lll // For readability - flagSet.Var(&cf.MaxScanPageSize, HnswHealerMaxScanPageSize, "Maximum number of records in a single scanned page.") //nolint:lll // For readability - flagSet.Var(&cf.ReindexPercent, HnswHealerReindexPercent, "Percentage of good records randomly selected for reindexing in a healer cycle.") //nolint:lll // For readability - flagSet.Var(&cf.ScheduleDelay, HnswHealerScheduleDelay, "The time delay between the termination of a healer run and the commencement of the next one for an index.") //nolint:lll // For readability - flagSet.Var(&cf.Parallelism, HnswHealerParallelism, "Maximum number of records to heal in parallel.") //nolint:lll // For readability + flagSet.Var(&cf.MaxScanRatePerNode, HnswHealerMaxScanRatePerNode, "Maximum allowed record scan rate per AVS node.") //nolint:lll // For readability + flagSet.Var(&cf.MaxScanPageSize, HnswHealerMaxScanPageSize, "Maximum number of records in a single scanned page.") //nolint:lll // For readability + flagSet.Var(&cf.ReindexPercent, HnswHealerReindexPercent, "Percentage of good records randomly selected for reindexing in a healer cycle.") //nolint:lll // For readability + flagSet.Var(&cf.Schedule, HnswHealerSchedule, "The quartz cron expression defining the schedule at which the index healer cycle is invoked.") //nolint:lll // For readability + flagSet.Var(&cf.Parallelism, HnswHealerParallelism, "Maximum number of records to heal in parallel.") //nolint:lll // For readability return flagSet } @@ -94,30 +94,31 @@ func (cf *HealerFlags) NewSLogAttr() []any { slog.Any(HnswHealerMaxScanRatePerNode, cf.MaxScanRatePerNode.String()), slog.Any(HnswHealerMaxScanPageSize, cf.MaxScanPageSize.String()), slog.Any(HnswHealerReindexPercent, cf.ReindexPercent.String()), - slog.Any(HnswHealerScheduleDelay, cf.ScheduleDelay.String()), + slog.Any(HnswHealerSchedule, cf.Schedule.String()), slog.Any(HnswHealerParallelism, cf.Parallelism.String()), } } type MergeFlags struct { - Parallelism Uint32OptionalFlag + IndexParallelism Uint32OptionalFlag + ReIndexParallelism Uint32OptionalFlag } func NewHnswMergeFlags() *MergeFlags { - return &MergeFlags{ - Parallelism: Uint32OptionalFlag{}, - } + return &MergeFlags{} } func (cf *MergeFlags) NewFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.Var(&cf.Parallelism, HnswMergeParallelism, "The number of vectors merged in parallel from a batch index to main index.") //nolint:lll // For readability + flagSet.Var(&cf.IndexParallelism, HnswMergeParallelism, "The number of vectors merged in parallel from a batch index to main index.") //nolint:lll // For readability + flagSet.Var(&cf.ReIndexParallelism, HnswMergeReIndexParallelism, "The number of vectors merged in parallel from a re-indexing record batch-index to the main index. ") //nolint:lll // For readability return flagSet } func (cf *MergeFlags) NewSLogAttr() []any { return []any{ - slog.Any(HnswMergeParallelism, cf.Parallelism.Val), + slog.Any(HnswMergeParallelism, cf.IndexParallelism.Val), + slog.Any(HnswMergeReIndexParallelism, cf.ReIndexParallelism.Val), } } diff --git a/cmd/indexCreate.go b/cmd/indexCreate.go index b6b0443..54520d5 100644 --- a/cmd/indexCreate.go +++ b/cmd/indexCreate.go @@ -341,11 +341,12 @@ func runCreateIndexFromFlags(client *avs.Client) error { MaxScanRatePerNode: indexCreateFlags.hnswHealer.MaxScanRatePerNode.Val, MaxScanPageSize: indexCreateFlags.hnswHealer.MaxScanPageSize.Val, ReindexPercent: indexCreateFlags.hnswHealer.ReindexPercent.Val, - ScheduleDelay: indexCreateFlags.hnswHealer.ScheduleDelay.Uint64(), + Schedule: indexCreateFlags.hnswHealer.Schedule.Val, Parallelism: indexCreateFlags.hnswHealer.Parallelism.Val, }, MergeParams: &protos.HnswIndexMergeParams{ - Parallelism: indexCreateFlags.hnswMerge.Parallelism.Val, + IndexParallelism: indexCreateFlags.hnswMerge.IndexParallelism.Val, + ReIndexParallelism: indexCreateFlags.hnswMerge.ReIndexParallelism.Val, }, }, } diff --git a/cmd/indexList.go b/cmd/indexList.go index c374f73..1dec462 100644 --- a/cmd/indexList.go +++ b/cmd/indexList.go @@ -73,7 +73,7 @@ asvec index ls ctx, cancel := context.WithTimeout(context.Background(), indexListFlags.clientFlags.Timeout) defer cancel() - indexList, err := client.IndexList(ctx) + indexList, err := client.IndexList(ctx, true) if err != nil { logger.Error("failed to list indexes", slog.Any("error", err)) return err diff --git a/cmd/indexUpdate.go b/cmd/indexUpdate.go index f0bdc1f..d5fe8ae 100644 --- a/cmd/indexUpdate.go +++ b/cmd/indexUpdate.go @@ -112,11 +112,12 @@ asvec index update -i myindex -n test --%s 10000 --%s 10000ms --%s 10s --%s 16 - MaxScanRatePerNode: indexUpdateFlags.hnswHealer.MaxScanRatePerNode.Val, MaxScanPageSize: indexUpdateFlags.hnswHealer.MaxScanPageSize.Val, ReindexPercent: indexUpdateFlags.hnswHealer.ReindexPercent.Val, - ScheduleDelay: indexUpdateFlags.hnswHealer.ScheduleDelay.Uint64(), + Schedule: indexUpdateFlags.hnswHealer.Schedule.Val, Parallelism: indexUpdateFlags.hnswHealer.Parallelism.Val, }, MergeParams: &protos.HnswIndexMergeParams{ - Parallelism: indexUpdateFlags.hnswMerge.Parallelism.Val, + IndexParallelism: indexUpdateFlags.hnswMerge.IndexParallelism.Val, + ReIndexParallelism: indexUpdateFlags.hnswMerge.ReIndexParallelism.Val, }, } diff --git a/cmd/query.go b/cmd/query.go index 3c19e7f..72f0ce0 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -146,7 +146,7 @@ asvec query -i my-index -n my-namespace -v "[1,0,1,0,0,0,1,0,1,1]" --max-keys 10 return } } else { - indexDef, err := client.IndexGet(ctx, queryFlags.namespace, queryFlags.indexName) + indexDef, err := client.IndexGet(ctx, queryFlags.namespace, queryFlags.indexName, false) if err != nil { logger.ErrorContext(ctx, "unable to get index definition", slog.Any("error", err)) view.Errorf("Failed to get index definition: %s", err) diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index 5a6ea26..4eae08c 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -82,9 +82,10 @@ func (itw *IndexTableWriter) AppendIndexRow( {"Healer Max Scan Rate / Node*", v.HnswParams.HealerParams.GetMaxScanRatePerNode()}, {"Healer Max Page Size*", v.HnswParams.HealerParams.GetMaxScanPageSize()}, {"Healer Re-index % *", convertFloatToPercentStr(v.HnswParams.HealerParams.GetReindexPercent())}, - {"Healer Schedule Delay*", convertMillisecondToDuration(v.HnswParams.HealerParams.GetScheduleDelay())}, + {"Healer Schedule*", v.HnswParams.HealerParams.GetSchedule()}, {"Healer Parallelism*", v.HnswParams.HealerParams.GetParallelism()}, - {"Merge Parallelism*", v.HnswParams.MergeParams.GetParallelism()}, + {"Merge Index Parallelism*", v.HnswParams.MergeParams.GetIndexParallelism()}, + {"Merge Re-Index Parallelism*", v.HnswParams.MergeParams.GetReIndexParallelism()}, }) row = append(row, renderTable(tHNSW, format)) diff --git a/docker/auth/docker-compose.yml b/docker/auth/docker-compose.yml index 811d09a..3a1c1a8 100644 --- a/docker/auth/docker-compose.yml +++ b/docker/auth/docker-compose.yml @@ -16,7 +16,7 @@ services: timeout: 20s retries: 20 avs: - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 depends_on: aerospike: condition: service_healthy diff --git a/docker/mtls/docker-compose.yml b/docker/mtls/docker-compose.yml index 811d09a..3a1c1a8 100644 --- a/docker/mtls/docker-compose.yml +++ b/docker/mtls/docker-compose.yml @@ -16,7 +16,7 @@ services: timeout: 20s retries: 20 avs: - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 depends_on: aerospike: condition: service_healthy diff --git a/docker/multi-node-LB/docker-compose.yml b/docker/multi-node-LB/docker-compose.yml index bfd08c3..3e59264 100644 --- a/docker/multi-node-LB/docker-compose.yml +++ b/docker/multi-node-LB/docker-compose.yml @@ -20,7 +20,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 volumes: - ./config/aerospike-vector-search-1.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml - ./config/features.conf:/etc/aerospike-vector-search/features.conf @@ -35,7 +35,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 volumes: - ./config/aerospike-vector-search-2.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml - ./config/features.conf:/etc/aerospike-vector-search/features.conf @@ -50,7 +50,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 volumes: - ./config/aerospike-vector-search-3.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml - ./config/features.conf:/etc/aerospike-vector-search/features.conf diff --git a/docker/multi-node-client-visibility-err/docker-compose.yml b/docker/multi-node-client-visibility-err/docker-compose.yml index 7ab5f6f..01d12e4 100644 --- a/docker/multi-node-client-visibility-err/docker-compose.yml +++ b/docker/multi-node-client-visibility-err/docker-compose.yml @@ -20,7 +20,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 ports: - "10000:10000" volumes: @@ -37,7 +37,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 ports: - "10001:10001" volumes: @@ -54,7 +54,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 # ports: # - "10002:10002" # This causes the visibility err volumes: diff --git a/docker/multi-node/docker-compose.yml b/docker/multi-node/docker-compose.yml index bb49fa6..2478015 100644 --- a/docker/multi-node/docker-compose.yml +++ b/docker/multi-node/docker-compose.yml @@ -20,7 +20,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 ports: - "10000:10000" volumes: @@ -37,7 +37,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 ports: - "10001:10001" volumes: @@ -54,7 +54,7 @@ services: depends_on: aerospike: condition: service_healthy - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 ports: - "10002:10002" volumes: diff --git a/docker/tls/docker-compose.yml b/docker/tls/docker-compose.yml index 811d09a..3a1c1a8 100644 --- a/docker/tls/docker-compose.yml +++ b/docker/tls/docker-compose.yml @@ -16,7 +16,7 @@ services: timeout: 20s retries: 20 avs: - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 depends_on: aerospike: condition: service_healthy diff --git a/docker/vanilla/docker-compose.yml b/docker/vanilla/docker-compose.yml index 811d09a..3a1c1a8 100644 --- a/docker/vanilla/docker-compose.yml +++ b/docker/vanilla/docker-compose.yml @@ -16,7 +16,7 @@ services: timeout: 20s retries: 20 avs: - image: aerospike/aerospike-vector-search:0.9.0 + image: aerospike/aerospike-vector-search:0.10.0 depends_on: aerospike: condition: service_healthy diff --git a/e2e_multi_node_LB_test.go b/e2e_multi_node_LB_test.go index 2eed5bd..1eb12fb 100644 --- a/e2e_multi_node_LB_test.go +++ b/e2e_multi_node_LB_test.go @@ -43,6 +43,10 @@ func TestMultiNodeLBCmdSuite(t *testing.T) { } func (suite *MultiNodeLBCmdTestSuite) TestNodeListCmd() { + about, err := suite.AvsClient.About(context.Background(), nil) + if err != nil { + suite.T().Fatal(err) + } testCases := []struct { name string @@ -57,7 +61,7 @@ func (suite *MultiNodeLBCmdTestSuite) TestNodeListCmd() { true, `Nodes ,Node,Endpoint,Cluster ID,Version,Visible Nodes -1,Seed,localhost:10000,,0.9.0,"{ +1,Seed,localhost:10000,,,"{ 1103823447824: [1.1.1.1:10000] 2207646885648: [2.2.2.2:10000] 3311470323472: [3.3.3.3:10000] @@ -76,7 +80,7 @@ Possible scenarios: false, `Nodes ,Node,Endpoint,Cluster ID,Version,Visible Nodes -1,LB,localhost:10000,,0.9.0,"{ +1,LB,localhost:10000,,,"{ 1103823447824: [1.1.1.1:10000] 2207646885648: [2.2.2.2:10000] 3311470323472: [3.3.3.3:10000] @@ -93,6 +97,7 @@ Possible scenarios: clusterIDStr := fmt.Sprintf("%d", state.ClusterId.GetId()) tc.expectedTable = strings.ReplaceAll(tc.expectedTable, "", clusterIDStr) + tc.expectedTable = strings.ReplaceAll(tc.expectedTable, "", about.Version) outLines, errLines, err := suite.RunSuiteCmd(strings.Split(tc.cmd, " ")...) if tc.expectErrCoded { diff --git a/e2e_multi_node_test.go b/e2e_multi_node_test.go index aa27226..baa1c6a 100644 --- a/e2e_multi_node_test.go +++ b/e2e_multi_node_test.go @@ -41,6 +41,10 @@ func TestMultiNodeCmdSuite(t *testing.T) { } func (suite *MultiNodeCmdTestSuite) TestNodeListCmd() { + about, err := suite.AvsClient.About(context.Background(), nil) + if err != nil { + suite.T().Fatal(err) + } testCases := []struct { name string @@ -52,15 +56,15 @@ func (suite *MultiNodeCmdTestSuite) TestNodeListCmd() { fmt.Sprintf("node ls --format 1 --no-color --seeds %s", suite.AvsHostPort.String()), `Nodes ,Node,Endpoint,Cluster ID,Version,Visible Nodes -1,139637976803088,127.0.0.1:10000,,0.9.0,"{ +1,139637976803088,127.0.0.1:10000,,,"{ 139637976803089: [127.0.0.1:10001] 139637976803090: [127.0.0.1:10002] }" -2,139637976803089,127.0.0.1:10001,,0.9.0,"{ +2,139637976803089,127.0.0.1:10001,,,"{ 139637976803088: [127.0.0.1:10000] 139637976803090: [127.0.0.1:10002] }" -3,139637976803090,127.0.0.1:10002,,0.9.0,"{ +3,139637976803090,127.0.0.1:10002,,,"{ 139637976803088: [127.0.0.1:10000] 139637976803089: [127.0.0.1:10001] }" @@ -75,6 +79,7 @@ func (suite *MultiNodeCmdTestSuite) TestNodeListCmd() { clusterIDStr := fmt.Sprintf("%d", state.ClusterId.GetId()) tc.expectedTable = strings.ReplaceAll(tc.expectedTable, "", clusterIDStr) + tc.expectedTable = strings.ReplaceAll(tc.expectedTable, "", about.Version) outLines, _, err := suite.RunSuiteCmd(strings.Split(tc.cmd, " ")...) // suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, // lines diff --git a/e2e_test.go b/e2e_test.go index fa203d0..50e88d6 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -172,7 +172,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "index0", "test", "index create -y -n test -i index0 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --index-labels model=all-MiniLM-L6-v2,foo=bar", - tests.NewIndexDefinitionBuilder("index0", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0"). + tests.NewIndexDefinitionBuilder(false, "index0", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0"). WithLabels(map[string]string{"model": "all-MiniLM-L6-v2", "foo": "bar"}). Build(), }, @@ -181,7 +181,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "index1", "test", "index create -y -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar", - tests.NewIndexDefinitionBuilder("index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1"). + tests.NewIndexDefinitionBuilder(false, "index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1"). WithStorageNamespace("bar"). WithStorageSet("testbar"). Build(), @@ -190,8 +190,8 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with hnsw params and seeds", "index2", "test", - "index create -y -n test -i index2 -d 256 -m HAMMING --vector-field vector2 --hnsw-max-edges 10 --hnsw-ef 11 --hnsw-ef-construction 12 --hnsw-max-mem-queue-size 10", - tests.NewIndexDefinitionBuilder("index2", "test", 256, protos.VectorDistanceMetric_HAMMING, "vector2"). + "index create -y -n test -i index2 -d 256 -m HAMMING --vector-field vector2 --hnsw-m 10 --hnsw-ef 11 --hnsw-ef-construction 12 --hnsw-max-mem-queue-size 10", + tests.NewIndexDefinitionBuilder(false, "index2", "test", 256, protos.VectorDistanceMetric_HAMMING, "vector2"). WithHnswM(10). WithHnswEf(11). WithHnswEfConstruction(12). @@ -202,9 +202,9 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with hnsw batch params", "index3", "test", - "index create -y -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-interval 50s --hnsw-batch-max-records 100", - tests.NewIndexDefinitionBuilder("index3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector3"). - WithHnswBatchingMaxRecord(100). + "index create -y -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-interval 50s --hnsw-batch-max-records 10001", + tests.NewIndexDefinitionBuilder(false, "index3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector3"). + WithHnswBatchingMaxRecord(10001). WithHnswBatchingInterval(50000). Build(), }, @@ -213,7 +213,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "index4", "test", "index create -y -n test -i index4 -d 256 -m COSINE --vector-field vector4 --hnsw-cache-max-entries 1000 --hnsw-cache-expiry 10s", - tests.NewIndexDefinitionBuilder("index4", "test", 256, protos.VectorDistanceMetric_COSINE, "vector4"). + tests.NewIndexDefinitionBuilder(false, "index4", "test", 256, protos.VectorDistanceMetric_COSINE, "vector4"). WithHnswCacheExpiry(10000). WithHnswCacheMaxEntries(1000). Build(), @@ -222,12 +222,12 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with hnsw healer params", "index5", "test", - "index create -y -n test -i index5 -d 256 -m COSINE --vector-field vector5 --hnsw-healer-max-scan-rate-per-node 1000 --hnsw-healer-max-scan-page-size 1000 --hnsw-healer-reindex-percent 10.10 --hnsw-healer-schedule-delay 10s --hnsw-healer-parallelism 10", - tests.NewIndexDefinitionBuilder("index5", "test", 256, protos.VectorDistanceMetric_COSINE, "vector5"). + "index create -y -n test -i index5 -d 256 -m COSINE --vector-field vector5 --hnsw-healer-max-scan-rate-per-node 1000 --hnsw-healer-max-scan-page-size 1000 --hnsw-healer-reindex-percent 10.10 --hnsw-healer-schedule \"0 0 0 ? * *\" --hnsw-healer-parallelism 10", + tests.NewIndexDefinitionBuilder(false, "index5", "test", 256, protos.VectorDistanceMetric_COSINE, "vector5"). WithHnswHealerMaxScanRatePerNode(1000). WithHnswHealerMaxScanPageSize(1000). WithHnswHealerReindexPercent(10.10). - WithHnswHealerScheduleDelay(10000). + WithHnswHealerSchedule("0 0 0 ? * *"). WithHnswHealerParallelism(10). Build(), }, @@ -235,9 +235,10 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "test with hnsw merge params", "index6", "test", - "index create -y -n test -i index6 -d 256 -m COSINE --vector-field vector6 --hnsw-merge-parallelism 10", - tests.NewIndexDefinitionBuilder("index6", "test", 256, protos.VectorDistanceMetric_COSINE, "vector6"). - WithHnswMergeParallelism(10). + "index create -y -n test -i index6 -d 256 -m COSINE --vector-field vector6 --hnsw-merge-index-parallelism 10 --hnsw-merge-reindex-parallelism 11", + tests.NewIndexDefinitionBuilder(false, "index6", "test", 256, protos.VectorDistanceMetric_COSINE, "vector6"). + WithHnswMergeIndexParallelism(10). + WithHnswMergeReIndexParallelism(11). Build(), }, { @@ -245,7 +246,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { "yaml-file-index", "test", fmt.Sprintf("index create -y --file tests/indexDef.yaml"), - tests.NewIndexDefinitionBuilder("yaml-file-index", "test", 10, protos.VectorDistanceMetric_COSINE, "vector"). + tests.NewIndexDefinitionBuilder(false, "yaml-file-index", "test", 10, protos.VectorDistanceMetric_COSINE, "vector"). WithSet("testset"). WithHnswEf(101). WithHnswEfConstruction(102). @@ -259,8 +260,10 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { WithHnswHealerMaxScanRatePerNode(1). WithHnswHealerMaxScanPageSize(2). WithHnswHealerReindexPercent(3). - WithHnswHealerScheduleDelay(4). - WithHnswMergeParallelism(7). + WithHnswHealerSchedule("0 15 10 ? * 6L 2022-2025"). + WithHnswMergeIndexParallelism(7). + WithHnswMergeReIndexParallelism(5). + WithStorageNamespace("test"). WithStorageSet("name"). Build(), }, @@ -268,14 +271,14 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { for _, tc := range testCases { suite.Run(tc.name, func() { - lines, _, err := suite.RunSuiteCmd(strings.Split(tc.cmd, " ")...) + lines, stderr, err := suite.RunSuiteCmd(strings.FieldsFunc(tc.cmd, tests.SplitQuotedString)...) if err != nil { - suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + suite.Assert().NoError(err, "error: %s, stdout: %s, stderr: %s", err, lines, stderr) suite.FailNow("unable to index create") } - actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) + actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName, false) if err != nil { suite.FailNowf("unable to get index", "%v", err) @@ -299,10 +302,10 @@ func (suite *CmdTestSuite) TestPipeFromListIndexToCreateIndex() { { "test with all indexes succeed", []*protos.IndexDefinition{ - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "exists1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", ).Build(), - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "exists2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", ).WithSet("barset").Build(), }, @@ -317,10 +320,10 @@ func (suite *CmdTestSuite) TestPipeFromListIndexToCreateIndex() { { "test with one index that fails", []*protos.IndexDefinition{ - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "exists3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", ).Build(), - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "exists4", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", ).WithSet("barset").Build(), }, @@ -335,10 +338,10 @@ func (suite *CmdTestSuite) TestPipeFromListIndexToCreateIndex() { { "test with no index successfully created", []*protos.IndexDefinition{ - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "exists1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", ).Build(), - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "exists2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", ).WithSet("barset").Build(), }, @@ -449,10 +452,13 @@ func (suite *CmdTestSuite) TestPipeFromListIndexToCreateIndex() { } func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { - suite.AvsClient.IndexCreate(context.Background(), "test", "successful-update", "field", uint32(256), protos.VectorDistanceMetric_COSINE, nil) ns := "test" index := "successful-update" - builder := tests.NewIndexDefinitionBuilder(index, ns, 256, protos.VectorDistanceMetric_COSINE, "field") + + newBuilder := func() *tests.IndexDefinitionBuilder { + return tests.NewIndexDefinitionBuilder(true, index, ns, 256, protos.VectorDistanceMetric_COSINE, "field") + } + testCases := []struct { name string indexName string // index names must be unique for the suite @@ -465,7 +471,7 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { "successful-update", ns, "index update -y -n test -i successful-update --index-labels new-label=foo --hnsw-max-mem-queue-size 10", - builder. + newBuilder(). WithLabels(map[string]string{"new-label": "foo"}). WithHnswMaxMemQueueSize(10). Build(), @@ -474,9 +480,9 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { "test with hnsw batch params", "successful-update", "test", - "index update -y -n test -i successful-update --hnsw-batch-interval 50s --hnsw-batch-max-records 100", - builder. - WithHnswBatchingMaxRecord(100). + "index update -y -n test -i successful-update --hnsw-batch-interval 50s --hnsw-batch-max-records 10001", + newBuilder(). + WithHnswBatchingMaxRecord(10001). WithHnswBatchingInterval(50000). Build(), }, @@ -485,7 +491,7 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { "successful-update", "test", "index update -y -n test -i successful-update --hnsw-cache-max-entries 1000 --hnsw-cache-expiry 10s", - builder. + newBuilder(). WithHnswCacheExpiry(10000). WithHnswCacheMaxEntries(1000). Build(), @@ -494,12 +500,12 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { "test with hnsw healer params", "successful-update", "test", - "index update -y -n test -i successful-update --hnsw-healer-max-scan-rate-per-node 1000 --hnsw-healer-max-scan-page-size 1000 --hnsw-healer-reindex-percent 10.10 --hnsw-healer-schedule-delay 10s --hnsw-healer-parallelism 10", - builder. + "index update -y -n test -i successful-update --hnsw-healer-max-scan-rate-per-node 1000 --hnsw-healer-max-scan-page-size 1000 --hnsw-healer-reindex-percent 10.10 --hnsw-healer-schedule \"0 30 11 ? * 6#2\" --hnsw-healer-parallelism 10", + newBuilder(). WithHnswHealerMaxScanRatePerNode(1000). WithHnswHealerMaxScanPageSize(1000). WithHnswHealerReindexPercent(10.10). - WithHnswHealerScheduleDelay(10000). + WithHnswHealerSchedule("0 30 11 ? * 6#2"). WithHnswHealerParallelism(10). Build(), }, @@ -507,25 +513,33 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { "test with hnsw merge params", "successful-update", "test", - "index update -y -n test -i successful-update --hnsw-merge-parallelism 10", - builder. - WithHnswMergeParallelism(10). + "index update -y -n test -i successful-update --hnsw-merge-index-parallelism 10 --hnsw-merge-reindex-parallelism 11", + newBuilder(). + WithHnswMergeIndexParallelism(10). + WithHnswMergeReIndexParallelism(11). Build(), }, } for _, tc := range testCases { suite.Run(tc.name, func() { - lines, _, err := suite.RunSuiteCmd(strings.Split(tc.cmd, " ")...) + err := suite.AvsClient.IndexCreate(context.Background(), ns, "successful-update", "field", uint32(256), protos.VectorDistanceMetric_COSINE, nil) + if err != nil { + suite.FailNowf("unable to index create", "%v", err) + } + + defer suite.AvsClient.IndexDrop(context.Background(), ns, "successful-update") + + lines, stderr, err := suite.RunSuiteCmd(strings.FieldsFunc(tc.cmd, tests.SplitQuotedString)...) if err != nil { - suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + suite.Assert().NoError(err, "error: %s, stdout: %s, stderr: %s", err, lines, stderr) suite.FailNow("unable to index update") } time.Sleep(5 * time.Second) - actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) + actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName, false) if err != nil { suite.FailNowf("unable to get index", "%v", err) @@ -538,7 +552,7 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { } func (suite *CmdTestSuite) TestUpdateIndexDoesNotExist() { - _, lines, err := suite.RunSuiteCmd(strings.Split("index update -y -n test -i DNE --hnsw-merge-parallelism 10", " ")...) + _, lines, err := suite.RunSuiteCmd(strings.Split("index update -y -n test -i DNE --hnsw-merge-index-parallelism 10", " ")...) suite.Assert().Error(err, "index should have NOT existed. stdout/err: %s", lines) suite.Assert().Contains(lines, "server error") } @@ -631,7 +645,7 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { suite.FailNow("unable to index drop") } - _, err = suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) + _, err = suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName, false) time.Sleep(time.Second * 3) @@ -666,7 +680,7 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { { "single index", []*protos.IndexDefinition{ - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "list", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", ).Build(), }, @@ -679,10 +693,10 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { { "double index with set", []*protos.IndexDefinition{ - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "list1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", ).Build(), - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "list2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", ).WithSet("barset").Build(), }, @@ -696,12 +710,18 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { { "double index with set and verbose", []*protos.IndexDefinition{ - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, "list1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", - ).WithLabels(map[string]string{"foo": "bar"}).Build(), - tests.NewIndexDefinitionBuilder( + ).WithLabels(map[string]string{"foo": "bar"}). + WithHnswMergeIndexParallelism(80). + WithHnswMergeReIndexParallelism(26). + Build(), + tests.NewIndexDefinitionBuilder(false, "list2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", - ).WithSet("barset").Build(), + ).WithSet("barset"). + WithHnswMergeIndexParallelism(80). + WithHnswMergeReIndexParallelism(26). + Build(), }, "index list --verbose --no-color --format 1", `Indexes @@ -711,33 +731,35 @@ Set\,list2","HNSW Max Edges\,16 Ef\,100 Construction Ef\,100 -MaxMemQueueSize*\,0 +MaxMemQueueSize*\,1000000 Batch Max Records*\,100000 Batch Interval*\,30s -Cache Max Entires*\,0 -Cache Expiry*\,0s -Healer Max Scan Rate / Node*\,0 -Healer Max Page Size*\,0 -Healer Re-index % *\,0.00% -Healer Schedule Delay*\,0s -Healer Parallelism*\,0 -Merge Parallelism*\,0" +Cache Max Entires*\,2000000 +Cache Expiry*\,1h0m0s +Healer Max Scan Rate / Node*\,1000 +Healer Max Page Size*\,10000 +Healer Re-index % *\,10.00% +Healer Schedule*\,0 0/15 * ? * * * +Healer Parallelism*\,1 +Merge Index Parallelism*\,80 +Merge Re-Index Parallelism*\,26" 2,list1,test,,vector,256,COSINE,0,map[foo:bar],"Namespace\,test Set\,list1","HNSW Max Edges\,16 Ef\,100 Construction Ef\,100 -MaxMemQueueSize*\,0 +MaxMemQueueSize*\,1000000 Batch Max Records*\,100000 Batch Interval*\,30s -Cache Max Entires*\,0 -Cache Expiry*\,0s -Healer Max Scan Rate / Node*\,0 -Healer Max Page Size*\,0 -Healer Re-index % *\,0.00% -Healer Schedule Delay*\,0s -Healer Parallelism*\,0 -Merge Parallelism*\,0" +Cache Max Entires*\,2000000 +Cache Expiry*\,1h0m0s +Healer Max Scan Rate / Node*\,1000 +Healer Max Page Size*\,10000 +Healer Re-index % *\,10.00% +Healer Schedule*\,0 0/15 * ? * * * +Healer Parallelism*\,1 +Merge Index Parallelism*\,80 +Merge Re-Index Parallelism*\,26" Values ending with * can be dynamically configured using the 'asvec index update' command. `, }, @@ -1089,10 +1111,10 @@ func (suite *CmdTestSuite) TestSuccessfulQueryCmd() { strIndexName := "query-str-index" intIndexName := "query-int-index" indexes := []*protos.IndexDefinition{ - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, strIndexName, "test", 10, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "float32-str", ).Build(), - tests.NewIndexDefinitionBuilder( + tests.NewIndexDefinitionBuilder(false, intIndexName, "test", 10, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "float32-int", ).Build(), } @@ -1634,9 +1656,9 @@ func (suite *CmdTestSuite) TestFailInvalidArg() { "Error: invalid argument \"foo\" for \"--hnsw-ef-construction\"", }, { - "test with bad hnsw-max-edges", - "index create -y --hnsw-max-edges foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", - "Error: invalid argument \"foo\" for \"--hnsw-max-edges\"", + "test with bad hnsw-m", + "index create -y --hnsw-m foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", + "Error: invalid argument \"foo\" for \"--hnsw-m\"", }, { "test with bad hnsw-batch-interval", @@ -1673,20 +1695,15 @@ func (suite *CmdTestSuite) TestFailInvalidArg() { "index create -y --hnsw-healer-reindex-percent foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", "Error: invalid argument \"foo\" for \"--hnsw-healer-reindex-percent\"", }, - { - "test with bad hnsw-healer-schedule-delay", - "index create -y --hnsw-healer-schedule-delay foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", - "Error: invalid argument \"foo\" for \"--hnsw-healer-schedule-delay\"", - }, { "test with bad hnsw-healer-parallelism", "index create -y --hnsw-healer-parallelism foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", "Error: invalid argument \"foo\" for \"--hnsw-healer-parallelism\"", }, { - "test with bad hnsw-merge-parallelism", - "index create -y --hnsw-merge-parallelism foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", - "Error: invalid argument \"foo\" for \"--hnsw-merge-parallelism\"", + "test with bad hnsw-merge-index-parallelism", + "index create -y --hnsw-merge-index-parallelism foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", + "Error: invalid argument \"foo\" for \"--hnsw-merge-index-parallelism\"", }, // { diff --git a/go.mod b/go.mod index 05e3a4d..4821161 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module asvec go 1.22.5 require ( - github.com/aerospike/avs-client-go v0.0.0-20240827160142-b7ab11d122ca + github.com/aerospike/avs-client-go v0.0.0-20240906211641-97c1df4005ae github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.1 diff --git a/go.sum b/go.sum index 3aceca3..4f8f03a 100644 --- a/go.sum +++ b/go.sum @@ -2,18 +2,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.6.0 h1:jAOlsxOaWbmtGzB1yP9x1komh4x14BvCb5HRu5AzVIo= github.com/aerospike/aerospike-client-go/v7 v7.6.0/go.mod h1:uCbSYMpjlRcH/9f26VSF/luzDDXrcDaV8c6/WIcKtT4= -github.com/aerospike/avs-client-go v0.0.0-20240806182530-9fc701011935 h1:8JigL+j50TKxdIhbfLSIxGbGHIdPibQrYOA9xZpnO+w= -github.com/aerospike/avs-client-go v0.0.0-20240806182530-9fc701011935/go.mod h1:aKGgEyTM/sAUj1HDtm292G/RPG7WoUxRLlKzFuAMipk= -github.com/aerospike/avs-client-go v0.0.0-20240815192859-55e852db364e h1:SdCD25vIEJMMiORKXnptqcUAZzqAm9mngPPk7wDkt3s= -github.com/aerospike/avs-client-go v0.0.0-20240815192859-55e852db364e/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY= -github.com/aerospike/avs-client-go v0.0.0-20240819201803-c3a4e6208ecb h1:h+DwEz9ia1jL7ltRiYU5ciL1PVcwq06mhSUaDbXWtN0= -github.com/aerospike/avs-client-go v0.0.0-20240819201803-c3a4e6208ecb/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY= -github.com/aerospike/avs-client-go v0.0.0-20240823224818-55e9f6e3d0dd h1:5BTrwvo2mfsPTwBrA68N1y05xPp7qvsBHltPqxbls/0= -github.com/aerospike/avs-client-go v0.0.0-20240823224818-55e9f6e3d0dd/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY= -github.com/aerospike/avs-client-go v0.0.0-20240824002832-084554ed5b09 h1:oLZi3aG57isOcDavxuUf+fVGwANKGB+JTpKgqVgaDm8= -github.com/aerospike/avs-client-go v0.0.0-20240824002832-084554ed5b09/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY= -github.com/aerospike/avs-client-go v0.0.0-20240827160142-b7ab11d122ca h1:hYgwO67I0B4pA1n3mOGEveI9TpNFfHcEI6G9nK5FibI= -github.com/aerospike/avs-client-go v0.0.0-20240827160142-b7ab11d122ca/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY= +github.com/aerospike/avs-client-go v0.0.0-20240906211641-97c1df4005ae h1:SF5A5zEeuBFTCV19ocWsdnK2bB7P3MnTnvyJANae5qM= +github.com/aerospike/avs-client-go v0.0.0-20240906211641-97c1df4005ae/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY= github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 h1:tOLqcGsc6A656WNEZhYZYxDiX7d6wCkEN6+jLDSDeUU= github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/tests/base_suite.go b/tests/base_suite.go index 60565ba..6905fef 100644 --- a/tests/base_suite.go +++ b/tests/base_suite.go @@ -80,7 +80,7 @@ func (suite *CmdBaseTestSuite) TearDownSuite() { } func (suite *CmdBaseTestSuite) CleanUpIndexes(ctx context.Context) { - indexes, err := suite.AvsClient.IndexList(ctx) + indexes, err := suite.AvsClient.IndexList(ctx, false) if err != nil { suite.FailNow(err.Error()) } diff --git a/tests/indexDef.yaml b/tests/indexDef.yaml index 2879271..2111f28 100644 --- a/tests/indexDef.yaml +++ b/tests/indexDef.yaml @@ -14,12 +14,13 @@ indices: maxScanRatePerNode: 1 maxScanPageSize: 2 reindexPercent: 3 - scheduleDelay: 4 + schedule: "0 15 10 ? * 6L 2022-2025" parallelism: 7 m: 103 maxMemQueueSize: 10004 mergeParams: - parallelism: 7 + indexParallelism: 7 + reIndexParallelism: 5 id: name: yaml-file-index namespace: test diff --git a/tests/utils.go b/tests/utils.go index fa31123..fb8a5f9 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -38,6 +38,7 @@ func CreateFlagStr(name, value string) string { } type IndexDefinitionBuilder struct { + updateCmd bool indexName string namespace string set *string @@ -59,11 +60,13 @@ type IndexDefinitionBuilder struct { hnswHealerMaxScanRatePerSecond *uint32 hnswHealerParallelism *uint32 HnswHealerReindexPercent *float32 - HnswHealerScheduleDelay *uint64 - hnswMergeParallelism *uint32 + HnswHealerSchedule *string + hnswMergeIndexParallelism *uint32 + hnswMergeReIndexParallelism *uint32 } func NewIndexDefinitionBuilder( + updateCmd bool, indexName, namespace string, dimension int, @@ -71,6 +74,7 @@ func NewIndexDefinitionBuilder( vectorField string, ) *IndexDefinitionBuilder { return &IndexDefinitionBuilder{ + updateCmd: updateCmd, indexName: indexName, namespace: namespace, dimension: dimension, @@ -159,44 +163,64 @@ func (idb *IndexDefinitionBuilder) WithHnswHealerReindexPercent(reindexPercent f return idb } -func (idb *IndexDefinitionBuilder) WithHnswHealerScheduleDelay(scheduleDelay uint64) *IndexDefinitionBuilder { - idb.HnswHealerScheduleDelay = &scheduleDelay +func (idb *IndexDefinitionBuilder) WithHnswHealerSchedule(schedule string) *IndexDefinitionBuilder { + idb.HnswHealerSchedule = &schedule return idb } -func (idb *IndexDefinitionBuilder) WithHnswMergeParallelism(mergeParallelism uint32) *IndexDefinitionBuilder { - idb.hnswMergeParallelism = &mergeParallelism +func (idb *IndexDefinitionBuilder) WithHnswMergeIndexParallelism(mergeParallelism uint32) *IndexDefinitionBuilder { + idb.hnswMergeIndexParallelism = &mergeParallelism + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswMergeReIndexParallelism(mergeParallelism uint32) *IndexDefinitionBuilder { + idb.hnswMergeReIndexParallelism = &mergeParallelism return idb } func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { - indexDef := &protos.IndexDefinition{ - Id: &protos.IndexId{ - Name: idb.indexName, - Namespace: idb.namespace, - }, - Dimensions: uint32(idb.dimension), - VectorDistanceMetric: idb.vectorDistanceMetric, - Field: idb.vectorField, - Type: protos.IndexType_HNSW, - Storage: &protos.IndexStorage{ - Namespace: &idb.namespace, - Set: &idb.indexName, - }, - Params: &protos.IndexDefinition_HnswParams{ - HnswParams: &protos.HnswParams{ - M: GetUint32Ptr(16), - EfConstruction: GetUint32Ptr(100), - Ef: GetUint32Ptr(100), - BatchingParams: &protos.HnswBatchingParams{ - MaxRecords: GetUint32Ptr(100000), - Interval: GetUint32Ptr(30000), + var indexDef *protos.IndexDefinition + + if idb.updateCmd { + indexDef = &protos.IndexDefinition{ + Id: &protos.IndexId{ + Name: idb.indexName, + Namespace: idb.namespace, + }, + Dimensions: uint32(idb.dimension), + VectorDistanceMetric: idb.vectorDistanceMetric, + Field: idb.vectorField, + Type: protos.IndexType_HNSW, + // Storage: , + Params: &protos.IndexDefinition_HnswParams{ + HnswParams: &protos.HnswParams{ + // BatchingParams: &protos.HnswBatchingParams{}, + CachingParams: &protos.HnswCachingParams{}, + HealerParams: &protos.HnswHealerParams{}, + MergeParams: &protos.HnswIndexMergeParams{}, }, - CachingParams: &protos.HnswCachingParams{}, - HealerParams: &protos.HnswHealerParams{}, - MergeParams: &protos.HnswIndexMergeParams{}, }, - }, + } + } else { + indexDef = &protos.IndexDefinition{ + Id: &protos.IndexId{ + Name: idb.indexName, + Namespace: idb.namespace, + }, + Dimensions: uint32(idb.dimension), + VectorDistanceMetric: idb.vectorDistanceMetric, + Field: idb.vectorField, + Type: protos.IndexType_HNSW, + Storage: &protos.IndexStorage{}, + Params: &protos.IndexDefinition_HnswParams{ + HnswParams: &protos.HnswParams{ + BatchingParams: &protos.HnswBatchingParams{}, + CachingParams: &protos.HnswCachingParams{}, + HealerParams: &protos.HnswHealerParams{}, + MergeParams: &protos.HnswIndexMergeParams{}, + }, + }, + } } indexDef.SetFilter = idb.set @@ -205,12 +229,11 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { indexDef.Labels = idb.labels } - if idb.storageNamespace != nil { - indexDef.Storage.Namespace = idb.storageNamespace - } - - if idb.storageSet != nil { - indexDef.Storage.Set = idb.storageSet + if idb.storageNamespace != nil || idb.storageSet != nil { + indexDef.Storage = &protos.IndexStorage{ + Namespace: idb.storageNamespace, + Set: idb.storageSet, + } } if idb.hnsfM != nil { @@ -225,6 +248,10 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.EfConstruction = idb.hnsfEfC } + if idb.hnsfBatchingInterval != nil || idb.hnsfBatchingMaxRecord != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams = &protos.HnswBatchingParams{} + } + if idb.hnsfBatchingMaxRecord != nil { indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.MaxRecords = idb.hnsfBatchingMaxRecord } @@ -261,12 +288,16 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.HealerParams.ReindexPercent = idb.HnswHealerReindexPercent } - if idb.HnswHealerScheduleDelay != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.HealerParams.ScheduleDelay = idb.HnswHealerScheduleDelay + if idb.HnswHealerSchedule != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.HealerParams.Schedule = idb.HnswHealerSchedule + } + + if idb.hnswMergeIndexParallelism != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.MergeParams.IndexParallelism = idb.hnswMergeIndexParallelism } - if idb.hnswMergeParallelism != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.MergeParams.Parallelism = idb.hnswMergeParallelism + if idb.hnswMergeReIndexParallelism != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.MergeParams.ReIndexParallelism = idb.hnswMergeReIndexParallelism } return indexDef @@ -351,7 +382,7 @@ func GetClient( // Wait for cluster to be ready for { - _, err := avsClient.IndexList(ctx) + _, err := avsClient.IndexList(ctx, false) if err == nil { break } @@ -394,3 +425,14 @@ func GetCertificates(certFile string, keyFile string) ([]tls.Certificate, error) return client.LoadServerCertAndKey([]byte(cert), []byte(key), nil) } + +var quoted = false + +// Used with strings.FieldsFunc. Obviously not thread save. +func SplitQuotedString(r rune) bool { + if r == '"' { + quoted = !quoted + } + + return (r == ' ' && !quoted) || r == '"' +}