VEC-317 0.10.0 support (#11)

* !feat: VEC-317 support AVS 0.9.1
- change --hnsw-max-records to --hnsw-m
- change --hnsw-healer-schedule-delay to --hnsw-healer-schedule
- change --hnsw-merge-parallelism to --hnsw-merge-index-parallelism
- add --hnsw-merge-reindex-parallelism
* fix: VEC-333 add symlink in deb and rpm installs
* test: Use avs 0.10.0 in tests

.github/workflows/create-prerelease.yml

@@ -34,11 +34,11 @@ jobs:
               run: |
                 /usr/local/bin/brew install --overwrite [email protected] || echo "I1.1"
                 /usr/local/bin/brew link --overwrite [email protected] || echo "I1.2"
-                /usr/local/bin/brew install --overwrite dpkg upx zip make wget jq rpm || echo "I2"
+                /usr/local/bin/brew install --overwrite dpkg zip make wget jq rpm || echo "I2"
                 /usr/local/bin/brew link --overwrite [email protected] || echo "I1.3"
                 /usr/local/bin/brew install [email protected] || echo "I1.4"
                 /usr/local/bin/brew install [email protected] || echo "I1.5"
-                for i in dpkg upx zip make wget jq rpm python3.11; do command -v $i || exit 1; done
+                for i in dpkg zip make wget jq rpm python3.11; do command -v $i || exit 1; done
                 echo "Dependencies checked"
             - name: Get go version from go.mod
               run: |

Makefile

@@ -329,9 +329,11 @@ pkg-deb-amd64:
 	cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec
 	rm -rf $(BIN_DIR)/deb
 	mkdir -p $(BIN_DIR)/deb/DEBIAN
-	mkdir -p $(BIN_DIR)/deb/usr/local/aerospike/bin
+	mkdir -p $(BIN_DIR)/deb/opt/aerospike/bin
+	mkdir -p $(BIN_DIR)/deb/usr/bin
 	@ eval "$$amddebscript"
-	mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/local/aerospike/bin/
+	mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/opt/aerospike/bin/
+	ln -s /opt/aerospike/bin/asvec $(BIN_DIR)/deb/usr/bin/asvec
 	sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb
 	rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb
 	mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb
@@ -342,9 +344,11 @@ pkg-deb-arm64:
 	cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec
 	rm -rf $(BIN_DIR)/deb
 	mkdir -p $(BIN_DIR)/deb/DEBIAN
-	mkdir -p $(BIN_DIR)/deb/usr/local/aerospike/bin
+	mkdir -p $(BIN_DIR)/deb/opt/aerospike/bin
+	mkdir -p $(BIN_DIR)/deb/usr/bin
 	@ eval "$$armdebscript"
-	mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/local/aerospike/bin/
+	mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/opt/aerospike/bin/
+	ln -s /opt/aerospike/bin/asvec $(BIN_DIR)/deb/usr/bin/asvec
 	sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb
 	rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb
 	mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb
@@ -391,8 +395,10 @@ pkg-zip: pkg-zip-amd64 pkg-zip-arm64
 pkg-rpm-amd64:
 	rm -rf $(BIN_DIR)/asvec-rpm-centos
 	cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/usr/bin
 	sed -i.bak "s/VERSIONHERE/${rpm_ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec
-	cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/usr/local/aerospike/bin/asvec
+	cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin/asvec
 	rm -f $(BIN_DIR)/asvec-linux-x86_64.rpm
 	bash -ce "cd $(BIN_DIR) && rpmbuild --target=x86_64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec"
 	rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${rpm_ver}.rpm
@@ -402,8 +408,10 @@ pkg-rpm-amd64:
 pkg-rpm-arm64:
 	rm -rf $(BIN_DIR)/asvec-rpm-centos
 	cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/usr/bin
 	sed -i.bak "s/VERSIONHERE/${rpm_ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec
-	cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/usr/local/aerospike/bin/asvec
+	cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin/asvec
 	rm -f $(BIN_DIR)/asvec-linux-arm64.rpm
 	bash -ce "cd $(BIN_DIR) && rpmbuild --target=arm64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec"
 	rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${rpm_ver}.rpm

README.md

@@ -30,8 +30,12 @@ asvec --help
 > [!NOTE]
 > More features are in the works. Don't worry!
 
+- **Data Browsing**: Easily run queries on an index.
 - **Index Management**: Listing, creating, and dropping indexes.
-- **User Management**: Listing, creating, and dropping users. Revoking and granting user's roles.
+- **User Management**: Listing, creating, and dropping users. Revoking and
+  granting user's roles.
+- **Node visibility**: Listing nodes and important metadata i.e. version, peers,
+  etc.
 
 ## Issues
 

bin/asvecrpm/asvec.spec

@@ -1,7 +1,7 @@
 Buildroot: ./
 Name: asvec
 Version: VERSIONHERE
-Release: 2
+Release: 1
 Summary: Tool for deploying non-prod Aerospike server clusters on docker or in AWS
 License: see github.com/aerospike/asvec
 Group: aerospike
@@ -12,9 +12,16 @@ Group: aerospike
 %define _binaries_in_noarch_packages_terminate_build 0
 
 %description
-
-
 Tool for deploying non-prod Aerospike server clusters on docker or in AWS
 
 %files
-"/usr/local/aerospike/bin/asvec"
+/opt/aerospike/bin/asvec
+/usr/bin/asvec
+
+%install
+# Ensure the buildroot directories exist
+mkdir -p %{buildroot}/opt/aerospike/bin
+mkdir -p %{buildroot}/usr/bin
+
+%prep
+ln -sf /opt/aerospike/bin/asvec %{buildroot}/usr/bin/asvec

cmd/flags/constants.go

@@ -37,7 +37,7 @@ const (
 	StorageNamespace             = "storage-namespace"
 	StorageSet                   = "storage-set"
 	CutoffTime                   = "cutoff-time"
-	HnswMaxEdges                 = "hnsw-max-edges"
+	HnswMaxEdges                 = "hnsw-m"
 	HnswConstructionEf           = "hnsw-ef-construction"
 	HnswEf                       = "hnsw-ef"
 	HnswMaxMemQueueSize          = "hnsw-max-mem-queue-size"
@@ -48,9 +48,10 @@ const (
 	HnswHealerMaxScanRatePerNode = "hnsw-healer-max-scan-rate-per-node"
 	HnswHealerMaxScanPageSize    = "hnsw-healer-max-scan-page-size"
 	HnswHealerReindexPercent     = "hnsw-healer-reindex-percent"
-	HnswHealerScheduleDelay      = "hnsw-healer-schedule-delay"
+	HnswHealerSchedule           = "hnsw-healer-schedule"
 	HnswHealerParallelism        = "hnsw-healer-parallelism"
-	HnswMergeParallelism         = "hnsw-merge-parallelism"
+	HnswMergeParallelism         = "hnsw-merge-index-parallelism"
+	HnswMergeReIndexParallelism  = "hnsw-merge-reindex-parallelism"
 	TLSProtocols                 = "tls-protocols"
 	TLSCaFile                    = "tls-cafile"
 	TLSCaPath                    = "tls-capath"

cmd/flags/hnsw.go

@@ -64,7 +64,7 @@ type HealerFlags struct {
 	MaxScanRatePerNode Uint32OptionalFlag
 	MaxScanPageSize    Uint32OptionalFlag
 	ReindexPercent     Float32OptionalFlag
-	ScheduleDelay      DurationOptionalFlag
+	Schedule           StringOptionalFlag
 	Parallelism        Uint32OptionalFlag
 }
 
@@ -73,18 +73,18 @@ func NewHnswHealerFlags() *HealerFlags {
 		MaxScanRatePerNode: Uint32OptionalFlag{},
 		MaxScanPageSize:    Uint32OptionalFlag{},
 		ReindexPercent:     Float32OptionalFlag{},
-		ScheduleDelay:      DurationOptionalFlag{},
+		Schedule:           StringOptionalFlag{},
 		Parallelism:        Uint32OptionalFlag{},
 	}
 }
 
 func (cf *HealerFlags) NewFlagSet() *pflag.FlagSet {
 	flagSet := &pflag.FlagSet{}
-	flagSet.Var(&cf.MaxScanRatePerNode, HnswHealerMaxScanRatePerNode, "Maximum allowed record scan rate per AVS node.")                                                  //nolint:lll // For readability
-	flagSet.Var(&cf.MaxScanPageSize, HnswHealerMaxScanPageSize, "Maximum number of records in a single scanned page.")                                                   //nolint:lll // For readability
-	flagSet.Var(&cf.ReindexPercent, HnswHealerReindexPercent, "Percentage of good records randomly selected for reindexing in a healer cycle.")                          //nolint:lll // For readability
-	flagSet.Var(&cf.ScheduleDelay, HnswHealerScheduleDelay, "The time delay between the termination of a healer run and the commencement of the next one for an index.") //nolint:lll // For readability
-	flagSet.Var(&cf.Parallelism, HnswHealerParallelism, "Maximum number of records to heal in parallel.")                                                                //nolint:lll // For readability
+	flagSet.Var(&cf.MaxScanRatePerNode, HnswHealerMaxScanRatePerNode, "Maximum allowed record scan rate per AVS node.")                           //nolint:lll // For readability
+	flagSet.Var(&cf.MaxScanPageSize, HnswHealerMaxScanPageSize, "Maximum number of records in a single scanned page.")                            //nolint:lll // For readability
+	flagSet.Var(&cf.ReindexPercent, HnswHealerReindexPercent, "Percentage of good records randomly selected for reindexing in a healer cycle.")   //nolint:lll // For readability
+	flagSet.Var(&cf.Schedule, HnswHealerSchedule, "The quartz cron expression defining the schedule at which the index healer cycle is invoked.") //nolint:lll // For readability
+	flagSet.Var(&cf.Parallelism, HnswHealerParallelism, "Maximum number of records to heal in parallel.")                                         //nolint:lll // For readability
 
 	return flagSet
 }
@@ -94,30 +94,31 @@ func (cf *HealerFlags) NewSLogAttr() []any {
 		slog.Any(HnswHealerMaxScanRatePerNode, cf.MaxScanRatePerNode.String()),
 		slog.Any(HnswHealerMaxScanPageSize, cf.MaxScanPageSize.String()),
 		slog.Any(HnswHealerReindexPercent, cf.ReindexPercent.String()),
-		slog.Any(HnswHealerScheduleDelay, cf.ScheduleDelay.String()),
+		slog.Any(HnswHealerSchedule, cf.Schedule.String()),
 		slog.Any(HnswHealerParallelism, cf.Parallelism.String()),
 	}
 }
 
 type MergeFlags struct {
-	Parallelism Uint32OptionalFlag
+	IndexParallelism   Uint32OptionalFlag
+	ReIndexParallelism Uint32OptionalFlag
 }
 
 func NewHnswMergeFlags() *MergeFlags {
-	return &MergeFlags{
-		Parallelism: Uint32OptionalFlag{},
-	}
+	return &MergeFlags{}
 }
 
 func (cf *MergeFlags) NewFlagSet() *pflag.FlagSet {
 	flagSet := &pflag.FlagSet{}
-	flagSet.Var(&cf.Parallelism, HnswMergeParallelism, "The number of vectors merged in parallel from a batch index to main index.") //nolint:lll // For readability
+	flagSet.Var(&cf.IndexParallelism, HnswMergeParallelism, "The number of vectors merged in parallel from a batch index to main index.")                                  //nolint:lll // For readability
+	flagSet.Var(&cf.ReIndexParallelism, HnswMergeReIndexParallelism, "The number of vectors merged in parallel from a re-indexing record batch-index to the main index. ") //nolint:lll // For readability
 
 	return flagSet
 }
 
 func (cf *MergeFlags) NewSLogAttr() []any {
 	return []any{
-		slog.Any(HnswMergeParallelism, cf.Parallelism.Val),
+		slog.Any(HnswMergeParallelism, cf.IndexParallelism.Val),
+		slog.Any(HnswMergeReIndexParallelism, cf.ReIndexParallelism.Val),
 	}
 }

cmd/indexCreate.go

@@ -341,11 +341,12 @@ func runCreateIndexFromFlags(client *avs.Client) error {
 				MaxScanRatePerNode: indexCreateFlags.hnswHealer.MaxScanRatePerNode.Val,
 				MaxScanPageSize:    indexCreateFlags.hnswHealer.MaxScanPageSize.Val,
 				ReindexPercent:     indexCreateFlags.hnswHealer.ReindexPercent.Val,
-				ScheduleDelay:      indexCreateFlags.hnswHealer.ScheduleDelay.Uint64(),
+				Schedule:           indexCreateFlags.hnswHealer.Schedule.Val,
 				Parallelism:        indexCreateFlags.hnswHealer.Parallelism.Val,
 			},
 			MergeParams: &protos.HnswIndexMergeParams{
-				Parallelism: indexCreateFlags.hnswMerge.Parallelism.Val,
+				IndexParallelism:   indexCreateFlags.hnswMerge.IndexParallelism.Val,
+				ReIndexParallelism: indexCreateFlags.hnswMerge.ReIndexParallelism.Val,
 			},
 		},
 	}

cmd/indexList.go

@@ -73,7 +73,7 @@ asvec index ls
 			ctx, cancel := context.WithTimeout(context.Background(), indexListFlags.clientFlags.Timeout)
 			defer cancel()
 
-			indexList, err := client.IndexList(ctx)
+			indexList, err := client.IndexList(ctx, true)
 			if err != nil {
 				logger.Error("failed to list indexes", slog.Any("error", err))
 				return err

cmd/indexUpdate.go

@@ -112,11 +112,12 @@ asvec index update -i myindex -n test --%s 10000 --%s 10000ms --%s 10s --%s 16 -
 					MaxScanRatePerNode: indexUpdateFlags.hnswHealer.MaxScanRatePerNode.Val,
 					MaxScanPageSize:    indexUpdateFlags.hnswHealer.MaxScanPageSize.Val,
 					ReindexPercent:     indexUpdateFlags.hnswHealer.ReindexPercent.Val,
-					ScheduleDelay:      indexUpdateFlags.hnswHealer.ScheduleDelay.Uint64(),
+					Schedule:           indexUpdateFlags.hnswHealer.Schedule.Val,
 					Parallelism:        indexUpdateFlags.hnswHealer.Parallelism.Val,
 				},
 				MergeParams: &protos.HnswIndexMergeParams{
-					Parallelism: indexUpdateFlags.hnswMerge.Parallelism.Val,
+					IndexParallelism:   indexUpdateFlags.hnswMerge.IndexParallelism.Val,
+					ReIndexParallelism: indexUpdateFlags.hnswMerge.ReIndexParallelism.Val,
 				},
 			}
 

cmd/query.go

@@ -146,7 +146,7 @@ asvec query -i my-index -n my-namespace -v "[1,0,1,0,0,0,1,0,1,1]" --max-keys 10
 					return
 				}
 			} else {
-				indexDef, err := client.IndexGet(ctx, queryFlags.namespace, queryFlags.indexName)
+				indexDef, err := client.IndexGet(ctx, queryFlags.namespace, queryFlags.indexName, false)
 				if err != nil {
 					logger.ErrorContext(ctx, "unable to get index definition", slog.Any("error", err))
 					view.Errorf("Failed to get index definition: %s", err)

cmd/writers/indexList.go

@@ -82,9 +82,10 @@ func (itw *IndexTableWriter) AppendIndexRow(
 				{"Healer Max Scan Rate / Node*", v.HnswParams.HealerParams.GetMaxScanRatePerNode()},
 				{"Healer Max Page Size*", v.HnswParams.HealerParams.GetMaxScanPageSize()},
 				{"Healer Re-index % *", convertFloatToPercentStr(v.HnswParams.HealerParams.GetReindexPercent())},
-				{"Healer Schedule Delay*", convertMillisecondToDuration(v.HnswParams.HealerParams.GetScheduleDelay())},
+				{"Healer Schedule*", v.HnswParams.HealerParams.GetSchedule()},
 				{"Healer Parallelism*", v.HnswParams.HealerParams.GetParallelism()},
-				{"Merge Parallelism*", v.HnswParams.MergeParams.GetParallelism()},
+				{"Merge Index Parallelism*", v.HnswParams.MergeParams.GetIndexParallelism()},
+				{"Merge Re-Index Parallelism*", v.HnswParams.MergeParams.GetReIndexParallelism()},
 			})
 
 			row = append(row, renderTable(tHNSW, format))

docker/auth/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       timeout: 20s
       retries: 20
   avs:
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     depends_on:
       aerospike:
         condition: service_healthy

docker/mtls/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       timeout: 20s
       retries: 20
   avs:
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     depends_on:
       aerospike:
         condition: service_healthy

docker/multi-node-LB/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     volumes:
       - ./config/aerospike-vector-search-1.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml
       - ./config/features.conf:/etc/aerospike-vector-search/features.conf
@@ -35,7 +35,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     volumes:
       - ./config/aerospike-vector-search-2.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml
       - ./config/features.conf:/etc/aerospike-vector-search/features.conf
@@ -50,7 +50,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     volumes:
       - ./config/aerospike-vector-search-3.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml
       - ./config/features.conf:/etc/aerospike-vector-search/features.conf

docker/multi-node-client-visibility-err/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     ports:
       - "10000:10000"
     volumes:
@@ -37,7 +37,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     ports:
       - "10001:10001"
     volumes:
@@ -54,7 +54,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     # ports:
     #   - "10002:10002" # This causes the visibility err
     volumes:

docker/multi-node/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     ports:
       - "10000:10000"
     volumes:
@@ -37,7 +37,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     ports:
       - "10001:10001"
     volumes:
@@ -54,7 +54,7 @@ services:
     depends_on:
       aerospike:
         condition: service_healthy
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     ports:
       - "10002:10002"
     volumes:

docker/tls/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       timeout: 20s
       retries: 20
   avs:
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     depends_on:
       aerospike:
         condition: service_healthy

docker/vanilla/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       timeout: 20s
       retries: 20
   avs:
-    image: aerospike/aerospike-vector-search:0.9.0
+    image: aerospike/aerospike-vector-search:0.10.0
     depends_on:
       aerospike:
         condition: service_healthy