diff --git a/.golangci.yml b/.golangci.yml index f4ce6fe..e4a3fd2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -76,20 +76,18 @@ run: # skip-files: # - sample -# issues: -# exclude-rules: -# - path: info/as_parser_test\.go -# linters: -# - lll # Test code is allowed to have long lines -# - path: asconfig/generate_test\.go -# linters: -# - dupl # Test code is allowed to have duplicate code -# - path: asconfig/asconfig_test\.go -# linters: -# - dupl # Test code is allowed to have duplicate code -# - path: '(.+)test\.go' -# linters: -# - govet # Test code field alignment for sake of space is not a concern -# - linters: -# - lll -# source: "// " +issues: + exclude-rules: + - path: info/as_parser_test\.go + linters: + - lll # Test code is allowed to have long lines + - path: asconfig/generate_test\.go + linters: + - dupl # Test code is allowed to have duplicate code + - path: asconfig/asconfig_test\.go + linters: + - dupl # Test code is allowed to have duplicate code + - path: '(.+)test\.go' + linters: + - govet # Test code field alignment for sake of space is not a concern + - wsl # Auto generated tests cuddle assignments diff --git a/cmd/flags/constants.go b/cmd/flags/constants.go index e878176..17c3012 100644 --- a/cmd/flags/constants.go +++ b/cmd/flags/constants.go @@ -47,8 +47,10 @@ const ( BatchIndexInterval = "hnsw-batch-index-interval" BatchMaxReindexRecords = "hnsw-batch-max-reindex-records" BatchReindexInterval = "hnsw-batch-reindex-interval" - HnswCacheMaxEntries = "hnsw-cache-max-entries" - HnswCacheExpiry = "hnsw-cache-expiry" + HnswIndexCacheMaxEntries = "hnsw-index-cache-max-entries" + HnswIndexCacheExpiry = "hnsw-index-cache-expiry" + HnswRecordCacheMaxEntries = "hnsw-record-cache-max-entries" + HnswRecordCacheExpiry = "hnsw-record-cache-expiry" HnswHealerMaxScanRatePerNode = "hnsw-healer-max-scan-rate-per-node" HnswHealerMaxScanPageSize = "hnsw-healer-max-scan-page-size" HnswHealerReindexPercent = "hnsw-healer-reindex-percent" diff --git a/cmd/flags/hnsw.go b/cmd/flags/hnsw.go index c5c908c..ee57cdb 100644 --- a/cmd/flags/hnsw.go +++ b/cmd/flags/hnsw.go @@ -41,31 +41,59 @@ func (cf *BatchingFlags) NewSLogAttr() []any { } } -type CachingFlags struct { +type IndexCachingFlags struct { MaxEntries Uint64OptionalFlag Expiry InfDurationOptionalFlag } -func NewHnswCachingFlags() *CachingFlags { - return &CachingFlags{ +func NewHnswIndexCachingFlags() *IndexCachingFlags { + return &IndexCachingFlags{ MaxEntries: Uint64OptionalFlag{}, Expiry: InfDurationOptionalFlag{}, } } //nolint:lll // For readability -func (cf *CachingFlags) NewFlagSet() *pflag.FlagSet { +func (cf *IndexCachingFlags) NewFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.Var(&cf.MaxEntries, HnswCacheMaxEntries, "Maximum number of entries to cache.") - flagSet.Var(&cf.Expiry, HnswCacheExpiry, "A cache entry will expire after this amount of time has passed since the entry was added to cache, or 'inf' to never expire.") + flagSet.Var(&cf.MaxEntries, HnswIndexCacheMaxEntries, "Maximum number of entries to cache.") + flagSet.Var(&cf.Expiry, HnswIndexCacheExpiry, "A cache entry will expire after this amount of time has passed since the entry was added to cache, or -1 to never expire.") return flagSet } -func (cf *CachingFlags) NewSLogAttr() []any { +func (cf *IndexCachingFlags) NewSLogAttr() []any { return []any{ - slog.Any(HnswCacheMaxEntries, cf.MaxEntries.Val), - slog.String(HnswCacheExpiry, cf.Expiry.String()), + slog.Any(HnswIndexCacheMaxEntries, cf.MaxEntries.Val), + slog.String(HnswIndexCacheExpiry, cf.Expiry.String()), + } +} + +type RecordCachingFlags struct { + MaxEntries Uint64OptionalFlag + Expiry InfDurationOptionalFlag +} + +func NewHnswRecordCachingFlags() *RecordCachingFlags { + return &RecordCachingFlags{ + MaxEntries: Uint64OptionalFlag{}, + Expiry: InfDurationOptionalFlag{}, + } +} + +//nolint:lll // For readability +func (cf *RecordCachingFlags) NewFlagSet() *pflag.FlagSet { + flagSet := &pflag.FlagSet{} + flagSet.Var(&cf.MaxEntries, HnswRecordCacheMaxEntries, "Maximum number of entries to cache.") + flagSet.Var(&cf.Expiry, HnswRecordCacheExpiry, "A cache entry will expire after this amount of time has passed since the entry was added to cache, or -1 to never expire.") + + return flagSet +} + +func (cf *RecordCachingFlags) NewSLogAttr() []any { + return []any{ + slog.Any(HnswRecordCacheMaxEntries, cf.MaxEntries.Val), + slog.String(HnswRecordCacheExpiry, cf.Expiry.String()), } } diff --git a/cmd/flags/optionals.go b/cmd/flags/optionals.go index 8fbe1f6..203ed0d 100644 --- a/cmd/flags/optionals.go +++ b/cmd/flags/optionals.go @@ -209,8 +209,8 @@ func (f *DurationOptionalFlag) Int64() *int64 { return &milli } -// InfDurationOptionalFlag is a flag that can be either a time.duration or infinity. -// It is used for flags like --hnsw-cache-expiry which can be set to "infinity" +// InfDurationOptionalFlag is a flag that can be either a time.duration or -1 (never expire). +// It is used for flags like --hnsw-index-cache-expiry which can be set to never expire (-1) type InfDurationOptionalFlag struct { duration DurationOptionalFlag isInfinite bool @@ -224,7 +224,7 @@ func (f *InfDurationOptionalFlag) Set(val string) error { val = strings.ToLower(val) - if val == "inf" || val == "infinity" || val == "-1" { + if val == strconv.Itoa(Infinity) { f.isInfinite = true } else { return fmt.Errorf("invalid duration %s", val) @@ -239,7 +239,7 @@ func (f *InfDurationOptionalFlag) Type() string { func (f *InfDurationOptionalFlag) String() string { if f.isInfinite { - return "infinity" + return "-1" } if f.duration.Val != nil { diff --git a/cmd/flags/optionals_test.go b/cmd/flags/optionals_test.go index 08c9f1c..6659d8f 100644 --- a/cmd/flags/optionals_test.go +++ b/cmd/flags/optionals_test.go @@ -114,30 +114,12 @@ func (suite *OptionalFlagSuite) TestDurationOptionalFlag() { func (suite *OptionalFlagSuite) TestInfDurationOptionalFlag() { f := &InfDurationOptionalFlag{} - err := f.Set("inf") + err := f.Set("-1") if err != nil { suite.T().Errorf("Unexpected error: %v", err) } - suite.Equal("infinity", f.String()) - suite.Equal(int64(-1), *f.Int64()) - f = &InfDurationOptionalFlag{} - - err = f.Set("infinity") - if err != nil { - suite.T().Errorf("Unexpected error: %v", err) - } - - suite.Equal("infinity", f.String()) - suite.Equal(int64(-1), *f.Int64()) - f = &InfDurationOptionalFlag{} - - err = f.Set("-1") - if err != nil { - suite.T().Errorf("Unexpected error: %v", err) - } - - suite.Equal("infinity", f.String()) + suite.Equal("-1", f.String()) suite.Equal(int64(-1), *f.Int64()) f = &InfDurationOptionalFlag{} diff --git a/cmd/indexCreate.go b/cmd/indexCreate.go index 5040c4a..5878b18 100644 --- a/cmd/indexCreate.go +++ b/cmd/indexCreate.go @@ -38,7 +38,8 @@ var indexCreateFlags = &struct { hnswConstructionEf flags.Uint32OptionalFlag hnswMaxMemQueueSize flags.Uint32OptionalFlag hnswBatch flags.BatchingFlags - hnswCache flags.CachingFlags + hnswIndexCache flags.IndexCachingFlags + hnswRecordCache flags.RecordCachingFlags hnswHealer flags.HealerFlags hnswMerge flags.MergeFlags hnswVectorIntegrityCheck flags.BoolOptionalFlag @@ -52,7 +53,8 @@ var indexCreateFlags = &struct { hnswConstructionEf: flags.Uint32OptionalFlag{}, hnswMaxMemQueueSize: flags.Uint32OptionalFlag{}, hnswBatch: *flags.NewHnswBatchingFlags(), - hnswCache: *flags.NewHnswCachingFlags(), + hnswIndexCache: *flags.NewHnswIndexCachingFlags(), + hnswRecordCache: *flags.NewHnswRecordCachingFlags(), hnswHealer: *flags.NewHnswHealerFlags(), hnswMerge: *flags.NewHnswMergeFlags(), hnswVectorIntegrityCheck: flags.BoolOptionalFlag{}, @@ -77,7 +79,8 @@ func newIndexCreateFlagSet() *pflag.FlagSet { flagSet.Var(&indexCreateFlags.hnswMaxMemQueueSize, flags.HnswMaxMemQueueSize, "Maximum size of in-memory queue for inserted/updated vector records.") //nolint:lll // For readability //nolint:lll // For readability flagSet.Var(&indexCreateFlags.hnswVectorIntegrityCheck, flags.HnswVectorIntegrityCheck, "Enable/disable vector integrity check. Defaults to enabled.") //nolint:lll // For readability flagSet.AddFlagSet(indexCreateFlags.hnswBatch.NewFlagSet()) - flagSet.AddFlagSet(indexCreateFlags.hnswCache.NewFlagSet()) + flagSet.AddFlagSet(indexCreateFlags.hnswIndexCache.NewFlagSet()) + flagSet.AddFlagSet(indexCreateFlags.hnswRecordCache.NewFlagSet()) flagSet.AddFlagSet(indexCreateFlags.hnswHealer.NewFlagSet()) flagSet.AddFlagSet(indexCreateFlags.hnswMerge.NewFlagSet()) @@ -212,7 +215,8 @@ asvec index create -i myindex -n test -s testset -d 256 -m COSINE --%s vector \ RunE: func(_ *cobra.Command, _ []string) error { debugFlags := indexCreateFlags.clientFlags.NewSLogAttr() debugFlags = append(debugFlags, indexCreateFlags.hnswBatch.NewSLogAttr()...) - debugFlags = append(debugFlags, indexCreateFlags.hnswCache.NewSLogAttr()...) + debugFlags = append(debugFlags, indexCreateFlags.hnswIndexCache.NewSLogAttr()...) + debugFlags = append(debugFlags, indexCreateFlags.hnswRecordCache.NewSLogAttr()...) debugFlags = append(debugFlags, indexCreateFlags.hnswHealer.NewSLogAttr()...) debugFlags = append(debugFlags, indexCreateFlags.hnswMerge.NewSLogAttr()...) logger.Debug("parsed flags", @@ -340,8 +344,12 @@ func runCreateIndexFromFlags(client *avs.Client) error { ReindexInterval: indexCreateFlags.hnswBatch.ReindexInterval.Uint32(), }, IndexCachingParams: &protos.HnswCachingParams{ - MaxEntries: indexCreateFlags.hnswCache.MaxEntries.Val, - Expiry: indexCreateFlags.hnswCache.Expiry.Int64(), + MaxEntries: indexCreateFlags.hnswIndexCache.MaxEntries.Val, + Expiry: indexCreateFlags.hnswIndexCache.Expiry.Int64(), + }, + RecordCachingParams: &protos.HnswCachingParams{ + MaxEntries: indexCreateFlags.hnswRecordCache.MaxEntries.Val, + Expiry: indexCreateFlags.hnswRecordCache.Expiry.Int64(), }, HealerParams: &protos.HnswHealerParams{ MaxScanRatePerNode: indexCreateFlags.hnswHealer.MaxScanRatePerNode.Val, diff --git a/cmd/indexUpdate.go b/cmd/indexUpdate.go index 1b71b13..c73f4a2 100644 --- a/cmd/indexUpdate.go +++ b/cmd/indexUpdate.go @@ -20,7 +20,8 @@ var indexUpdateFlags = &struct { indexLabels map[string]string hnswMaxMemQueueSize flags.Uint32OptionalFlag hnswBatch flags.BatchingFlags - hnswCache flags.CachingFlags + hnswIndexCache flags.IndexCachingFlags + hnswRecordCache flags.RecordCachingFlags hnswHealer flags.HealerFlags hnswMerge flags.MergeFlags hnswVectorIntegrityCheck flags.BoolOptionalFlag @@ -28,7 +29,8 @@ var indexUpdateFlags = &struct { clientFlags: rootFlags.clientFlags, hnswMaxMemQueueSize: flags.Uint32OptionalFlag{}, hnswBatch: *flags.NewHnswBatchingFlags(), - hnswCache: *flags.NewHnswCachingFlags(), + hnswIndexCache: *flags.NewHnswIndexCachingFlags(), + hnswRecordCache: *flags.NewHnswRecordCachingFlags(), hnswHealer: *flags.NewHnswHealerFlags(), hnswMerge: *flags.NewHnswMergeFlags(), hnswVectorIntegrityCheck: flags.BoolOptionalFlag{}, @@ -43,7 +45,8 @@ func newIndexUpdateFlagSet() *pflag.FlagSet { flagSet.Var(&indexUpdateFlags.hnswMaxMemQueueSize, flags.HnswMaxMemQueueSize, "Maximum size of in-memory queue for inserted/updated vector records.") //nolint:lll // For readability flagSet.Var(&indexUpdateFlags.hnswVectorIntegrityCheck, flags.HnswVectorIntegrityCheck, "Enable/disable vector integrity check. Defaults to enabled.") //nolint:lll // For readability flagSet.AddFlagSet(indexUpdateFlags.hnswBatch.NewFlagSet()) - flagSet.AddFlagSet(indexUpdateFlags.hnswCache.NewFlagSet()) + flagSet.AddFlagSet(indexUpdateFlags.hnswIndexCache.NewFlagSet()) + flagSet.AddFlagSet(indexUpdateFlags.hnswRecordCache.NewFlagSet()) flagSet.AddFlagSet(indexUpdateFlags.hnswHealer.NewFlagSet()) flagSet.AddFlagSet(indexUpdateFlags.hnswMerge.NewFlagSet()) @@ -70,14 +73,15 @@ For example: %s asvec index update -i myindex -n test --%s 10000 --%s 10000ms --%s 10s --%s 16 --%s 16 `, HelpTxtSetupEnv, flags.BatchMaxIndexRecords, flags.BatchIndexInterval, - flags.HnswCacheExpiry, flags.HnswHealerParallelism, flags.HnswMergeParallelism), + flags.HnswIndexCacheExpiry, flags.HnswHealerParallelism, flags.HnswMergeParallelism), PreRunE: func(_ *cobra.Command, _ []string) error { return checkSeedsAndHost() }, RunE: func(_ *cobra.Command, _ []string) error { debugFlags := indexUpdateFlags.clientFlags.NewSLogAttr() debugFlags = append(debugFlags, indexUpdateFlags.hnswBatch.NewSLogAttr()...) - debugFlags = append(debugFlags, indexUpdateFlags.hnswCache.NewSLogAttr()...) + debugFlags = append(debugFlags, indexUpdateFlags.hnswIndexCache.NewSLogAttr()...) + debugFlags = append(debugFlags, indexUpdateFlags.hnswRecordCache.NewSLogAttr()...) debugFlags = append(debugFlags, indexUpdateFlags.hnswHealer.NewSLogAttr()...) debugFlags = append(debugFlags, indexUpdateFlags.hnswMerge.NewSLogAttr()...) logger.Debug("parsed flags", @@ -114,8 +118,12 @@ asvec index update -i myindex -n test --%s 10000 --%s 10000ms --%s 10s --%s 16 - MaxMemQueueSize: indexUpdateFlags.hnswMaxMemQueueSize.Val, BatchingParams: batchingParams, IndexCachingParams: &protos.HnswCachingParams{ - MaxEntries: indexUpdateFlags.hnswCache.MaxEntries.Val, - Expiry: indexUpdateFlags.hnswCache.Expiry.Int64(), + MaxEntries: indexUpdateFlags.hnswIndexCache.MaxEntries.Val, + Expiry: indexUpdateFlags.hnswIndexCache.Expiry.Int64(), + }, + RecordCachingParams: &protos.HnswCachingParams{ + MaxEntries: indexUpdateFlags.hnswRecordCache.MaxEntries.Val, + Expiry: indexUpdateFlags.hnswRecordCache.Expiry.Int64(), }, HealerParams: &protos.HnswHealerParams{ MaxScanRatePerNode: indexUpdateFlags.hnswHealer.MaxScanRatePerNode.Val, diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index 44b5f5a..819025e 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -30,11 +30,13 @@ func NewIndexTableWriter(writer io.Writer, verbose bool, logger *slog.Logger) *I "Dimensions", "Distance Metric", "Unmerged", + "Vector Records", + "Size", + "Unmerged %", } verboseHeadings := append(table.Row{}, headings...) verboseHeadings = append( verboseHeadings, - "Vector Records", "Vertices", "Labels*", "Storage", @@ -79,11 +81,13 @@ func (itw *IndexTableWriter) AppendIndexRow( index.Dimensions, index.VectorDistanceMetric, status.GetUnmergedRecordCount(), + status.GetIndexHealerVectorRecordsIndexed(), + formatBytes(calculateIndexSize(index, status)), + getPercentUnmerged(status), } if itw.verbose { row = append(row, - status.GetIndexHealerVectorRecordsIndexed(), status.GetIndexHealerVerticesValid(), index.Labels, ) @@ -107,8 +111,10 @@ func (itw *IndexTableWriter) AppendIndexRow( {"Batch Index Interval*", convertMillisecondToDuration(uint64(v.HnswParams.BatchingParams.GetIndexInterval()))}, {"Batch Max Reindex Records*", v.HnswParams.BatchingParams.GetMaxReindexRecords()}, {"Batch Reindex Interval*", convertMillisecondToDuration(uint64(v.HnswParams.BatchingParams.GetReindexInterval()))}, - {"Cache Max Entries*", v.HnswParams.IndexCachingParams.GetMaxEntries()}, - {"Cache Expiry*", convertMillisecondToDuration(v.HnswParams.IndexCachingParams.GetExpiry())}, + {"Index Cache Max Entries*", v.HnswParams.IndexCachingParams.GetMaxEntries()}, + {"Index Cache Expiry*", convertMillisecondToDuration(v.HnswParams.IndexCachingParams.GetExpiry())}, + {"Record Cache Max Entries*", v.HnswParams.RecordCachingParams.GetMaxEntries()}, + {"Record Cache Expiry*", convertMillisecondToDuration(v.HnswParams.RecordCachingParams.GetExpiry())}, {"Healer Max Scan Rate / Node*", v.HnswParams.HealerParams.GetMaxScanRatePerNode()}, {"Healer Max Page Size*", v.HnswParams.HealerParams.GetMaxScanPageSize()}, {"Healer Re-index % *", convertFloatToPercentStr(v.HnswParams.HealerParams.GetReindexPercent())}, @@ -143,3 +149,51 @@ func convertMillisecondToDuration[T int64 | uint64 | uint32](m T) time.Duration func convertFloatToPercentStr(f float32) string { return fmt.Sprintf("%.2f%%", f) } + +// calculateIndexSize calculates the size of the index in bytes +func calculateIndexSize(index *protos.IndexDefinition, status *protos.IndexStatusResponse) int64 { + // Each dimension is a float32 + vectorSize := int64(index.Dimensions) * 4 + // Each index record has ~500 bytes of overhead + the vector size + indexRecSize := 500 + vectorSize + // The total size is the number of records times the size of each record + return indexRecSize * status.GetIndexHealerVerticesValid() +} + +// formatBytes converts bytes to human readable string format +func formatBytes(bytes int64) string { + const ( + B = 1 + KB = 1024 * B + MB = 1024 * KB + GB = 1024 * MB + TB = 1024 * GB + PB = 1024 * TB + ) + + switch { + case bytes >= PB: + return fmt.Sprintf("%.2f PB", float64(bytes)/float64(PB)) + case bytes >= TB: + return fmt.Sprintf("%.2f TB", float64(bytes)/float64(TB)) + case bytes >= GB: + return fmt.Sprintf("%.2f GB", float64(bytes)/float64(GB)) + case bytes >= MB: + return fmt.Sprintf("%.2f MB", float64(bytes)/float64(MB)) + case bytes >= KB: + return fmt.Sprintf("%.2f KB", float64(bytes)/float64(KB)) + default: + return fmt.Sprintf("%d B", bytes) + } +} + +func getPercentUnmerged(status *protos.IndexStatusResponse) string { + unmergedCount := status.GetUnmergedRecordCount() + + verticies := status.GetIndexHealerVerticesValid() + if verticies == 0 { + return "0%" + } + + return fmt.Sprintf("%.2f%%", float64(unmergedCount)/float64(verticies)*100) +} diff --git a/cmd/writers/indexList_test.go b/cmd/writers/indexList_test.go new file mode 100644 index 0000000..2a3e23f --- /dev/null +++ b/cmd/writers/indexList_test.go @@ -0,0 +1,175 @@ +package writers + +import ( + "testing" + + "github.com/aerospike/avs-client-go/protos" +) + +func Test_calculateIndexSize(t *testing.T) { + type args struct { + index *protos.IndexDefinition + status *protos.IndexStatusResponse + } + tests := []struct { + name string + args args + want int64 + }{ + { + name: "positive simple", + args: args{ + index: &protos.IndexDefinition{ + Dimensions: 100, + }, + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 10, + }, + }, + want: 9000, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := calculateIndexSize(tt.args.index, tt.args.status); got != tt.want { + t.Errorf("calculateIndexSize() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_formatBytes(t *testing.T) { + type args struct { + bytes int64 + } + tests := []struct { + name string + args args + want string + }{ + { + name: "petabytes", + args: args{ + bytes: 1024 * 1024 * 1024 * 1024 * 1024, + }, + want: "1.00 PB", + }, + { + name: "terabytes", + args: args{ + bytes: 1024 * 1024 * 1024 * 1024, + }, + want: "1.00 TB", + }, + { + name: "gigabytes", + args: args{ + bytes: 1024 * 1024 * 1024, + }, + want: "1.00 GB", + }, + { + name: "megabytes", + args: args{ + bytes: 1024 * 1024, + }, + want: "1.00 MB", + }, + { + name: "kilobytes", + args: args{ + bytes: 1024, + }, + want: "1.00 KB", + }, + { + name: "bytes", + args: args{ + bytes: 512, + }, + want: "512 B", + }, + { + name: "zero bytes", + args: args{ + bytes: 0, + }, + want: "0 B", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatBytes(tt.args.bytes); got != tt.want { + t.Errorf("formatBytes() = %v, want %v", got, tt.want) + } + }) + } +} +func Test_getPercentUnmerged(t *testing.T) { + type args struct { + status *protos.IndexStatusResponse + } + tests := []struct { + name string + args args + want string + }{ + { + name: "zero vertices", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 0, + UnmergedRecordCount: 10, + }, + }, + want: "0%", + }, + { + name: "zero unmerged records", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 100, + UnmergedRecordCount: 0, + }, + }, + want: "0.00%", + }, + { + name: "50 percent unmerged", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 100, + UnmergedRecordCount: 50, + }, + }, + want: "50.00%", + }, + { + name: "100 percent unmerged", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 100, + UnmergedRecordCount: 100, + }, + }, + want: "100.00%", + }, + { + name: "33.33 percent unmerged", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 300, + UnmergedRecordCount: 100, + }, + }, + want: "33.33%", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getPercentUnmerged(tt.args.status); got != tt.want { + t.Errorf("getPercentUnmerged() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/e2e_test.go b/e2e_test.go index 33b3eec..218db75 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -219,10 +219,10 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { name: "test with hnsw cache params", indexName: "index4", indexNamespace: "test", - cmd: "index create -y -n test -i index4 -d 256 -m COSINE --vector-field vector4 --hnsw-cache-max-entries 1000 --hnsw-cache-expiry 10s", + cmd: "index create -y -n test -i index4 -d 256 -m COSINE --vector-field vector4 --hnsw-index-cache-max-entries 1000 --hnsw-index-cache-expiry 10s", expectedIndex: tests.NewIndexDefinitionBuilder(false, "index4", "test", 256, protos.VectorDistanceMetric_COSINE, "vector4"). - WithHnswCacheExpiry(10000). - WithHnswCacheMaxEntries(1000). + WithHnswIndexCacheExpiry(10000). + WithHnswIndexCacheMaxEntries(1000). Build(), }, { @@ -263,8 +263,10 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { WithHnswBatchingMaxIndexRecord(100001). WithHnswBatchingReindexInterval(30002). WithHnswBatchingMaxReindexRecord(100002). - WithHnswCacheMaxEntries(1001). - WithHnswCacheExpiry(1002). + WithHnswIndexCacheMaxEntries(1001). + WithHnswIndexCacheExpiry(1002). + WithHnswRecordCacheMaxEntries(1006). + WithHnswRecordCacheExpiry(1007). WithHnswHealerParallelism(7). WithHnswHealerMaxScanRatePerNode(1). WithHnswHealerMaxScanPageSize(2). @@ -290,6 +292,34 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { WithHnswVectorIntegrityCheck(false). Build(), }, + { + name: "test with record caching", + indexName: "recidx", + indexNamespace: "test", + cmd: "index create -y -n test -i recidx -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --hnsw-record-cache-max-entries 1001 --hnsw-record-cache-expiry 20s", + expectedIndex: tests.NewIndexDefinitionBuilder(false, "recidx", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0"). + WithHnswRecordCacheMaxEntries(1001). + WithHnswRecordCacheExpiry(20000). + Build(), + }, + { + name: "test with infinite record cache expiry", + indexName: "recinfidx", + indexNamespace: "test", + cmd: "index create -y -n test -i recinfidx -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --hnsw-record-cache-expiry -1", + expectedIndex: tests.NewIndexDefinitionBuilder(false, "recinfidx", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0"). + WithHnswRecordCacheExpiry(-1). + Build(), + }, + { + name: "test with infinite index cache expiry", + indexName: "idxinfidx", + indexNamespace: "test", + cmd: "index create -y -n test -i idxinfidx -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --hnsw-index-cache-expiry -1", + expectedIndex: tests.NewIndexDefinitionBuilder(false, "idxinfidx", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0"). + WithHnswIndexCacheExpiry(-1). + Build(), + }, } for _, tc := range testCases { @@ -515,10 +545,10 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() { name: "test with hnsw cache params", indexName: "successful-update", indexNamespace: "test", - cmd: "index update -y -n test -i successful-update --hnsw-cache-max-entries 1000 --hnsw-cache-expiry 10s", + cmd: "index update -y -n test -i successful-update --hnsw-index-cache-max-entries 1000 --hnsw-index-cache-expiry 10s", expectedIndex: newBuilder(). - WithHnswCacheExpiry(10000). - WithHnswCacheMaxEntries(1000). + WithHnswIndexCacheExpiry(10000). + WithHnswIndexCacheMaxEntries(1000). Build(), }, { @@ -721,8 +751,8 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { }, cmd: "index list --no-color --format 1", expectedTable: `Indexes -,Name,Namespace,Field,Dimensions,Distance Metric,Unmerged -1,list,test,vector,256,COSINE,0 +,Name,Namespace,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Size,Unmerged % +1,list,test,vector,256,COSINE,0,0,0 B,0% `, }, { @@ -737,9 +767,9 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { }, cmd: "index list --no-color --format 1", expectedTable: `Indexes -,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged -1,list2,bar,barset,vector,256,HAMMING,0 -2,list1,test,,vector,256,COSINE,0 +,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Size,Unmerged % +1,list2,bar,barset,vector,256,HAMMING,0,0,0 B,0% +2,list1,test,,vector,256,COSINE,0,0,0 B,0% `, }, { @@ -750,18 +780,23 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { ).WithLabels(map[string]string{"foo": "bar"}). WithHnswMergeIndexParallelism(80). WithHnswMergeReIndexParallelism(26). + WithHnswRecordCacheExpiry(20000). + WithHnswRecordCacheMaxEntries(1003). Build(), tests.NewIndexDefinitionBuilder(false, "list2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", ).WithSet("barset"). WithHnswMergeIndexParallelism(80). WithHnswMergeReIndexParallelism(26). + // -1 means never expire + WithHnswRecordCacheExpiry(-1). + WithHnswRecordCacheMaxEntries(1002). Build(), }, cmd: "index list --verbose --no-color --format 1", expectedTable: `Indexes -,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Vertices,Labels*,Storage,Index Parameters -1,list2,bar,barset,vector,256,HAMMING,0,0,0,map[],"Namespace\,bar +,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Size,Unmerged %,Vertices,Labels*,Storage,Index Parameters +1,list2,bar,barset,vector,256,HAMMING,0,0,0 B,0%,0,map[],"Namespace\,bar Set\,list2","HNSW Max Edges\,16 Ef\,100 @@ -771,8 +806,10 @@ Batch Max Index Records*\,100000 Batch Index Interval*\,30s Batch Max Reindex Records*\,10000 Batch Reindex Interval*\,30s -Cache Max Entries*\,2000000 -Cache Expiry*\,1h0m0s +Index Cache Max Entries*\,2000000 +Index Cache Expiry*\,1h0m0s +Record Cache Max Entries*\,1002 +Record Cache Expiry*\,-1ms Healer Max Scan Rate / Node*\,1000 Healer Max Page Size*\,10000 Healer Re-index % *\,10.00% @@ -781,7 +818,7 @@ Healer Parallelism*\,1 Merge Index Parallelism*\,80 Merge Re-Index Parallelism*\,26 Enable Vector Integrity Check\,true" -2,list1,test,,vector,256,COSINE,0,0,0,map[foo:bar],"Namespace\,test +2,list1,test,,vector,256,COSINE,0,0,0 B,0%,0,map[foo:bar],"Namespace\,test Set\,list1","HNSW Max Edges\,16 Ef\,100 @@ -791,8 +828,10 @@ Batch Max Index Records*\,100000 Batch Index Interval*\,30s Batch Max Reindex Records*\,10000 Batch Reindex Interval*\,30s -Cache Max Entries*\,2000000 -Cache Expiry*\,1h0m0s +Index Cache Max Entries*\,2000000 +Index Cache Expiry*\,1h0m0s +Record Cache Max Entries*\,1003 +Record Cache Expiry*\,20s Healer Max Scan Rate / Node*\,1000 Healer Max Page Size*\,10000 Healer Re-index % *\,10.00% @@ -1798,14 +1837,24 @@ func (suite *CmdTestSuite) TestFailInvalidArg() { expectedErrStr: "Error: invalid argument \"foo\" for \"--hnsw-batch-max-index-records\"", }, { - name: "test with bad hnsw-cache-max-entries", - cmd: "index create -y --hnsw-cache-max-entries foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", - expectedErrStr: "Error: invalid argument \"foo\" for \"--hnsw-cache-max-entries\"", + name: "test with bad hnsw-index-cache-max-entries", + cmd: "index create -y --hnsw-index-cache-max-entries foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", + expectedErrStr: "Error: invalid argument \"foo\" for \"--hnsw-index-cache-max-entries\"", + }, + { + name: "test with bad hnsw-index-cache-expiry", + cmd: "index create -y --hnsw-index-cache-expiry 10 --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", + expectedErrStr: "Error: invalid argument \"10\" for \"--hnsw-index-cache-expiry\"", + }, + { + name: "test with bad hnsw-record-cache-max-entries", + cmd: "index create -y --hnsw-record-cache-max-entries foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", + expectedErrStr: "Error: invalid argument \"foo\" for \"--hnsw-record-cache-max-entries\"", }, { - name: "test with bad hnsw-cache-expiry", - cmd: "index create -y --hnsw-cache-expiry 10 --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", - expectedErrStr: "Error: invalid argument \"10\" for \"--hnsw-cache-expiry\"", + name: "test with bad hnsw-record-cache-expiry", + cmd: "index create -y --hnsw-record-cache-expiry 10 --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar ", + expectedErrStr: "Error: invalid argument \"10\" for \"--hnsw-record-cache-expiry\"", }, { name: "test with bad hnsw-healer-max-scan-rate-per-node", diff --git a/go.mod b/go.mod index 9bbfb6f..6a9601e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module asvec go 1.22.5 require ( - github.com/aerospike/avs-client-go v0.0.0-20241022175841-0802ad7be5d7 + github.com/aerospike/avs-client-go v0.0.0-20241025173655-4553b2b412f8 github.com/aerospike/tools-common-go v0.0.0-20240927170813-c352c1917359 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.1 diff --git a/go.sum b/go.sum index a3ec3f0..834e295 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/aerospike/avs-client-go v0.0.0-20241021200908-69baf3bf651d h1:IO+H1Qe github.com/aerospike/avs-client-go v0.0.0-20241021200908-69baf3bf651d/go.mod h1:gveJAyzXlND0g/lA+R6USRg9UcnGAUtyHzcngWrHSTo= github.com/aerospike/avs-client-go v0.0.0-20241022175841-0802ad7be5d7 h1:V9MX8SxKPld/jotSLE6VfMaSGUnyZ5s77TlIz7RrS4M= github.com/aerospike/avs-client-go v0.0.0-20241022175841-0802ad7be5d7/go.mod h1:gveJAyzXlND0g/lA+R6USRg9UcnGAUtyHzcngWrHSTo= +github.com/aerospike/avs-client-go v0.0.0-20241025173655-4553b2b412f8 h1:5awce0R9Sn6W7ywQdoKaNG2zWoG15B0fPed615nUN/o= +github.com/aerospike/avs-client-go v0.0.0-20241025173655-4553b2b412f8/go.mod h1:gveJAyzXlND0g/lA+R6USRg9UcnGAUtyHzcngWrHSTo= github.com/aerospike/tools-common-go v0.0.0-20240927170813-c352c1917359 h1:NXCigFN6GWJizwJSe7xAp5kTz7ZxdNjpyRxaIp6MkuA= github.com/aerospike/tools-common-go v0.0.0-20240927170813-c352c1917359/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/tests/indexDef.yaml b/tests/indexDef.yaml index f4cf581..0df539f 100644 --- a/tests/indexDef.yaml +++ b/tests/indexDef.yaml @@ -10,6 +10,9 @@ indices: indexCachingParams: maxEntries: 1001 expiry: 1002 + recordCachingParams: + maxEntries: 1006 + expiry: 1007 ef: 101 efConstruction: 102 healerParams: diff --git a/tests/utils.go b/tests/utils.go index a08b7d9..0e21039 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -42,8 +42,10 @@ type IndexDefinitionBuilder struct { hnswBatchingInterval *uint32 hnswBatchingMaxReindexRecord *uint32 hnswBatchingReindexInterval *uint32 - hnswCacheExpiry *int64 - hnswCacheMaxEntries *uint64 + hnswIndexCacheExpiry *int64 + hnswIndexCacheMaxEntries *uint64 + hnswRecordCacheExpiry *int64 + hnswRecordCacheMaxEntries *uint64 hnswHealerMaxScanPageSize *uint32 hnswHealerMaxScanRatePerSecond *uint32 hnswHealerParallelism *uint32 @@ -132,13 +134,23 @@ func (idb *IndexDefinitionBuilder) WithHnswBatchingReindexInterval(interval uint return idb } -func (idb *IndexDefinitionBuilder) WithHnswCacheExpiry(expiry int64) *IndexDefinitionBuilder { - idb.hnswCacheExpiry = &expiry +func (idb *IndexDefinitionBuilder) WithHnswIndexCacheExpiry(expiry int64) *IndexDefinitionBuilder { + idb.hnswIndexCacheExpiry = &expiry return idb } -func (idb *IndexDefinitionBuilder) WithHnswCacheMaxEntries(maxEntries uint64) *IndexDefinitionBuilder { - idb.hnswCacheMaxEntries = &maxEntries +func (idb *IndexDefinitionBuilder) WithHnswIndexCacheMaxEntries(maxEntries uint64) *IndexDefinitionBuilder { + idb.hnswIndexCacheMaxEntries = &maxEntries + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswRecordCacheExpiry(expiry int64) *IndexDefinitionBuilder { + idb.hnswRecordCacheExpiry = &expiry + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswRecordCacheMaxEntries(maxEntries uint64) *IndexDefinitionBuilder { + idb.hnswRecordCacheMaxEntries = &maxEntries return idb } @@ -198,9 +210,10 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { Params: &protos.IndexDefinition_HnswParams{ HnswParams: &protos.HnswParams{ // BatchingParams: &protos.HnswBatchingParams{}, - IndexCachingParams: &protos.HnswCachingParams{}, - HealerParams: &protos.HnswHealerParams{}, - MergeParams: &protos.HnswIndexMergeParams{}, + IndexCachingParams: &protos.HnswCachingParams{}, + RecordCachingParams: &protos.HnswCachingParams{}, + HealerParams: &protos.HnswHealerParams{}, + MergeParams: &protos.HnswIndexMergeParams{}, }, }, } @@ -216,10 +229,11 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { Storage: &protos.IndexStorage{}, Params: &protos.IndexDefinition_HnswParams{ HnswParams: &protos.HnswParams{ - BatchingParams: &protos.HnswBatchingParams{}, - IndexCachingParams: &protos.HnswCachingParams{}, - HealerParams: &protos.HnswHealerParams{}, - MergeParams: &protos.HnswIndexMergeParams{}, + BatchingParams: &protos.HnswBatchingParams{}, + IndexCachingParams: &protos.HnswCachingParams{}, + RecordCachingParams: &protos.HnswCachingParams{}, + HealerParams: &protos.HnswHealerParams{}, + MergeParams: &protos.HnswIndexMergeParams{}, }, }, } @@ -278,12 +292,20 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.MaxMemQueueSize = idb.hnswMemQueueSize } - if idb.hnswCacheExpiry != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.IndexCachingParams.Expiry = idb.hnswCacheExpiry + if idb.hnswIndexCacheExpiry != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.IndexCachingParams.Expiry = idb.hnswIndexCacheExpiry + } + + if idb.hnswIndexCacheMaxEntries != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.IndexCachingParams.MaxEntries = idb.hnswIndexCacheMaxEntries + } + + if idb.hnswRecordCacheExpiry != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.RecordCachingParams.Expiry = idb.hnswRecordCacheExpiry } - if idb.hnswCacheMaxEntries != nil { - indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.IndexCachingParams.MaxEntries = idb.hnswCacheMaxEntries + if idb.hnswRecordCacheMaxEntries != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.RecordCachingParams.MaxEntries = idb.hnswRecordCacheMaxEntries } if idb.hnswHealerMaxScanPageSize != nil {