From 955053245221babd660620d7177344610eb7c377 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Wed, 6 Nov 2024 21:02:48 +0800 Subject: [PATCH 01/19] Add leaderboard configs --- config/config.go | 22 ++++++++++++++++++++++ config/config.toml | 11 +++++++++++ config/config_test.go | 5 +++++ go.mod | 10 ++++++---- go.sum | 31 ++++++++++++++----------------- 5 files changed, 58 insertions(+), 21 deletions(-) diff --git a/config/config.go b/config/config.go index f13d92f0b..2fc3af02a 100644 --- a/config/config.go +++ b/config/config.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/expr-lang/expr/parser" "github.com/go-playground/locales/en" ut "github.com/go-playground/universal-translator" "github.com/go-playground/validator/v10" @@ -104,6 +105,7 @@ type RecommendConfig struct { CacheExpire time.Duration `mapstructure:"cache_expire" validate:"gt=0"` ActiveUserTTL int `mapstructure:"active_user_ttl" validate:"gte=0"` DataSource DataSourceConfig `mapstructure:"data_source"` + LeaderBoards []LeaderBoardConfig `mapstructure:"leaderboards" validate:"dive"` Popular PopularConfig `mapstructure:"popular"` UserNeighbors NeighborsConfig `mapstructure:"user_neighbors"` ItemNeighbors NeighborsConfig `mapstructure:"item_neighbors"` @@ -120,6 +122,12 @@ type DataSourceConfig struct { ItemTTL uint `mapstructure:"item_ttl" validate:"gte=0"` // item-to-live of items } +type LeaderBoardConfig struct { + Name string `mapstructure:"name"` + Score string `mapstructure:"score" validate:"required,item_expr"` + Filter string `mapstructure:"filter" validate:"item_expr"` +} + type PopularConfig struct { PopularWindow time.Duration `mapstructure:"popular_window" validate:"gte=0"` } @@ -645,6 +653,12 @@ func (config *Config) Validate(oneModel bool) error { }); err != nil { return errors.Trace(err) } + if err := validate.RegisterValidation("item_expr", func(fl validator.FieldLevel) bool { + _, err := parser.Parse(fl.Field().String()) + return err == nil + }); err != nil { + return errors.Trace(err) + } validate.RegisterTagNameFunc(func(fld reflect.StructField) string { return strings.SplitN(fld.Tag.Get("mapstructure"), ",", 2)[0] }) @@ -671,6 +685,14 @@ func (config *Config) Validate(oneModel bool) error { }); err != nil { return errors.Trace(err) } + if err := validate.RegisterTranslation("item_expr", trans, func(ut ut.Translator) error { + return ut.Add("item_expr", "invalid item expression", true) + }, func(ut ut.Translator, fe validator.FieldError) string { + t, _ := ut.T("item_expr", fe.Field()) + return t + }); err != nil { + return errors.Trace(err) + } errs := err.(validator.ValidationErrors) for _, e := range errs { return errors.New(e.Translate(trans)) diff --git a/config/config.toml b/config/config.toml index e6fb194a5..0241709bf 100644 --- a/config/config.toml +++ b/config/config.toml @@ -116,6 +116,17 @@ item_ttl = 0 # The time window of popular items. The default values is 4320h. popular_window = "720h" +[[recommend.leaderboards]] + +# The name of the leaderboard. +name = "most_starred_weekly" + +# The score function for items in the leaderboard. +score = "count(Feedback.FeedbackType, .FeedbackType == 'star')" + +# The filter for items in the leaderboard. +filter = "(Item.Timestamp > now()).Hours() < 168" + [recommend.user_neighbors] # The type of neighbors for users. There are three types: diff --git a/config/config_test.go b/config/config_test.go index 63c222c4e..7d8dbdeaa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -90,6 +90,11 @@ func TestUnmarshal(t *testing.T) { assert.Equal(t, uint(0), config.Recommend.DataSource.ItemTTL) // [recommend.popular] assert.Equal(t, 30*24*time.Hour, config.Recommend.Popular.PopularWindow) + // [recommend.leaderboards] + assert.Len(t, config.Recommend.LeaderBoards, 1) + assert.Equal(t, "most_starred_weekly", config.Recommend.LeaderBoards[0].Name) + assert.Equal(t, "count(Feedback.FeedbackType, .FeedbackType == 'star')", config.Recommend.LeaderBoards[0].Score) + assert.Equal(t, "Item.Timestamp > now() < 7d", config.Recommend.LeaderBoards[0].Filter) // [recommend.user_neighbors] assert.Equal(t, "similar", config.Recommend.UserNeighbors.NeighborType) assert.True(t, config.Recommend.UserNeighbors.EnableIndex) diff --git a/go.mod b/go.mod index ad9b3ee3d..7dc83154a 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,11 @@ require ( github.com/deckarep/golang-set/v2 v2.3.1 github.com/emicklei/go-restful-openapi/v2 v2.9.0 github.com/emicklei/go-restful/v3 v3.9.0 + github.com/expr-lang/expr v1.16.9 github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449 - github.com/go-playground/locales v0.14.0 - github.com/go-playground/universal-translator v0.18.0 - github.com/go-playground/validator/v10 v10.11.0 + github.com/go-playground/locales v0.14.1 + github.com/go-playground/universal-translator v0.18.1 + github.com/go-playground/validator/v10 v10.22.1 github.com/go-resty/resty/v2 v2.7.0 github.com/go-sql-driver/mysql v1.6.0 github.com/golang/protobuf v1.5.2 @@ -93,6 +94,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -120,7 +122,7 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.16 // indirect diff --git a/go.sum b/go.sum index ab7e3b29f..65157b34e 100644 --- a/go.sum +++ b/go.sum @@ -141,6 +141,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= +github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -152,6 +154,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449 h1:HOYnhuVrhAVGKdg3rZapII640so7QfXQmkLkefUN/uM= github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449/go.mod h1:i+vbdOOivRRh2j+WwBkjZXloGN/+KAqfKDwNfUJeugc= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= @@ -192,14 +196,14 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2BOGlCyvTqsp/xIw= -github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= @@ -419,8 +423,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -433,8 +435,8 @@ github.com/lafikl/consistent v0.0.0-20220512074542-bdd3606bfc3e/go.mod h1:JmowIn github.com/leesper/go_rng v0.0.0-20171009123644-5344a9259b21/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -501,7 +503,6 @@ github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -553,8 +554,6 @@ github.com/rivo/uniseg v0.3.4 h1:3Z3Eu6FGHZWSfNKJTOUiPatWwfc7DzJRU04jFUqJODw= github.com/rivo/uniseg v0.3.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -714,7 +713,6 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= @@ -907,7 +905,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From c24b449b2aeffd86de5bbf974ab9dc35bd6475a1 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Wed, 6 Nov 2024 21:29:16 +0800 Subject: [PATCH 02/19] Add leaderboard recommender --- recommend/leaderboard.go | 90 +++++++++++++++++++++++++++++++++++ recommend/leaderboard_test.go | 20 ++++++++ 2 files changed, 110 insertions(+) create mode 100644 recommend/leaderboard.go create mode 100644 recommend/leaderboard_test.go diff --git a/recommend/leaderboard.go b/recommend/leaderboard.go new file mode 100644 index 000000000..d8558307a --- /dev/null +++ b/recommend/leaderboard.go @@ -0,0 +1,90 @@ +// Copyright 2024 gorse Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recommend + +import ( + "github.com/expr-lang/expr" + "github.com/expr-lang/expr/vm" + "github.com/zhenghaoz/gorse/base/heap" + "github.com/zhenghaoz/gorse/base/log" + "github.com/zhenghaoz/gorse/config" + "github.com/zhenghaoz/gorse/storage/cache" + "github.com/zhenghaoz/gorse/storage/data" + "go.uber.org/zap" +) + +type LeaderBoard struct { + scoreFunc *vm.Program + filterFunc *vm.Program + heap heap.TopKFilter[cache.Document, float64] +} + +func NewLeaderBoard(cfg config.LeaderBoardConfig) (*LeaderBoard, error) { + // Compile score expression + scoreFunc, err := expr.Compile(cfg.Score, expr.Env(map[string]any{ + "item": data.Item{}, + "feedback": []data.Feedback{}, + })) + if err != nil { + return nil, err + } + // Compile filter expression + var filterFunc *vm.Program + if cfg.Filter != "" { + filterFunc, err = expr.Compile(cfg.Filter, expr.Env(map[string]any{ + "item": data.Item{}, + "feedback": []data.Feedback{}, + })) + if err != nil { + return nil, err + } + } + return &LeaderBoard{ + scoreFunc: scoreFunc, + filterFunc: filterFunc, + }, nil +} + +func (l *LeaderBoard) Push(item data.Item, feedback []data.Feedback) { + // Evaluate filter function + if l.filterFunc != nil { + result, err := expr.Run(l.filterFunc, map[string]any{ + "item": item, + "feedback": feedback, + }) + if err != nil { + log.Logger().Error("evaluate filter function", zap.Error(err)) + return + } + if !result.(bool) { + return + } + } + // Evaluate score function + result, err := expr.Run(l.scoreFunc, map[string]any{ + "item": item, + "feedback": feedback, + }) + if err != nil { + log.Logger().Error("evaluate score function", zap.Error(err)) + return + } + score := result.(float64) + l.heap.Push(item, score) +} + +func (l *LeaderBoard) PopAll() []data.Item { + return nil +} diff --git a/recommend/leaderboard_test.go b/recommend/leaderboard_test.go new file mode 100644 index 000000000..5ea0adc1a --- /dev/null +++ b/recommend/leaderboard_test.go @@ -0,0 +1,20 @@ +// Copyright 2024 gorse Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recommend + +import "testing" + +func TestLeaderBoard(t *testing.T) { +} From b1e3bd6d3acdb35b599f200e19a885e6a3412248 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sat, 9 Nov 2024 09:27:43 +0800 Subject: [PATCH 03/19] implement LeaderBoard --- recommend/leaderboard.go | 69 ++++++++++++++++++++++++++--- recommend/leaderboard_test.go | 82 ++++++++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 8 deletions(-) diff --git a/recommend/leaderboard.go b/recommend/leaderboard.go index d8558307a..83161d554 100644 --- a/recommend/leaderboard.go +++ b/recommend/leaderboard.go @@ -17,21 +17,26 @@ package recommend import ( "github.com/expr-lang/expr" "github.com/expr-lang/expr/vm" + "github.com/juju/errors" + "github.com/samber/lo" "github.com/zhenghaoz/gorse/base/heap" "github.com/zhenghaoz/gorse/base/log" "github.com/zhenghaoz/gorse/config" "github.com/zhenghaoz/gorse/storage/cache" "github.com/zhenghaoz/gorse/storage/data" "go.uber.org/zap" + "reflect" + "time" ) type LeaderBoard struct { + timestamp time.Time scoreFunc *vm.Program filterFunc *vm.Program - heap heap.TopKFilter[cache.Document, float64] + heap *heap.TopKFilter[cache.Score, float64] } -func NewLeaderBoard(cfg config.LeaderBoardConfig) (*LeaderBoard, error) { +func NewLeaderBoard(cfg config.LeaderBoardConfig, n int, timestamp time.Time) (*LeaderBoard, error) { // Compile score expression scoreFunc, err := expr.Compile(cfg.Score, expr.Env(map[string]any{ "item": data.Item{}, @@ -40,6 +45,11 @@ func NewLeaderBoard(cfg config.LeaderBoardConfig) (*LeaderBoard, error) { if err != nil { return nil, err } + switch scoreFunc.Node().Type().Kind() { + case reflect.Float64, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + default: + return nil, errors.New("score function must return float64") + } // Compile filter expression var filterFunc *vm.Program if cfg.Filter != "" { @@ -50,14 +60,37 @@ func NewLeaderBoard(cfg config.LeaderBoardConfig) (*LeaderBoard, error) { if err != nil { return nil, err } + if filterFunc.Node().Type().Kind() != reflect.Bool { + return nil, errors.New("filter function must return bool") + } } return &LeaderBoard{ + timestamp: timestamp, scoreFunc: scoreFunc, filterFunc: filterFunc, + heap: heap.NewTopKFilter[cache.Score, float64](n), }, nil } +func NewLatest(n int, timestamp time.Time) *LeaderBoard { + return lo.Must(NewLeaderBoard(config.LeaderBoardConfig{ + Name: "latest", + Score: "item.Timestamp.Unix()", + }, n, timestamp)) +} + +func NewPopular(n int, timestamp time.Time) *LeaderBoard { + return lo.Must(NewLeaderBoard(config.LeaderBoardConfig{ + Name: "popular", + Score: "len(feedback)", + }, n, timestamp)) +} + func (l *LeaderBoard) Push(item data.Item, feedback []data.Feedback) { + // Skip hidden items + if item.IsHidden { + return + } // Evaluate filter function if l.filterFunc != nil { result, err := expr.Run(l.filterFunc, map[string]any{ @@ -81,10 +114,34 @@ func (l *LeaderBoard) Push(item data.Item, feedback []data.Feedback) { log.Logger().Error("evaluate score function", zap.Error(err)) return } - score := result.(float64) - l.heap.Push(item, score) + var score float64 + switch typed := result.(type) { + case float64: + score = typed + case int: + score = float64(typed) + case int8: + score = float64(typed) + case int16: + score = float64(typed) + case int32: + score = float64(typed) + case int64: + score = float64(typed) + default: + log.Logger().Error("score function must return float64", zap.Any("result", result)) + return + } + l.heap.Push(cache.Score{ + Id: item.ItemId, + Score: score, + IsHidden: item.IsHidden, + Categories: item.Categories, + Timestamp: l.timestamp, + }, score) } -func (l *LeaderBoard) PopAll() []data.Item { - return nil +func (l *LeaderBoard) PopAll() []cache.Score { + scores, _ := l.heap.PopAll() + return scores } diff --git a/recommend/leaderboard_test.go b/recommend/leaderboard_test.go index 5ea0adc1a..80ed784b0 100644 --- a/recommend/leaderboard_test.go +++ b/recommend/leaderboard_test.go @@ -14,7 +14,85 @@ package recommend -import "testing" +import ( + "github.com/stretchr/testify/assert" + "github.com/zhenghaoz/gorse/config" + "github.com/zhenghaoz/gorse/storage/data" + "strconv" + "testing" + "time" +) -func TestLeaderBoard(t *testing.T) { +func TestLatest(t *testing.T) { + timestamp := time.Now() + latest := NewLatest(10, timestamp) + for i := 0; i < 100; i++ { + item := data.Item{ItemId: strconv.Itoa(i), Timestamp: timestamp.Add(time.Duration(-i) * time.Second)} + latest.Push(item, nil) + } + scores := latest.PopAll() + assert.Len(t, scores, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, strconv.Itoa(i), scores[i].Id) + assert.Equal(t, float64(timestamp.Add(time.Duration(-i)*time.Second).Unix()), scores[i].Score) + assert.Equal(t, timestamp, scores[i].Timestamp) + } +} + +func TestPopular(t *testing.T) { + timestamp := time.Now() + popular := NewPopular(10, timestamp) + for i := 0; i < 100; i++ { + item := data.Item{ItemId: strconv.Itoa(i)} + var feedback []data.Feedback + for j := 0; j < i; j++ { + feedback = append(feedback, data.Feedback{FeedbackKey: data.FeedbackKey{UserId: strconv.Itoa(j)}, Timestamp: timestamp}) + } + popular.Push(item, feedback) + } + scores := popular.PopAll() + assert.Len(t, scores, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, strconv.Itoa(99-i), scores[i].Id) + assert.Equal(t, float64(99-i), scores[i].Score) + } +} + +func TestFilter(t *testing.T) { + timestamp := time.Now() + latest, err := NewLeaderBoard(config.LeaderBoardConfig{ + Name: "latest", + Score: "item.Timestamp.Unix()", + Filter: "!item.IsHidden", + }, 10, timestamp) + assert.NoError(t, err) + for i := 0; i < 100; i++ { + item := data.Item{ItemId: strconv.Itoa(i), Timestamp: timestamp.Add(time.Duration(-i) * time.Second)} + item.IsHidden = i < 10 + latest.Push(item, nil) + } + scores := latest.PopAll() + assert.Len(t, scores, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, strconv.Itoa(i+10), scores[i].Id) + assert.Equal(t, float64(timestamp.Add(time.Duration(-i-10)*time.Second).Unix()), scores[i].Score) + assert.Equal(t, timestamp, scores[i].Timestamp) + } +} + +func TestHidden(t *testing.T) { + timestamp := time.Now() + latest := NewLatest(10, timestamp) + for i := 0; i < 100; i++ { + item := data.Item{ItemId: strconv.Itoa(i), Timestamp: timestamp.Add(time.Duration(-i) * time.Second)} + item.IsHidden = i < 10 + latest.Push(item, nil) + } + scores := latest.PopAll() + assert.Len(t, scores, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, strconv.Itoa(i+10), scores[i].Id) + assert.Equal(t, float64(timestamp.Add(time.Duration(-i-10)*time.Second).Unix()), scores[i].Score) + assert.Equal(t, timestamp, scores[i].Timestamp) + } } From c7e1c587b3db8eb728d8bcf162c9e885760b9a84 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sat, 9 Nov 2024 10:18:55 +0800 Subject: [PATCH 04/19] implement API --- cmd/gorse-cli/main.go | 31 ++++++++++++++++++++++++ server/bench_test.go | 4 ++-- server/rest.go | 40 ++++++++++++++++++++++--------- server/rest_test.go | 39 +++++++++++++++++------------- storage/cache/database.go | 16 ++++--------- storage/cache/database_test.go | 10 ++++---- storage/cache/mongodb.go | 14 +++++++---- storage/cache/no_database.go | 2 +- storage/cache/no_database_test.go | 2 +- storage/cache/redis.go | 5 +++- storage/cache/redis_test.go | 2 +- storage/cache/sql.go | 6 +++-- 12 files changed, 115 insertions(+), 56 deletions(-) create mode 100644 cmd/gorse-cli/main.go diff --git a/cmd/gorse-cli/main.go b/cmd/gorse-cli/main.go new file mode 100644 index 000000000..a16f9e36c --- /dev/null +++ b/cmd/gorse-cli/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "github.com/spf13/cobra" + "github.com/zhenghaoz/gorse/base/log" + "go.uber.org/zap" +) + +var rootCmd = &cobra.Command{ + Use: "gorse-cli", + Short: "Gorse command line tool", +} + +var dumpCmd = &cobra.Command{ + Use: "dump", +} + +var restoreCmd = &cobra.Command{ + Use: "restore", +} + +func init() { + rootCmd.AddCommand(dumpCmd) + rootCmd.AddCommand(restoreCmd) +} + +func main() { + if err := rootCmd.Execute(); err != nil { + log.Logger().Fatal("failed to execute command", zap.Error(err)) + } +} diff --git a/server/bench_test.go b/server/bench_test.go index 53afe0493..66e497ad1 100644 --- a/server/bench_test.go +++ b/server/bench_test.go @@ -804,7 +804,7 @@ func BenchmarkGetRecommendCache(b *testing.B) { documents[i].Categories = []string{""} } lo.Reverse(documents) - err := s.CacheClient.AddScores(ctx, cache.PopularItems, "", documents) + err := s.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, documents) require.NoError(b, err) s.Config.Recommend.CacheSize = len(documents) @@ -903,7 +903,7 @@ func BenchmarkRecommendFromLatest(b *testing.B) { } lo.Reverse(documents) lo.Reverse(expects) - err := s.CacheClient.AddScores(ctx, cache.LatestItems, "", documents) + err := s.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, documents) require.NoError(b, err) err = s.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) require.NoError(b, err) diff --git a/server/rest.go b/server/rest.go index c0bab8ebb..6bff7b011 100644 --- a/server/rest.go +++ b/server/rest.go @@ -471,6 +471,17 @@ func (s *RestServer) CreateWebService() { Param(ws.QueryParameter("user-id", "Remove read items of a user").DataType("string")). Returns(http.StatusOK, "OK", []cache.Score{}). Writes([]cache.Score{})) + // Get leaderboard + ws.Route(ws.GET("/leaderboard/{name}").To(s.getLeaderboard). + Doc("Get the leaderboard."). + Metadata(restfulspec.KeyOpenAPITags, []string{RecommendationAPITag}). + Param(ws.HeaderParameter("X-API-Key", "API key").DataType("string")). + Param(ws.QueryParameter("category", "Category of returned items.").DataType("string")). + Param(ws.QueryParameter("n", "Number of returned users").DataType("integer")). + Param(ws.QueryParameter("offset", "Offset of returned users").DataType("integer")). + Param(ws.QueryParameter("user-id", "Remove read items of a user").DataType("string")). + Returns(http.StatusOK, "OK", []cache.Score{}). + Writes([]cache.Score{})) // Get neighbors ws.Route(ws.GET("/item/{item-id}/neighbors/").To(s.getItemNeighbors). Doc("Get neighbors of a item"). @@ -639,13 +650,20 @@ func (s *RestServer) searchDocuments(collection, subset, category string, isItem func (s *RestServer) getPopular(request *restful.Request, response *restful.Response) { category := request.PathParameter("category") log.ResponseLogger(response).Debug("get category popular items in category", zap.String("category", category)) - s.searchDocuments(cache.PopularItems, "", category, true, request, response) + s.searchDocuments(cache.Leaderboard, cache.Popular, category, true, request, response) } func (s *RestServer) getLatest(request *restful.Request, response *restful.Response) { category := request.PathParameter("category") log.ResponseLogger(response).Debug("get category latest items in category", zap.String("category", category)) - s.searchDocuments(cache.LatestItems, "", category, true, request, response) + s.searchDocuments(cache.Leaderboard, cache.Latest, category, true, request, response) +} + +func (s *RestServer) getLeaderboard(request *restful.Request, response *restful.Response) { + name := request.PathParameter("name") + category := request.QueryParameter("category") + log.ResponseLogger(response).Debug("get leaderboard", zap.String("name", name)) + s.searchDocuments(cache.Leaderboard, name, category, false, request, response) } // get feedback by item-id with feedback type @@ -928,7 +946,7 @@ func (s *RestServer) RecommendItemBased(ctx *recommendContext) error { func (s *RestServer) RecommendLatest(ctx *recommendContext) error { if len(ctx.results) < ctx.n { start := time.Now() - items, err := s.CacheClient.SearchScores(ctx.context, cache.LatestItems, "", ctx.categories, 0, s.Config.Recommend.CacheSize) + items, err := s.CacheClient.SearchScores(ctx.context, cache.Leaderboard, cache.Latest, ctx.categories, 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -948,7 +966,7 @@ func (s *RestServer) RecommendLatest(ctx *recommendContext) error { func (s *RestServer) RecommendPopular(ctx *recommendContext) error { if len(ctx.results) < ctx.n { start := time.Now() - items, err := s.CacheClient.SearchScores(ctx.context, cache.PopularItems, "", ctx.categories, 0, s.Config.Recommend.CacheSize) + items, err := s.CacheClient.SearchScores(ctx.context, cache.Leaderboard, cache.Popular, ctx.categories, 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -1378,7 +1396,7 @@ func (s *RestServer) batchInsertItems(ctx context.Context, response *restful.Res Comment: item.Comment, }) // insert to latest items cache - if err = s.CacheClient.AddScores(ctx, cache.LatestItems, "", []cache.Score{{ + if err = s.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, []cache.Score{{ Id: item.ItemId, Score: float64(timestamp.Unix()), Categories: withWildCard(item.Categories), @@ -1388,7 +1406,7 @@ func (s *RestServer) batchInsertItems(ctx context.Context, response *restful.Res return } // update items cache - if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, item.ItemId, cache.ScorePatch{ + if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", item.ItemId, cache.ScorePatch{ Categories: withWildCard(item.Categories), IsHidden: &item.IsHidden, }); err != nil { @@ -1492,21 +1510,21 @@ func (s *RestServer) modifyItem(request *restful.Request, response *restful.Resp } // remove hidden item from cache if patch.IsHidden != nil { - if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, itemId, cache.ScorePatch{IsHidden: patch.IsHidden}); err != nil { + if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{IsHidden: patch.IsHidden}); err != nil { InternalServerError(response, err) return } } // add item to latest items cache if patch.Timestamp != nil { - if err := s.CacheClient.UpdateScores(ctx, []string{cache.LatestItems}, itemId, cache.ScorePatch{Score: proto.Float64(float64(patch.Timestamp.Unix()))}); err != nil { + if err := s.CacheClient.UpdateScores(ctx, []string{cache.Leaderboard}, cache.Latest, itemId, cache.ScorePatch{Score: proto.Float64(float64(patch.Timestamp.Unix()))}); err != nil { InternalServerError(response, err) return } } // update categories in cache if patch.Categories != nil { - if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, itemId, cache.ScorePatch{Categories: withWildCard(patch.Categories)}); err != nil { + if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{Categories: withWildCard(patch.Categories)}); err != nil { InternalServerError(response, err) return } @@ -1610,7 +1628,7 @@ func (s *RestServer) insertItemCategory(request *restful.Request, response *rest return } // insert category to cache - if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, itemId, cache.ScorePatch{Categories: withWildCard(item.Categories)}); err != nil { + if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{Categories: withWildCard(item.Categories)}); err != nil { InternalServerError(response, err) return } @@ -1639,7 +1657,7 @@ func (s *RestServer) deleteItemCategory(request *restful.Request, response *rest } item.Categories = categories // delete category from cache - if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, itemId, cache.ScorePatch{Categories: withWildCard(categories)}); err != nil { + if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{Categories: withWildCard(categories)}); err != nil { InternalServerError(response, err) return } diff --git a/server/rest_test.go b/server/rest_test.go index 95c544fb9..3a5158135 100644 --- a/server/rest_test.go +++ b/server/rest_test.go @@ -243,7 +243,7 @@ func (suite *ServerTestSuite) TestItems() { }, } // insert popular scores - err := suite.CacheClient.AddScores(ctx, cache.PopularItems, "", []cache.Score{ + err := suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, []cache.Score{ {Id: "0", Score: 10}, {Id: "2", Score: 12}, {Id: "4", Score: 14}, @@ -822,10 +822,12 @@ func (suite *ServerTestSuite) TestNonPersonalizedRecommend() { //{"User Neighbors", cache.Collection(cache.UserNeighbors, "0"), "/api/user/0/neighbors"}, {"Item Neighbors", cache.ItemNeighbors, "0", "", "/api/item/0/neighbors"}, {"Item Neighbors in Category", cache.ItemNeighbors, "0", "0", "/api/item/0/neighbors/0"}, - {"Latest Items", cache.LatestItems, "", "", "/api/latest/"}, - {"Latest Items in Category", cache.LatestItems, "", "0", "/api/latest/0"}, - {"Popular Items", cache.PopularItems, "", "", "/api/popular/"}, - {"Popular Items in Category", cache.PopularItems, "", "0", "/api/popular/0"}, + {"LatestItems", cache.Leaderboard, cache.Latest, "", "/api/latest/"}, + {"LatestItemsCategory", cache.Leaderboard, cache.Latest, "0", "/api/latest/0"}, + {"PopularItems", cache.Leaderboard, cache.Popular, "", "/api/popular/"}, + {"PopularItemsCategory", cache.Leaderboard, cache.Popular, "0", "/api/popular/0"}, + {"Leaderboard", cache.Leaderboard, "trending", "", "/api/leaderboard/trending"}, + {"LeaderboardCategory", cache.Leaderboard, "trending", "0", "/api/leaderboard/trending"}, {"Offline Recommend", cache.OfflineRecommend, "0", "", "/api/intermediate/recommend/0"}, {"Offline Recommend in Category", cache.OfflineRecommend, "0", "0", "/api/intermediate/recommend/0/0"}, } @@ -865,47 +867,52 @@ func (suite *ServerTestSuite) TestNonPersonalizedRecommend() { apitest.New(). Handler(suite.handler). Get(operator.URL). + Query("category", operator.Category). Header("X-API-Key", apiKey). Expect(t). Status(http.StatusOK). - Body(suite.marshal(([]cache.Score{documents[0], documents[1], documents[2], documents[4]}))). + Body(suite.marshal([]cache.Score{documents[0], documents[1], documents[2], documents[4]})). End() apitest.New(). Handler(suite.handler). Get(operator.URL). + Query("category", operator.Category). Header("X-API-Key", apiKey). QueryParams(map[string]string{ "offset": "0", "n": "3"}). Expect(t). Status(http.StatusOK). - Body(suite.marshal(([]cache.Score{documents[0], documents[1], documents[2]}))). + Body(suite.marshal([]cache.Score{documents[0], documents[1], documents[2]})). End() apitest.New(). Handler(suite.handler). Get(operator.URL). + Query("category", operator.Category). Header("X-API-Key", apiKey). QueryParams(map[string]string{ "offset": "1", "n": "3"}). Expect(t). Status(http.StatusOK). - Body(suite.marshal(([]cache.Score{documents[1], documents[2], documents[4]}))). + Body(suite.marshal([]cache.Score{documents[1], documents[2], documents[4]})). End() apitest.New(). Handler(suite.handler). Get(operator.URL). + Query("category", operator.Category). Header("X-API-Key", apiKey). QueryParams(map[string]string{ "offset": "0", "n": "0"}). Expect(t). Status(http.StatusOK). - Body(suite.marshal(([]cache.Score{documents[0], documents[1], documents[2], documents[4]}))). + Body(suite.marshal([]cache.Score{documents[0], documents[1], documents[2], documents[4]})). End() apitest.New(). Handler(suite.handler). Get(operator.URL). + Query("category", operator.Category). Header("X-API-Key", apiKey). QueryParams(map[string]string{ "user-id": "0", @@ -913,7 +920,7 @@ func (suite *ServerTestSuite) TestNonPersonalizedRecommend() { "n": "0"}). Expect(t). Status(http.StatusOK). - Body(suite.marshal(([]cache.Score{documents[0], documents[2], documents[4]}))). + Body(suite.marshal([]cache.Score{documents[0], documents[2], documents[4]})). End() }) } @@ -1410,26 +1417,26 @@ func (suite *ServerTestSuite) TestGetRecommendsFallbackPreCached() { {Id: "104", Score: 96, Categories: []string{"*"}}}) assert.NoError(t, err) // insert latest - err = suite.CacheClient.AddScores(ctx, cache.LatestItems, "", []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, []cache.Score{ {Id: "5", Score: 95, Categories: []string{""}}, {Id: "6", Score: 94, Categories: []string{""}}, {Id: "7", Score: 93, Categories: []string{""}}, {Id: "8", Score: 92, Categories: []string{""}}}) assert.NoError(t, err) - err = suite.CacheClient.AddScores(ctx, cache.LatestItems, "", []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, []cache.Score{ {Id: "105", Score: 95, Categories: []string{"*"}}, {Id: "106", Score: 94, Categories: []string{"*"}}, {Id: "107", Score: 93, Categories: []string{"*"}}, {Id: "108", Score: 92, Categories: []string{"*"}}}) assert.NoError(t, err) // insert popular - err = suite.CacheClient.AddScores(ctx, cache.PopularItems, "", []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, []cache.Score{ {Id: "9", Score: 91, Categories: []string{""}}, {Id: "10", Score: 90, Categories: []string{""}}, {Id: "11", Score: 89, Categories: []string{""}}, {Id: "12", Score: 88, Categories: []string{""}}}) assert.NoError(t, err) - err = suite.CacheClient.AddScores(ctx, cache.PopularItems, "", []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, []cache.Score{ {Id: "109", Score: 91, Categories: []string{"*"}}, {Id: "110", Score: 90, Categories: []string{"*"}}, {Id: "111", Score: 89, Categories: []string{"*"}}, @@ -1669,9 +1676,9 @@ func (suite *ServerTestSuite) TestVisibility() { }) } lo.Reverse(documents) - err := suite.CacheClient.AddScores(ctx, cache.LatestItems, "", documents) + err := suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, documents) assert.NoError(t, err) - err = suite.CacheClient.AddScores(ctx, cache.PopularItems, "", documents) + err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, documents) assert.NoError(t, err) err = suite.CacheClient.AddScores(ctx, cache.ItemNeighbors, "100", documents) assert.NoError(t, err) diff --git a/storage/cache/database.go b/storage/cache/database.go index 60b58d636..e332cda7d 100644 --- a/storage/cache/database.go +++ b/storage/cache/database.go @@ -76,15 +76,9 @@ const ( // Recommendation digest - offline_recommend_digest/{user_id} OfflineRecommendDigest = "offline_recommend_digest" - // PopularItems is sorted set of popular items. The format of key: - // Global popular items - latest_items - // Categorized popular items - latest_items/{category} - PopularItems = "popular_items" - - // LatestItems is sorted set of the latest items. The format of key: - // Global latest items - latest_items - // Categorized the latest items - latest_items/{category} - LatestItems = "latest_items" + Leaderboard = "leaderboard" + Latest = "latest" + Popular = "popular" // ItemCategories is the set of item categories. The format of key: // Global item categories - item_categories @@ -115,7 +109,7 @@ const ( MatchingIndexRecall = "matching_index_recall" ) -var ItemCache = []string{PopularItems, LatestItems, ItemNeighbors, OfflineRecommend} +var ItemCache = []string{Leaderboard, ItemNeighbors, OfflineRecommend} var ( ErrObjectNotExist = errors.NotFoundf("object") @@ -293,7 +287,7 @@ type Database interface { AddScores(ctx context.Context, collection, subset string, documents []Score) error SearchScores(ctx context.Context, collection, subset string, query []string, begin, end int) ([]Score, error) DeleteScores(ctx context.Context, collection []string, condition ScoreCondition) error - UpdateScores(ctx context.Context, collection []string, id string, patch ScorePatch) error + UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error AddTimeSeriesPoints(ctx context.Context, points []TimeSeriesPoint) error GetTimeSeriesPoints(ctx context.Context, name string, begin, end time.Time) ([]TimeSeriesPoint, error) diff --git a/storage/cache/database_test.go b/storage/cache/database_test.go index 5046d603d..bbfb6f30b 100644 --- a/storage/cache/database_test.go +++ b/storage/cache/database_test.go @@ -324,20 +324,20 @@ func (suite *baseTestSuite) TestDocument() { suite.Equal("2", documents[0].Id) // update categories - err = suite.UpdateScores(ctx, []string{"a"}, "2", ScorePatch{Categories: []string{"c", "s"}}) + err = suite.UpdateScores(ctx, []string{"a"}, "", "2", ScorePatch{Categories: []string{"c", "s"}}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "", []string{"s"}, 0, 1) suite.NoError(err) suite.Len(documents, 1) suite.Equal("2", documents[0].Id) - err = suite.UpdateScores(ctx, []string{"a"}, "2", ScorePatch{Categories: []string{"c"}}) + err = suite.UpdateScores(ctx, []string{"a"}, "", "2", ScorePatch{Categories: []string{"c"}}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "", []string{"s"}, 0, 1) suite.NoError(err) suite.Empty(documents) // update is hidden - err = suite.UpdateScores(ctx, []string{"a"}, "0", ScorePatch{IsHidden: proto.Bool(false)}) + err = suite.UpdateScores(ctx, []string{"a"}, "", "0", ScorePatch{IsHidden: proto.Bool(false)}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "", []string{"b"}, 0, 1) suite.NoError(err) @@ -401,7 +401,7 @@ func (suite *baseTestSuite) TestSubsetDocument() { }, documents) // update categories - err = suite.UpdateScores(ctx, []string{"a", "b"}, "2", ScorePatch{Categories: []string{"b", "s"}}) + err = suite.UpdateScores(ctx, []string{"a", "b"}, "", "2", ScorePatch{Categories: []string{"b", "s"}}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "a", []string{"s"}, 0, 1) suite.NoError(err) @@ -551,7 +551,7 @@ func benchmarkUpdateDocuments(b *testing.B, database Database) { // select a random number n := rand.Intn(benchmarkDataSize) + 1 // update documents - err := database.UpdateScores(ctx, []string{"a"}, strconv.Itoa(n), ScorePatch{ + err := database.UpdateScores(ctx, []string{"a"}, "", strconv.Itoa(n), ScorePatch{ Score: proto.Float64(float64(n)), }) assert.NoError(b, err) diff --git a/storage/cache/mongodb.go b/storage/cache/mongodb.go index 54dfb831f..865913a37 100644 --- a/storage/cache/mongodb.go +++ b/storage/cache/mongodb.go @@ -369,13 +369,20 @@ func (m MongoDB) SearchScores(ctx context.Context, collection, subset string, qu return documents, nil } -func (m MongoDB) UpdateScores(ctx context.Context, collections []string, id string, patch ScorePatch) error { +func (m MongoDB) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { if len(collections) == 0 { return nil } if patch.IsHidden == nil && patch.Categories == nil && patch.Score == nil { return nil } + filter := bson.M{ + "collection": bson.M{"$in": collections}, + "id": id, + } + if subset != "" { + filter["subset"] = subset + } update := bson.D{} if patch.IsHidden != nil { update = append(update, bson.E{Key: "$set", Value: bson.M{"is_hidden": *patch.IsHidden}}) @@ -386,10 +393,7 @@ func (m MongoDB) UpdateScores(ctx context.Context, collections []string, id stri if patch.Score != nil { update = append(update, bson.E{Key: "$set", Value: bson.M{"score": *patch.Score}}) } - _, err := m.client.Database(m.dbName).Collection(m.DocumentTable()).UpdateMany(ctx, bson.M{ - "collection": bson.M{"$in": collections}, - "id": id, - }, update) + _, err := m.client.Database(m.dbName).Collection(m.DocumentTable()).UpdateMany(ctx, subset, update) return errors.Trace(err) } diff --git a/storage/cache/no_database.go b/storage/cache/no_database.go index 3ca70e932..bd572fca7 100644 --- a/storage/cache/no_database.go +++ b/storage/cache/no_database.go @@ -98,7 +98,7 @@ func (NoDatabase) SearchScores(_ context.Context, _, _ string, _ []string, _, _ return nil, ErrNoDatabase } -func (NoDatabase) UpdateScores(_ context.Context, _ []string, _ string, _ ScorePatch) error { +func (NoDatabase) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { return ErrNoDatabase } diff --git a/storage/cache/no_database_test.go b/storage/cache/no_database_test.go index b49127754..3e78b1dc9 100644 --- a/storage/cache/no_database_test.go +++ b/storage/cache/no_database_test.go @@ -66,7 +66,7 @@ func TestNoDatabase(t *testing.T) { assert.ErrorIs(t, err, ErrNoDatabase) _, err = database.SearchScores(ctx, "", "", nil, 0, 0) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.UpdateScores(ctx, nil, "", ScorePatch{}) + err = database.UpdateScores(ctx, nil, "", "", ScorePatch{}) assert.ErrorIs(t, err, ErrNoDatabase) err = database.DeleteScores(ctx, nil, ScoreCondition{}) assert.ErrorIs(t, err, ErrNoDatabase) diff --git a/storage/cache/redis.go b/storage/cache/redis.go index 18c74c56f..0a5a5dfff 100644 --- a/storage/cache/redis.go +++ b/storage/cache/redis.go @@ -307,7 +307,7 @@ func (r *Redis) SearchScores(ctx context.Context, collection, subset string, que return documents, nil } -func (r *Redis) UpdateScores(ctx context.Context, collections []string, id string, patch ScorePatch) error { +func (r *Redis) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { if len(collections) == 0 { return nil } @@ -317,6 +317,9 @@ func (r *Redis) UpdateScores(ctx context.Context, collections []string, id strin var builder strings.Builder builder.WriteString(fmt.Sprintf("@collection:{ %s }", escape(strings.Join(collections, " | ")))) builder.WriteString(fmt.Sprintf(" @id:{ %s }", escape(id))) + if subset != "" { + builder.WriteString(fmt.Sprintf(" @subset:{ %s }", escape(subset))) + } for { // search documents result, err := r.client.FTSearchWithArgs(ctx, r.DocumentTable(), builder.String(), &redis.FTSearchOptions{ diff --git a/storage/cache/redis_test.go b/storage/cache/redis_test.go index a63ab6949..cfba0b281 100644 --- a/storage/cache/redis_test.go +++ b/storage/cache/redis_test.go @@ -78,7 +78,7 @@ func (suite *RedisTestSuite) TestEscapeCharacters() { suite.NoError(err) suite.Equal([]Score{{Id: id, Score: math.MaxFloat64, Categories: []string{"a", "b"}, Timestamp: ts}}, documents) - err = suite.UpdateScores(ctx, []string{collection}, id, ScorePatch{Score: proto.Float64(1)}) + err = suite.UpdateScores(ctx, []string{collection}, "", id, ScorePatch{Score: proto.Float64(1)}) suite.NoError(err) documents, err = suite.SearchScores(ctx, collection, subset, []string{"b"}, 0, -1) suite.NoError(err) diff --git a/storage/cache/sql.go b/storage/cache/sql.go index bef865d75..a3703da6d 100644 --- a/storage/cache/sql.go +++ b/storage/cache/sql.go @@ -477,14 +477,16 @@ func (db *SQLDatabase) SearchScores(ctx context.Context, collection, subset stri return documents, nil } -func (db *SQLDatabase) UpdateScores(ctx context.Context, collections []string, id string, patch ScorePatch) error { +func (db *SQLDatabase) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { if len(collections) == 0 { return nil } if patch.Score == nil && patch.IsHidden == nil && patch.Categories == nil { return nil } - tx := db.gormDB.WithContext(ctx).Model(&PostgresDocument{}).Where("collection in (?) and id = ?", collections, id) + tx := db.gormDB.WithContext(ctx).Model(&PostgresDocument{}). + Where("collection in (?) and id = ? and (subset = ? or ? = '')", + collections, id, subset, subset) if patch.Score != nil { tx = tx.Update("score", *patch.Score) } From 9fd59ad803d4b5623ada0a67a373352c5ce4ce39 Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Tue, 26 Nov 2024 21:55:22 +0800 Subject: [PATCH 05/19] Rename logics --- config/config.go | 26 +++++++++++------------ config/config.toml | 2 +- config/config_test.go | 8 +++---- {recommend => logics}/leaderboard.go | 8 +++---- {recommend => logics}/leaderboard_test.go | 4 ++-- master/rest.go | 4 ++-- master/tasks.go | 10 +++++---- server/bench_test.go | 4 ++-- server/rest.go | 14 ++++++------ server/rest_test.go | 26 +++++++++++------------ storage/cache/database.go | 8 +++---- 11 files changed, 58 insertions(+), 56 deletions(-) rename {recommend => logics}/leaderboard.go (93%) rename {recommend => logics}/leaderboard_test.go (97%) diff --git a/config/config.go b/config/config.go index 30dfbf57c..0921def72 100644 --- a/config/config.go +++ b/config/config.go @@ -101,18 +101,18 @@ type ServerConfig struct { // RecommendConfig is the configuration of recommendation setup. type RecommendConfig struct { - CacheSize int `mapstructure:"cache_size" validate:"gt=0"` - CacheExpire time.Duration `mapstructure:"cache_expire" validate:"gt=0"` - ActiveUserTTL int `mapstructure:"active_user_ttl" validate:"gte=0"` - DataSource DataSourceConfig `mapstructure:"data_source"` - LeaderBoards []LeaderBoardConfig `mapstructure:"leaderboards" validate:"dive"` - Popular PopularConfig `mapstructure:"popular"` - UserNeighbors NeighborsConfig `mapstructure:"user_neighbors"` - ItemNeighbors NeighborsConfig `mapstructure:"item_neighbors"` - Collaborative CollaborativeConfig `mapstructure:"collaborative"` - Replacement ReplacementConfig `mapstructure:"replacement"` - Offline OfflineConfig `mapstructure:"offline"` - Online OnlineConfig `mapstructure:"online"` + CacheSize int `mapstructure:"cache_size" validate:"gt=0"` + CacheExpire time.Duration `mapstructure:"cache_expire" validate:"gt=0"` + ActiveUserTTL int `mapstructure:"active_user_ttl" validate:"gte=0"` + DataSource DataSourceConfig `mapstructure:"data_source"` + NonPersonalized []NonPersonalizedConfig `mapstructure:"non-personalized" validate:"dive"` + Popular PopularConfig `mapstructure:"popular"` + UserNeighbors NeighborsConfig `mapstructure:"user_neighbors"` + ItemNeighbors NeighborsConfig `mapstructure:"item_neighbors"` + Collaborative CollaborativeConfig `mapstructure:"collaborative"` + Replacement ReplacementConfig `mapstructure:"replacement"` + Offline OfflineConfig `mapstructure:"offline"` + Online OnlineConfig `mapstructure:"online"` } type DataSourceConfig struct { @@ -122,7 +122,7 @@ type DataSourceConfig struct { ItemTTL uint `mapstructure:"item_ttl" validate:"gte=0"` // item-to-live of items } -type LeaderBoardConfig struct { +type NonPersonalizedConfig struct { Name string `mapstructure:"name"` Score string `mapstructure:"score" validate:"required,item_expr"` Filter string `mapstructure:"filter" validate:"item_expr"` diff --git a/config/config.toml b/config/config.toml index 732d8872d..04f124685 100644 --- a/config/config.toml +++ b/config/config.toml @@ -116,7 +116,7 @@ item_ttl = 0 # The time window of popular items. The default values is 4320h. popular_window = "720h" -[[recommend.leaderboards]] +[[recommend.non-personalized]] # The name of the leaderboard. name = "most_starred_weekly" diff --git a/config/config_test.go b/config/config_test.go index 2c34cb8b6..f5cda366a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -95,10 +95,10 @@ func TestUnmarshal(t *testing.T) { // [recommend.popular] assert.Equal(t, 30*24*time.Hour, config.Recommend.Popular.PopularWindow) // [recommend.leaderboards] - assert.Len(t, config.Recommend.LeaderBoards, 1) - assert.Equal(t, "most_starred_weekly", config.Recommend.LeaderBoards[0].Name) - assert.Equal(t, "count(Feedback.FeedbackType, .FeedbackType == 'star')", config.Recommend.LeaderBoards[0].Score) - assert.Equal(t, "Item.Timestamp > now() < 7d", config.Recommend.LeaderBoards[0].Filter) + assert.Len(t, config.Recommend.NonPersonalized, 1) + assert.Equal(t, "most_starred_weekly", config.Recommend.NonPersonalized[0].Name) + assert.Equal(t, "count(Feedback.FeedbackType, .FeedbackType == 'star')", config.Recommend.NonPersonalized[0].Score) + assert.Equal(t, "(Item.Timestamp > now()).Hours() < 168", config.Recommend.NonPersonalized[0].Filter) // [recommend.user_neighbors] assert.Equal(t, "similar", config.Recommend.UserNeighbors.NeighborType) assert.True(t, config.Recommend.UserNeighbors.EnableIndex) diff --git a/recommend/leaderboard.go b/logics/leaderboard.go similarity index 93% rename from recommend/leaderboard.go rename to logics/leaderboard.go index 83161d554..cdfacbda3 100644 --- a/recommend/leaderboard.go +++ b/logics/leaderboard.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package recommend +package logics import ( "github.com/expr-lang/expr" @@ -36,7 +36,7 @@ type LeaderBoard struct { heap *heap.TopKFilter[cache.Score, float64] } -func NewLeaderBoard(cfg config.LeaderBoardConfig, n int, timestamp time.Time) (*LeaderBoard, error) { +func NewLeaderBoard(cfg config.NonPersonalizedConfig, n int, timestamp time.Time) (*LeaderBoard, error) { // Compile score expression scoreFunc, err := expr.Compile(cfg.Score, expr.Env(map[string]any{ "item": data.Item{}, @@ -73,14 +73,14 @@ func NewLeaderBoard(cfg config.LeaderBoardConfig, n int, timestamp time.Time) (* } func NewLatest(n int, timestamp time.Time) *LeaderBoard { - return lo.Must(NewLeaderBoard(config.LeaderBoardConfig{ + return lo.Must(NewLeaderBoard(config.NonPersonalizedConfig{ Name: "latest", Score: "item.Timestamp.Unix()", }, n, timestamp)) } func NewPopular(n int, timestamp time.Time) *LeaderBoard { - return lo.Must(NewLeaderBoard(config.LeaderBoardConfig{ + return lo.Must(NewLeaderBoard(config.NonPersonalizedConfig{ Name: "popular", Score: "len(feedback)", }, n, timestamp)) diff --git a/recommend/leaderboard_test.go b/logics/leaderboard_test.go similarity index 97% rename from recommend/leaderboard_test.go rename to logics/leaderboard_test.go index 80ed784b0..c32fd7e4a 100644 --- a/recommend/leaderboard_test.go +++ b/logics/leaderboard_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package recommend +package logics import ( "github.com/stretchr/testify/assert" @@ -60,7 +60,7 @@ func TestPopular(t *testing.T) { func TestFilter(t *testing.T) { timestamp := time.Now() - latest, err := NewLeaderBoard(config.LeaderBoardConfig{ + latest, err := NewLeaderBoard(config.NonPersonalizedConfig{ Name: "latest", Score: "item.Timestamp.Unix()", Filter: "!item.IsHidden", diff --git a/master/rest.go b/master/rest.go index 892a2d9d7..dc00389c4 100644 --- a/master/rest.go +++ b/master/rest.go @@ -924,12 +924,12 @@ func (m *Master) searchDocuments(collection, subset, category string, request *r // getPopular gets popular items from database. func (m *Master) getPopular(request *restful.Request, response *restful.Response) { category := request.PathParameter("category") - m.searchDocuments(cache.PopularItems, "", category, request, response, data.Item{}) + m.searchDocuments(cache.NonPersonalized, cache.Popular, category, request, response, data.Item{}) } func (m *Master) getLatest(request *restful.Request, response *restful.Response) { category := request.PathParameter("category") - m.searchDocuments(cache.LatestItems, "", category, request, response, data.Item{}) + m.searchDocuments(cache.NonPersonalized, cache.Latest, category, request, response, data.Item{}) } func (m *Master) getItemNeighbors(request *restful.Request, response *restful.Response) { diff --git a/master/tasks.go b/master/tasks.go index 7116d8196..c14824853 100644 --- a/master/tasks.go +++ b/master/tasks.go @@ -91,10 +91,11 @@ func (m *Master) runLoadDatasetTask() error { } // save popular items to cache - if err = m.CacheClient.AddScores(ctx, cache.PopularItems, "", popularItems.ToSlice()); err != nil { + if err = m.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, popularItems.ToSlice()); err != nil { log.Logger().Error("failed to cache popular items", zap.Error(err)) } - if err = m.CacheClient.DeleteScores(ctx, []string{cache.PopularItems}, cache.ScoreCondition{Before: &popularItems.Timestamp}); err != nil { + if err = m.CacheClient.DeleteScores(ctx, []string{cache.NonPersonalized}, + cache.ScoreCondition{Subset: proto.String(cache.Popular), Before: &popularItems.Timestamp}); err != nil { log.Logger().Error("failed to reclaim outdated items", zap.Error(err)) } if err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdatePopularItemsTime), time.Now())); err != nil { @@ -102,10 +103,11 @@ func (m *Master) runLoadDatasetTask() error { } // save the latest items to cache - if err = m.CacheClient.AddScores(ctx, cache.LatestItems, "", latestItems.ToSlice()); err != nil { + if err = m.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, latestItems.ToSlice()); err != nil { log.Logger().Error("failed to cache latest items", zap.Error(err)) } - if err = m.CacheClient.DeleteScores(ctx, []string{cache.LatestItems}, cache.ScoreCondition{Before: &latestItems.Timestamp}); err != nil { + if err = m.CacheClient.DeleteScores(ctx, []string{cache.NonPersonalized}, + cache.ScoreCondition{Subset: proto.String(cache.Latest), Before: &latestItems.Timestamp}); err != nil { log.Logger().Error("failed to reclaim outdated items", zap.Error(err)) } if err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateLatestItemsTime), time.Now())); err != nil { diff --git a/server/bench_test.go b/server/bench_test.go index 66e497ad1..0a0c4a6a2 100644 --- a/server/bench_test.go +++ b/server/bench_test.go @@ -804,7 +804,7 @@ func BenchmarkGetRecommendCache(b *testing.B) { documents[i].Categories = []string{""} } lo.Reverse(documents) - err := s.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, documents) + err := s.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, documents) require.NoError(b, err) s.Config.Recommend.CacheSize = len(documents) @@ -903,7 +903,7 @@ func BenchmarkRecommendFromLatest(b *testing.B) { } lo.Reverse(documents) lo.Reverse(expects) - err := s.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, documents) + err := s.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, documents) require.NoError(b, err) err = s.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) require.NoError(b, err) diff --git a/server/rest.go b/server/rest.go index 6bff7b011..2c707dd68 100644 --- a/server/rest.go +++ b/server/rest.go @@ -650,20 +650,20 @@ func (s *RestServer) searchDocuments(collection, subset, category string, isItem func (s *RestServer) getPopular(request *restful.Request, response *restful.Response) { category := request.PathParameter("category") log.ResponseLogger(response).Debug("get category popular items in category", zap.String("category", category)) - s.searchDocuments(cache.Leaderboard, cache.Popular, category, true, request, response) + s.searchDocuments(cache.NonPersonalized, cache.Popular, category, true, request, response) } func (s *RestServer) getLatest(request *restful.Request, response *restful.Response) { category := request.PathParameter("category") log.ResponseLogger(response).Debug("get category latest items in category", zap.String("category", category)) - s.searchDocuments(cache.Leaderboard, cache.Latest, category, true, request, response) + s.searchDocuments(cache.NonPersonalized, cache.Latest, category, true, request, response) } func (s *RestServer) getLeaderboard(request *restful.Request, response *restful.Response) { name := request.PathParameter("name") category := request.QueryParameter("category") log.ResponseLogger(response).Debug("get leaderboard", zap.String("name", name)) - s.searchDocuments(cache.Leaderboard, name, category, false, request, response) + s.searchDocuments(cache.NonPersonalized, name, category, false, request, response) } // get feedback by item-id with feedback type @@ -946,7 +946,7 @@ func (s *RestServer) RecommendItemBased(ctx *recommendContext) error { func (s *RestServer) RecommendLatest(ctx *recommendContext) error { if len(ctx.results) < ctx.n { start := time.Now() - items, err := s.CacheClient.SearchScores(ctx.context, cache.Leaderboard, cache.Latest, ctx.categories, 0, s.Config.Recommend.CacheSize) + items, err := s.CacheClient.SearchScores(ctx.context, cache.NonPersonalized, cache.Latest, ctx.categories, 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -966,7 +966,7 @@ func (s *RestServer) RecommendLatest(ctx *recommendContext) error { func (s *RestServer) RecommendPopular(ctx *recommendContext) error { if len(ctx.results) < ctx.n { start := time.Now() - items, err := s.CacheClient.SearchScores(ctx.context, cache.Leaderboard, cache.Popular, ctx.categories, 0, s.Config.Recommend.CacheSize) + items, err := s.CacheClient.SearchScores(ctx.context, cache.NonPersonalized, cache.Popular, ctx.categories, 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -1396,7 +1396,7 @@ func (s *RestServer) batchInsertItems(ctx context.Context, response *restful.Res Comment: item.Comment, }) // insert to latest items cache - if err = s.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, []cache.Score{{ + if err = s.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, []cache.Score{{ Id: item.ItemId, Score: float64(timestamp.Unix()), Categories: withWildCard(item.Categories), @@ -1517,7 +1517,7 @@ func (s *RestServer) modifyItem(request *restful.Request, response *restful.Resp } // add item to latest items cache if patch.Timestamp != nil { - if err := s.CacheClient.UpdateScores(ctx, []string{cache.Leaderboard}, cache.Latest, itemId, cache.ScorePatch{Score: proto.Float64(float64(patch.Timestamp.Unix()))}); err != nil { + if err := s.CacheClient.UpdateScores(ctx, []string{cache.NonPersonalized}, cache.Latest, itemId, cache.ScorePatch{Score: proto.Float64(float64(patch.Timestamp.Unix()))}); err != nil { InternalServerError(response, err) return } diff --git a/server/rest_test.go b/server/rest_test.go index 3a5158135..e9c3f51bb 100644 --- a/server/rest_test.go +++ b/server/rest_test.go @@ -243,7 +243,7 @@ func (suite *ServerTestSuite) TestItems() { }, } // insert popular scores - err := suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, []cache.Score{ + err := suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, []cache.Score{ {Id: "0", Score: 10}, {Id: "2", Score: 12}, {Id: "4", Score: 14}, @@ -822,12 +822,12 @@ func (suite *ServerTestSuite) TestNonPersonalizedRecommend() { //{"User Neighbors", cache.Collection(cache.UserNeighbors, "0"), "/api/user/0/neighbors"}, {"Item Neighbors", cache.ItemNeighbors, "0", "", "/api/item/0/neighbors"}, {"Item Neighbors in Category", cache.ItemNeighbors, "0", "0", "/api/item/0/neighbors/0"}, - {"LatestItems", cache.Leaderboard, cache.Latest, "", "/api/latest/"}, - {"LatestItemsCategory", cache.Leaderboard, cache.Latest, "0", "/api/latest/0"}, - {"PopularItems", cache.Leaderboard, cache.Popular, "", "/api/popular/"}, - {"PopularItemsCategory", cache.Leaderboard, cache.Popular, "0", "/api/popular/0"}, - {"Leaderboard", cache.Leaderboard, "trending", "", "/api/leaderboard/trending"}, - {"LeaderboardCategory", cache.Leaderboard, "trending", "0", "/api/leaderboard/trending"}, + {"LatestItems", cache.NonPersonalized, cache.Latest, "", "/api/latest/"}, + {"LatestItemsCategory", cache.NonPersonalized, cache.Latest, "0", "/api/latest/0"}, + {"PopularItems", cache.NonPersonalized, cache.Popular, "", "/api/popular/"}, + {"PopularItemsCategory", cache.NonPersonalized, cache.Popular, "0", "/api/popular/0"}, + {"NonPersonalized", cache.NonPersonalized, "trending", "", "/api/leaderboard/trending"}, + {"LeaderboardCategory", cache.NonPersonalized, "trending", "0", "/api/leaderboard/trending"}, {"Offline Recommend", cache.OfflineRecommend, "0", "", "/api/intermediate/recommend/0"}, {"Offline Recommend in Category", cache.OfflineRecommend, "0", "0", "/api/intermediate/recommend/0/0"}, } @@ -1417,26 +1417,26 @@ func (suite *ServerTestSuite) TestGetRecommendsFallbackPreCached() { {Id: "104", Score: 96, Categories: []string{"*"}}}) assert.NoError(t, err) // insert latest - err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, []cache.Score{ {Id: "5", Score: 95, Categories: []string{""}}, {Id: "6", Score: 94, Categories: []string{""}}, {Id: "7", Score: 93, Categories: []string{""}}, {Id: "8", Score: 92, Categories: []string{""}}}) assert.NoError(t, err) - err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, []cache.Score{ {Id: "105", Score: 95, Categories: []string{"*"}}, {Id: "106", Score: 94, Categories: []string{"*"}}, {Id: "107", Score: 93, Categories: []string{"*"}}, {Id: "108", Score: 92, Categories: []string{"*"}}}) assert.NoError(t, err) // insert popular - err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, []cache.Score{ {Id: "9", Score: 91, Categories: []string{""}}, {Id: "10", Score: 90, Categories: []string{""}}, {Id: "11", Score: 89, Categories: []string{""}}, {Id: "12", Score: 88, Categories: []string{""}}}) assert.NoError(t, err) - err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, []cache.Score{ {Id: "109", Score: 91, Categories: []string{"*"}}, {Id: "110", Score: 90, Categories: []string{"*"}}, {Id: "111", Score: 89, Categories: []string{"*"}}, @@ -1676,9 +1676,9 @@ func (suite *ServerTestSuite) TestVisibility() { }) } lo.Reverse(documents) - err := suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Latest, documents) + err := suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, documents) assert.NoError(t, err) - err = suite.CacheClient.AddScores(ctx, cache.Leaderboard, cache.Popular, documents) + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, documents) assert.NoError(t, err) err = suite.CacheClient.AddScores(ctx, cache.ItemNeighbors, "100", documents) assert.NoError(t, err) diff --git a/storage/cache/database.go b/storage/cache/database.go index e332cda7d..94d4ec024 100644 --- a/storage/cache/database.go +++ b/storage/cache/database.go @@ -76,9 +76,9 @@ const ( // Recommendation digest - offline_recommend_digest/{user_id} OfflineRecommendDigest = "offline_recommend_digest" - Leaderboard = "leaderboard" - Latest = "latest" - Popular = "popular" + NonPersonalized = "non-personalized" + Latest = "latest" + Popular = "popular" // ItemCategories is the set of item categories. The format of key: // Global item categories - item_categories @@ -109,7 +109,7 @@ const ( MatchingIndexRecall = "matching_index_recall" ) -var ItemCache = []string{Leaderboard, ItemNeighbors, OfflineRecommend} +var ItemCache = []string{NonPersonalized, ItemNeighbors, OfflineRecommend} var ( ErrObjectNotExist = errors.NotFoundf("object") From faf15b68b97e71ff15cafc6ff04420d6361b19ab Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Sat, 7 Dec 2024 21:46:11 +0800 Subject: [PATCH 06/19] proto: add optional subset field --- protocol/cache_store.pb.go | 234 +++++++++++++++++--------------- protocol/cache_store.proto | 5 +- protocol/cache_store_grpc.pb.go | 2 +- protocol/data_store.pb.go | 4 +- protocol/data_store_grpc.pb.go | 2 +- protocol/protocol.pb.go | 4 +- protocol/protocol_grpc.pb.go | 2 +- 7 files changed, 133 insertions(+), 120 deletions(-) diff --git a/protocol/cache_store.pb.go b/protocol/cache_store.pb.go index 8501c881d..42e1711d8 100644 --- a/protocol/cache_store.pb.go +++ b/protocol/cache_store.pb.go @@ -14,8 +14,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.28.3 +// protoc-gen-go v1.35.2 +// protoc v5.29.0 // source: cache_store.proto package protocol @@ -1540,8 +1540,9 @@ type UpdateScoresRequest struct { unknownFields protoimpl.UnknownFields Collection []string `protobuf:"bytes,1,rep,name=collection,proto3" json:"collection,omitempty"` - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` - Patch *ScorePatch `protobuf:"bytes,3,opt,name=patch,proto3" json:"patch,omitempty"` + Subset *string `protobuf:"bytes,2,opt,name=subset,proto3,oneof" json:"subset,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` + Patch *ScorePatch `protobuf:"bytes,4,opt,name=patch,proto3" json:"patch,omitempty"` } func (x *UpdateScoresRequest) Reset() { @@ -1581,6 +1582,13 @@ func (x *UpdateScoresRequest) GetCollection() []string { return nil } +func (x *UpdateScoresRequest) GetSubset() string { + if x != nil && x.Subset != nil { + return *x.Subset + } + return "" +} + func (x *UpdateScoresRequest) GetId() string { if x != nil { return x.Id @@ -1943,114 +1951,117 @@ var file_cache_store_proto_rawDesc = []byte{ 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x16, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x71, 0x0a, 0x13, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, - 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, - 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x05, - 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x63, - 0x68, 0x52, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x22, 0x16, 0x0a, 0x14, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x4f, 0x0a, 0x1a, 0x41, 0x64, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, - 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, - 0x0a, 0x06, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, - 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x06, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x73, 0x22, 0x1d, 0x0a, 0x1b, 0x41, 0x64, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, - 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x90, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, - 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, - 0x62, 0x65, 0x67, 0x69, 0x6e, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, - 0x65, 0x6e, 0x64, 0x22, 0x50, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, - 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x06, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x32, 0xa1, 0x09, 0x0a, 0x0a, 0x43, 0x61, 0x63, 0x68, 0x65, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x12, 0x37, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x15, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, - 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x34, 0x0a, - 0x03, 0x47, 0x65, 0x74, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x03, 0x53, 0x65, 0x74, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x99, 0x01, 0x0a, 0x13, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, + 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, + 0x06, 0x73, 0x75, 0x62, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x06, 0x73, 0x75, 0x62, 0x73, 0x65, 0x74, 0x88, 0x01, 0x01, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x05, 0x70, 0x61, + 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x63, 0x68, 0x52, + 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x75, 0x62, 0x73, 0x65, + 0x74, 0x22, 0x16, 0x0a, 0x14, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x4f, 0x0a, 0x1a, 0x41, 0x64, 0x64, + 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x06, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x52, 0x06, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x1d, 0x0a, 0x1b, 0x41, 0x64, + 0x64, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x90, 0x01, 0x0a, 0x1a, 0x47, 0x65, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x05, + 0x62, 0x65, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x12, 0x2c, + 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x50, 0x0a, 0x1b, + 0x47, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, + 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x06, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x32, 0xa1, + 0x09, 0x0a, 0x0a, 0x43, 0x61, 0x63, 0x68, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x37, 0x0a, + 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x14, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x03, + 0x53, 0x65, 0x74, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x53, - 0x65, 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x53, 0x65, 0x74, 0x53, 0x65, - 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, - 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x41, 0x64, 0x64, 0x53, 0x65, 0x74, - 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x53, - 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x12, - 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x53, 0x65, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x04, 0x50, 0x75, 0x73, 0x68, 0x12, 0x15, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, - 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x34, 0x0a, - 0x03, 0x50, 0x6f, 0x70, 0x12, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x50, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x17, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x46, 0x0a, 0x09, 0x41, 0x64, 0x64, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x12, - 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x63, - 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x53, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x63, 0x6f, 0x72, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x63, 0x6f, 0x72, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x63, 0x6f, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, - 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, 0x6f, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, - 0x13, 0x41, 0x64, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, - 0x69, 0x6e, 0x74, 0x73, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x41, 0x64, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, - 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, - 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, - 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, - 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x54, - 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x7a, 0x68, 0x65, 0x6e, 0x67, 0x68, 0x61, 0x6f, - 0x7a, 0x2f, 0x67, 0x6f, 0x72, 0x73, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x3d, 0x0a, 0x06, 0x53, 0x65, 0x74, 0x53, 0x65, 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x74, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x3d, 0x0a, 0x06, 0x41, 0x64, 0x64, 0x53, 0x65, 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, + 0x64, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, + 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, + 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x37, 0x0a, + 0x04, 0x50, 0x75, 0x73, 0x68, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x2e, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x03, 0x50, 0x6f, 0x70, 0x12, 0x14, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, + 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, + 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x61, 0x69, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x09, 0x41, + 0x64, 0x64, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, + 0x41, 0x64, 0x64, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x63, 0x6f, + 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x63, + 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x63, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x13, 0x41, 0x64, 0x64, 0x54, 0x69, 0x6d, + 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x24, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x54, 0x69, 0x6d, 0x65, + 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x41, + 0x64, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x13, + 0x47, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x73, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x69, + 0x65, 0x73, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x42, 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x7a, 0x68, 0x65, 0x6e, 0x67, 0x68, 0x61, 0x6f, 0x7a, 0x2f, 0x67, 0x6f, 0x72, 0x73, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -2172,6 +2183,7 @@ func file_cache_store_proto_init() { file_cache_store_proto_msgTypes[3].OneofWrappers = []any{} file_cache_store_proto_msgTypes[6].OneofWrappers = []any{} file_cache_store_proto_msgTypes[22].OneofWrappers = []any{} + file_cache_store_proto_msgTypes[31].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/protocol/cache_store.proto b/protocol/cache_store.proto index ddd5cf610..f764e25d9 100644 --- a/protocol/cache_store.proto +++ b/protocol/cache_store.proto @@ -153,8 +153,9 @@ message DeleteScoresResponse {} message UpdateScoresRequest { repeated string collection = 1; - string id = 2; - ScorePatch patch = 3; + optional string subset = 2; + string id = 3; + ScorePatch patch = 4; } message UpdateScoresResponse {} diff --git a/protocol/cache_store_grpc.pb.go b/protocol/cache_store_grpc.pb.go index 090f70e1c..a63b0bb3e 100644 --- a/protocol/cache_store_grpc.pb.go +++ b/protocol/cache_store_grpc.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.5.1 -// - protoc v5.28.3 +// - protoc v5.29.0 // source: cache_store.proto package protocol diff --git a/protocol/data_store.pb.go b/protocol/data_store.pb.go index 98a96ec57..726e9aea5 100644 --- a/protocol/data_store.pb.go +++ b/protocol/data_store.pb.go @@ -14,8 +14,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.28.3 +// protoc-gen-go v1.35.2 +// protoc v5.29.0 // source: data_store.proto package protocol diff --git a/protocol/data_store_grpc.pb.go b/protocol/data_store_grpc.pb.go index 2d814a9a7..23ea72086 100644 --- a/protocol/data_store_grpc.pb.go +++ b/protocol/data_store_grpc.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.5.1 -// - protoc v5.28.3 +// - protoc v5.29.0 // source: data_store.proto package protocol diff --git a/protocol/protocol.pb.go b/protocol/protocol.pb.go index 5c087c4e9..ac23fefc0 100644 --- a/protocol/protocol.pb.go +++ b/protocol/protocol.pb.go @@ -14,8 +14,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.28.3 +// protoc-gen-go v1.35.2 +// protoc v5.29.0 // source: protocol.proto package protocol diff --git a/protocol/protocol_grpc.pb.go b/protocol/protocol_grpc.pb.go index 7a720c69d..864fc7691 100644 --- a/protocol/protocol_grpc.pb.go +++ b/protocol/protocol_grpc.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.5.1 -// - protoc v5.28.3 +// - protoc v5.29.0 // source: protocol.proto package protocol From e6557882c64c1a34d7357cf5bd5bcf61c520e430 Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Sat, 7 Dec 2024 22:02:03 +0800 Subject: [PATCH 07/19] cache: add test --- storage/cache/database.go | 2 +- storage/cache/database_test.go | 21 ++++++++++++++++----- storage/cache/mongodb.go | 4 ++-- storage/cache/no_database.go | 2 +- storage/cache/no_database_test.go | 2 +- storage/cache/proxy.go | 5 +++-- storage/cache/redis.go | 6 +++--- storage/cache/redis_test.go | 2 +- storage/cache/sql.go | 4 ++-- 9 files changed, 30 insertions(+), 18 deletions(-) diff --git a/storage/cache/database.go b/storage/cache/database.go index 4215f6717..f550774c3 100644 --- a/storage/cache/database.go +++ b/storage/cache/database.go @@ -288,7 +288,7 @@ type Database interface { AddScores(ctx context.Context, collection, subset string, documents []Score) error SearchScores(ctx context.Context, collection, subset string, query []string, begin, end int) ([]Score, error) DeleteScores(ctx context.Context, collection []string, condition ScoreCondition) error - UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error + UpdateScores(ctx context.Context, collections []string, subset *string, id string, patch ScorePatch) error AddTimeSeriesPoints(ctx context.Context, points []TimeSeriesPoint) error GetTimeSeriesPoints(ctx context.Context, name string, begin, end time.Time) ([]TimeSeriesPoint, error) diff --git a/storage/cache/database_test.go b/storage/cache/database_test.go index bbfb6f30b..00f214b6d 100644 --- a/storage/cache/database_test.go +++ b/storage/cache/database_test.go @@ -324,20 +324,20 @@ func (suite *baseTestSuite) TestDocument() { suite.Equal("2", documents[0].Id) // update categories - err = suite.UpdateScores(ctx, []string{"a"}, "", "2", ScorePatch{Categories: []string{"c", "s"}}) + err = suite.UpdateScores(ctx, []string{"a"}, nil, "2", ScorePatch{Categories: []string{"c", "s"}}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "", []string{"s"}, 0, 1) suite.NoError(err) suite.Len(documents, 1) suite.Equal("2", documents[0].Id) - err = suite.UpdateScores(ctx, []string{"a"}, "", "2", ScorePatch{Categories: []string{"c"}}) + err = suite.UpdateScores(ctx, []string{"a"}, nil, "2", ScorePatch{Categories: []string{"c"}}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "", []string{"s"}, 0, 1) suite.NoError(err) suite.Empty(documents) // update is hidden - err = suite.UpdateScores(ctx, []string{"a"}, "", "0", ScorePatch{IsHidden: proto.Bool(false)}) + err = suite.UpdateScores(ctx, []string{"a"}, nil, "0", ScorePatch{IsHidden: proto.Bool(false)}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "", []string{"b"}, 0, 1) suite.NoError(err) @@ -401,7 +401,7 @@ func (suite *baseTestSuite) TestSubsetDocument() { }, documents) // update categories - err = suite.UpdateScores(ctx, []string{"a", "b"}, "", "2", ScorePatch{Categories: []string{"b", "s"}}) + err = suite.UpdateScores(ctx, []string{"a", "b"}, nil, "2", ScorePatch{Categories: []string{"b", "s"}}) suite.NoError(err) documents, err = suite.SearchScores(ctx, "a", "a", []string{"s"}, 0, 1) suite.NoError(err) @@ -412,6 +412,17 @@ func (suite *baseTestSuite) TestSubsetDocument() { suite.Len(documents, 1) suite.Equal("2", documents[0].Id) + // update categories in subset + err = suite.UpdateScores(ctx, []string{"a", "b"}, proto.String("a"), "2", ScorePatch{Categories: []string{"b", "x"}}) + suite.NoError(err) + documents, err = suite.SearchScores(ctx, "a", "a", []string{"x"}, 0, 1) + suite.NoError(err) + suite.Len(documents, 1) + suite.Equal("2", documents[0].Id) + documents, err = suite.SearchScores(ctx, "b", "", []string{"x"}, 0, 1) + suite.NoError(err) + suite.Empty(documents) + // delete by value err = suite.DeleteScores(ctx, []string{"a", "b"}, ScoreCondition{Id: proto.String("3")}) suite.NoError(err) @@ -551,7 +562,7 @@ func benchmarkUpdateDocuments(b *testing.B, database Database) { // select a random number n := rand.Intn(benchmarkDataSize) + 1 // update documents - err := database.UpdateScores(ctx, []string{"a"}, "", strconv.Itoa(n), ScorePatch{ + err := database.UpdateScores(ctx, []string{"a"}, nil, strconv.Itoa(n), ScorePatch{ Score: proto.Float64(float64(n)), }) assert.NoError(b, err) diff --git a/storage/cache/mongodb.go b/storage/cache/mongodb.go index 865913a37..82fa4c01e 100644 --- a/storage/cache/mongodb.go +++ b/storage/cache/mongodb.go @@ -369,7 +369,7 @@ func (m MongoDB) SearchScores(ctx context.Context, collection, subset string, qu return documents, nil } -func (m MongoDB) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { +func (m MongoDB) UpdateScores(ctx context.Context, collections []string, subset *string, id string, patch ScorePatch) error { if len(collections) == 0 { return nil } @@ -380,7 +380,7 @@ func (m MongoDB) UpdateScores(ctx context.Context, collections []string, subset, "collection": bson.M{"$in": collections}, "id": id, } - if subset != "" { + if subset != nil { filter["subset"] = subset } update := bson.D{} diff --git a/storage/cache/no_database.go b/storage/cache/no_database.go index bd572fca7..5bc65cbd7 100644 --- a/storage/cache/no_database.go +++ b/storage/cache/no_database.go @@ -98,7 +98,7 @@ func (NoDatabase) SearchScores(_ context.Context, _, _ string, _ []string, _, _ return nil, ErrNoDatabase } -func (NoDatabase) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { +func (NoDatabase) UpdateScores(context.Context, []string, *string, string, ScorePatch) error { return ErrNoDatabase } diff --git a/storage/cache/no_database_test.go b/storage/cache/no_database_test.go index 3e78b1dc9..d064b1466 100644 --- a/storage/cache/no_database_test.go +++ b/storage/cache/no_database_test.go @@ -66,7 +66,7 @@ func TestNoDatabase(t *testing.T) { assert.ErrorIs(t, err, ErrNoDatabase) _, err = database.SearchScores(ctx, "", "", nil, 0, 0) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.UpdateScores(ctx, nil, "", "", ScorePatch{}) + err = database.UpdateScores(ctx, nil, nil, "", ScorePatch{}) assert.ErrorIs(t, err, ErrNoDatabase) err = database.DeleteScores(ctx, nil, ScoreCondition{}) assert.ErrorIs(t, err, ErrNoDatabase) diff --git a/storage/cache/proxy.go b/storage/cache/proxy.go index d79b3aad4..db1371cf9 100644 --- a/storage/cache/proxy.go +++ b/storage/cache/proxy.go @@ -162,7 +162,7 @@ func (p *ProxyServer) DeleteScores(ctx context.Context, request *protocol.Delete } func (p *ProxyServer) UpdateScores(ctx context.Context, request *protocol.UpdateScoresRequest) (*protocol.UpdateScoresResponse, error) { - return &protocol.UpdateScoresResponse{}, p.database.UpdateScores(ctx, request.GetCollection(), request.GetId(), ScorePatch{ + return &protocol.UpdateScoresResponse{}, p.database.UpdateScores(ctx, request.GetCollection(), request.Subset, request.GetId(), ScorePatch{ IsHidden: request.GetPatch().IsHidden, Categories: request.GetPatch().Categories, Score: request.GetPatch().Score, @@ -380,9 +380,10 @@ func (p ProxyClient) DeleteScores(ctx context.Context, collection []string, cond return err } -func (p ProxyClient) UpdateScores(ctx context.Context, collection []string, id string, patch ScorePatch) error { +func (p ProxyClient) UpdateScores(ctx context.Context, collection []string, subset *string, id string, patch ScorePatch) error { _, err := p.CacheStoreClient.UpdateScores(ctx, &protocol.UpdateScoresRequest{ Collection: collection, + Subset: subset, Id: id, Patch: &protocol.ScorePatch{ Score: patch.Score, diff --git a/storage/cache/redis.go b/storage/cache/redis.go index 0a5a5dfff..b2a0a7386 100644 --- a/storage/cache/redis.go +++ b/storage/cache/redis.go @@ -307,7 +307,7 @@ func (r *Redis) SearchScores(ctx context.Context, collection, subset string, que return documents, nil } -func (r *Redis) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { +func (r *Redis) UpdateScores(ctx context.Context, collections []string, subset *string, id string, patch ScorePatch) error { if len(collections) == 0 { return nil } @@ -317,8 +317,8 @@ func (r *Redis) UpdateScores(ctx context.Context, collections []string, subset, var builder strings.Builder builder.WriteString(fmt.Sprintf("@collection:{ %s }", escape(strings.Join(collections, " | ")))) builder.WriteString(fmt.Sprintf(" @id:{ %s }", escape(id))) - if subset != "" { - builder.WriteString(fmt.Sprintf(" @subset:{ %s }", escape(subset))) + if subset != nil { + builder.WriteString(fmt.Sprintf(" @subset:{ %s }", escape(*subset))) } for { // search documents diff --git a/storage/cache/redis_test.go b/storage/cache/redis_test.go index cfba0b281..ceefdd88a 100644 --- a/storage/cache/redis_test.go +++ b/storage/cache/redis_test.go @@ -78,7 +78,7 @@ func (suite *RedisTestSuite) TestEscapeCharacters() { suite.NoError(err) suite.Equal([]Score{{Id: id, Score: math.MaxFloat64, Categories: []string{"a", "b"}, Timestamp: ts}}, documents) - err = suite.UpdateScores(ctx, []string{collection}, "", id, ScorePatch{Score: proto.Float64(1)}) + err = suite.UpdateScores(ctx, []string{collection}, nil, id, ScorePatch{Score: proto.Float64(1)}) suite.NoError(err) documents, err = suite.SearchScores(ctx, collection, subset, []string{"b"}, 0, -1) suite.NoError(err) diff --git a/storage/cache/sql.go b/storage/cache/sql.go index a3703da6d..89880fd7c 100644 --- a/storage/cache/sql.go +++ b/storage/cache/sql.go @@ -477,7 +477,7 @@ func (db *SQLDatabase) SearchScores(ctx context.Context, collection, subset stri return documents, nil } -func (db *SQLDatabase) UpdateScores(ctx context.Context, collections []string, subset, id string, patch ScorePatch) error { +func (db *SQLDatabase) UpdateScores(ctx context.Context, collections []string, subset *string, id string, patch ScorePatch) error { if len(collections) == 0 { return nil } @@ -485,7 +485,7 @@ func (db *SQLDatabase) UpdateScores(ctx context.Context, collections []string, s return nil } tx := db.gormDB.WithContext(ctx).Model(&PostgresDocument{}). - Where("collection in (?) and id = ? and (subset = ? or ? = '')", + Where("collection in (?) and id = ? and (subset = ? or ? is null)", collections, id, subset, subset) if patch.Score != nil { tx = tx.Update("score", *patch.Score) From d6fdf772aa6594af491002ced9c4c15b3ace1bba Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Sat, 7 Dec 2024 22:10:12 +0800 Subject: [PATCH 08/19] fix build --- master/rest_test.go | 8 ++++---- master/tasks_test.go | 8 ++++---- server/rest.go | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/master/rest_test.go b/master/rest_test.go index 2bfcab71b..9027c41ee 100644 --- a/master/rest_test.go +++ b/master/rest_test.go @@ -494,10 +494,10 @@ func TestServer_SearchDocumentsOfItems(t *testing.T) { operators := []ListOperator{ {"Item Neighbors", cache.ItemNeighbors, "0", "", "/api/dashboard/item/0/neighbors"}, {"Item Neighbors in Category", cache.ItemNeighbors, "0", "*", "/api/dashboard/item/0/neighbors/*"}, - {"Latest Items", cache.LatestItems, "", "", "/api/dashboard/latest/"}, - {"Popular Items", cache.PopularItems, "", "", "/api/dashboard/popular/"}, - {"Latest Items in Category", cache.LatestItems, "", "*", "/api/dashboard/latest/*"}, - {"Popular Items in Category", cache.PopularItems, "", "*", "/api/dashboard/popular/*"}, + {"Latest Items", cache.NonPersonalized, cache.Latest, "", "/api/dashboard/latest/"}, + {"Popular Items", cache.NonPersonalized, cache.Popular, "", "/api/dashboard/popular/"}, + {"Latest Items in Category", cache.NonPersonalized, cache.Latest, "*", "/api/dashboard/latest/*"}, + {"Popular Items in Category", cache.NonPersonalized, cache.Popular, "*", "/api/dashboard/popular/*"}, } for i, operator := range operators { t.Run(operator.Name, func(t *testing.T) { diff --git a/master/tasks_test.go b/master/tasks_test.go index 8fa037047..3e5143858 100644 --- a/master/tasks_test.go +++ b/master/tasks_test.go @@ -568,7 +568,7 @@ func (s *MasterTestSuite) TestLoadDataFromDatabase() { s.Equal(45, s.clickTrainSet.NegativeCount+s.clickTestSet.NegativeCount) // check latest items - latest, err := s.CacheClient.SearchScores(ctx, cache.LatestItems, "", []string{""}, 0, 100) + latest, err := s.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Latest, []string{""}, 0, 100) s.NoError(err) s.Equal([]cache.Score{ {Id: items[8].ItemId, Score: float64(items[8].Timestamp.Unix())}, @@ -577,7 +577,7 @@ func (s *MasterTestSuite) TestLoadDataFromDatabase() { }, lo.Map(latest, func(document cache.Score, _ int) cache.Score { return cache.Score{Id: document.Id, Score: document.Score} })) - latest, err = s.CacheClient.SearchScores(ctx, cache.LatestItems, "", []string{"2"}, 0, 100) + latest, err = s.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Latest, []string{"2"}, 0, 100) s.NoError(err) s.Equal([]cache.Score{ {Id: items[8].ItemId, Score: float64(items[8].Timestamp.Unix())}, @@ -588,7 +588,7 @@ func (s *MasterTestSuite) TestLoadDataFromDatabase() { })) // check popular items - popular, err := s.CacheClient.SearchScores(ctx, cache.PopularItems, "", []string{""}, 0, 3) + popular, err := s.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Popular, []string{""}, 0, 3) s.NoError(err) s.Equal([]cache.Score{ {Id: items[8].ItemId, Score: 9}, @@ -597,7 +597,7 @@ func (s *MasterTestSuite) TestLoadDataFromDatabase() { }, lo.Map(popular, func(document cache.Score, _ int) cache.Score { return cache.Score{Id: document.Id, Score: document.Score} })) - popular, err = s.CacheClient.SearchScores(ctx, cache.PopularItems, "", []string{"2"}, 0, 3) + popular, err = s.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Popular, []string{"2"}, 0, 3) s.NoError(err) s.Equal([]cache.Score{ {Id: items[8].ItemId, Score: 9}, diff --git a/server/rest.go b/server/rest.go index 2c707dd68..6294a6d71 100644 --- a/server/rest.go +++ b/server/rest.go @@ -1406,7 +1406,7 @@ func (s *RestServer) batchInsertItems(ctx context.Context, response *restful.Res return } // update items cache - if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", item.ItemId, cache.ScorePatch{ + if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, nil, item.ItemId, cache.ScorePatch{ Categories: withWildCard(item.Categories), IsHidden: &item.IsHidden, }); err != nil { @@ -1510,21 +1510,21 @@ func (s *RestServer) modifyItem(request *restful.Request, response *restful.Resp } // remove hidden item from cache if patch.IsHidden != nil { - if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{IsHidden: patch.IsHidden}); err != nil { + if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, nil, itemId, cache.ScorePatch{IsHidden: patch.IsHidden}); err != nil { InternalServerError(response, err) return } } // add item to latest items cache if patch.Timestamp != nil { - if err := s.CacheClient.UpdateScores(ctx, []string{cache.NonPersonalized}, cache.Latest, itemId, cache.ScorePatch{Score: proto.Float64(float64(patch.Timestamp.Unix()))}); err != nil { + if err := s.CacheClient.UpdateScores(ctx, []string{cache.NonPersonalized}, proto.String(cache.Latest), itemId, cache.ScorePatch{Score: proto.Float64(float64(patch.Timestamp.Unix()))}); err != nil { InternalServerError(response, err) return } } // update categories in cache if patch.Categories != nil { - if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{Categories: withWildCard(patch.Categories)}); err != nil { + if err := s.CacheClient.UpdateScores(ctx, cache.ItemCache, nil, itemId, cache.ScorePatch{Categories: withWildCard(patch.Categories)}); err != nil { InternalServerError(response, err) return } @@ -1628,7 +1628,7 @@ func (s *RestServer) insertItemCategory(request *restful.Request, response *rest return } // insert category to cache - if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{Categories: withWildCard(item.Categories)}); err != nil { + if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, nil, itemId, cache.ScorePatch{Categories: withWildCard(item.Categories)}); err != nil { InternalServerError(response, err) return } @@ -1657,7 +1657,7 @@ func (s *RestServer) deleteItemCategory(request *restful.Request, response *rest } item.Categories = categories // delete category from cache - if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, "", itemId, cache.ScorePatch{Categories: withWildCard(categories)}); err != nil { + if err = s.CacheClient.UpdateScores(ctx, cache.ItemCache, nil, itemId, cache.ScorePatch{Categories: withWildCard(categories)}); err != nil { InternalServerError(response, err) return } From 0c5ef3ac9f6850ccd4a6f8372a365201daa94789 Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Sat, 7 Dec 2024 22:11:58 +0800 Subject: [PATCH 09/19] remove gorse-cli --- cmd/gorse-cli/main.go | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 cmd/gorse-cli/main.go diff --git a/cmd/gorse-cli/main.go b/cmd/gorse-cli/main.go deleted file mode 100644 index a16f9e36c..000000000 --- a/cmd/gorse-cli/main.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "github.com/spf13/cobra" - "github.com/zhenghaoz/gorse/base/log" - "go.uber.org/zap" -) - -var rootCmd = &cobra.Command{ - Use: "gorse-cli", - Short: "Gorse command line tool", -} - -var dumpCmd = &cobra.Command{ - Use: "dump", -} - -var restoreCmd = &cobra.Command{ - Use: "restore", -} - -func init() { - rootCmd.AddCommand(dumpCmd) - rootCmd.AddCommand(restoreCmd) -} - -func main() { - if err := rootCmd.Execute(); err != nil { - log.Logger().Fatal("failed to execute command", zap.Error(err)) - } -} From ce6a5358aa1e4ffc5faafc325957f90ed3b88385 Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Sat, 7 Dec 2024 22:19:15 +0800 Subject: [PATCH 10/19] remove leaderboard to non-personalized --- server/rest.go | 8 ++++---- server/rest_test.go | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/rest.go b/server/rest.go index 6294a6d71..327e82b7e 100644 --- a/server/rest.go +++ b/server/rest.go @@ -471,9 +471,9 @@ func (s *RestServer) CreateWebService() { Param(ws.QueryParameter("user-id", "Remove read items of a user").DataType("string")). Returns(http.StatusOK, "OK", []cache.Score{}). Writes([]cache.Score{})) - // Get leaderboard - ws.Route(ws.GET("/leaderboard/{name}").To(s.getLeaderboard). - Doc("Get the leaderboard."). + // Get non-personalized + ws.Route(ws.GET("/non-personalized/{name}").To(s.getNonPersonalized). + Doc("Get non-personalized recommendations."). Metadata(restfulspec.KeyOpenAPITags, []string{RecommendationAPITag}). Param(ws.HeaderParameter("X-API-Key", "API key").DataType("string")). Param(ws.QueryParameter("category", "Category of returned items.").DataType("string")). @@ -659,7 +659,7 @@ func (s *RestServer) getLatest(request *restful.Request, response *restful.Respo s.searchDocuments(cache.NonPersonalized, cache.Latest, category, true, request, response) } -func (s *RestServer) getLeaderboard(request *restful.Request, response *restful.Response) { +func (s *RestServer) getNonPersonalized(request *restful.Request, response *restful.Response) { name := request.PathParameter("name") category := request.QueryParameter("category") log.ResponseLogger(response).Debug("get leaderboard", zap.String("name", name)) diff --git a/server/rest_test.go b/server/rest_test.go index e9c3f51bb..cdfa8f588 100644 --- a/server/rest_test.go +++ b/server/rest_test.go @@ -826,8 +826,8 @@ func (suite *ServerTestSuite) TestNonPersonalizedRecommend() { {"LatestItemsCategory", cache.NonPersonalized, cache.Latest, "0", "/api/latest/0"}, {"PopularItems", cache.NonPersonalized, cache.Popular, "", "/api/popular/"}, {"PopularItemsCategory", cache.NonPersonalized, cache.Popular, "0", "/api/popular/0"}, - {"NonPersonalized", cache.NonPersonalized, "trending", "", "/api/leaderboard/trending"}, - {"LeaderboardCategory", cache.NonPersonalized, "trending", "0", "/api/leaderboard/trending"}, + {"NonPersonalized", cache.NonPersonalized, "trending", "", "/api/non-personalized/trending"}, + {"NonPersonalizedCategory", cache.NonPersonalized, "trending", "0", "/api/non-personalized/trending"}, {"Offline Recommend", cache.OfflineRecommend, "0", "", "/api/intermediate/recommend/0"}, {"Offline Recommend in Category", cache.OfflineRecommend, "0", "0", "/api/intermediate/recommend/0/0"}, } @@ -1168,8 +1168,8 @@ func (suite *ServerTestSuite) TestGetRecommendsWithMultiCategories() { Get("/api/recommend/0"). Header("X-API-Key", apiKey). QueryCollection(map[string][]string{ - "n": []string{"3"}, - "category": []string{"2", "3"}, + "n": {"3"}, + "category": {"2", "3"}, }). Expect(t). Status(http.StatusOK). @@ -1721,7 +1721,7 @@ func (suite *ServerTestSuite) TestVisibility() { JSON(items). Expect(t). Status(http.StatusOK). - Body(suite.marshal((documents[:2]))). + Body(suite.marshal(documents[:2])). End() apitest.New(). Handler(suite.handler). @@ -1730,7 +1730,7 @@ func (suite *ServerTestSuite) TestVisibility() { JSON(items). Expect(t). Status(http.StatusOK). - Body(suite.marshal((documents[:2]))). + Body(suite.marshal(documents[:2])). End() apitest.New(). Handler(suite.handler). @@ -1739,7 +1739,7 @@ func (suite *ServerTestSuite) TestVisibility() { JSON(items). Expect(t). Status(http.StatusOK). - Body(suite.marshal((documents[:2]))). + Body(suite.marshal(documents[:2])). End() apitest.New(). Handler(suite.handler). From e5ea0e2d72bdc10543abb2554037cf594eb2dce6 Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Sun, 8 Dec 2024 09:53:39 +0800 Subject: [PATCH 11/19] remove leaderboard to non-personalized --- config/config.toml | 4 +- config/config_test.go | 4 +- .../{leaderboard.go => non_personalized.go} | 38 ++++--- ...board_test.go => non_personalized_test.go} | 98 +++++++++++++++++-- 4 files changed, 123 insertions(+), 21 deletions(-) rename logics/{leaderboard.go => non_personalized.go} (77%) rename logics/{leaderboard_test.go => non_personalized_test.go} (53%) diff --git a/config/config.toml b/config/config.toml index 6d58e5761..0bb6a458c 100644 --- a/config/config.toml +++ b/config/config.toml @@ -139,10 +139,10 @@ popular_window = "720h" name = "most_starred_weekly" # The score function for items in the leaderboard. -score = "count(Feedback.FeedbackType, .FeedbackType == 'star')" +score = "count(feedback, .FeedbackType == 'star')" # The filter for items in the leaderboard. -filter = "(Item.Timestamp > now()).Hours() < 168" +filter = "(now() - item.Timestamp).Hours() < 168" [recommend.user_neighbors] diff --git a/config/config_test.go b/config/config_test.go index 77f65867a..d525275bd 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -106,8 +106,8 @@ func TestUnmarshal(t *testing.T) { // [recommend.leaderboards] assert.Len(t, config.Recommend.NonPersonalized, 1) assert.Equal(t, "most_starred_weekly", config.Recommend.NonPersonalized[0].Name) - assert.Equal(t, "count(Feedback.FeedbackType, .FeedbackType == 'star')", config.Recommend.NonPersonalized[0].Score) - assert.Equal(t, "(Item.Timestamp > now()).Hours() < 168", config.Recommend.NonPersonalized[0].Filter) + assert.Equal(t, "count(feedback, .FeedbackType == 'star')", config.Recommend.NonPersonalized[0].Score) + assert.Equal(t, "(now() - item.Timestamp).Hours() < 168", config.Recommend.NonPersonalized[0].Filter) // [recommend.user_neighbors] assert.Equal(t, "similar", config.Recommend.UserNeighbors.NeighborType) assert.True(t, config.Recommend.UserNeighbors.EnableIndex) diff --git a/logics/leaderboard.go b/logics/non_personalized.go similarity index 77% rename from logics/leaderboard.go rename to logics/non_personalized.go index cdfacbda3..5a19a4ffd 100644 --- a/logics/leaderboard.go +++ b/logics/non_personalized.go @@ -15,6 +15,7 @@ package logics import ( + "fmt" "github.com/expr-lang/expr" "github.com/expr-lang/expr/vm" "github.com/juju/errors" @@ -29,14 +30,15 @@ import ( "time" ) -type LeaderBoard struct { +type NonPersonalized struct { + name string timestamp time.Time scoreFunc *vm.Program filterFunc *vm.Program heap *heap.TopKFilter[cache.Score, float64] } -func NewLeaderBoard(cfg config.NonPersonalizedConfig, n int, timestamp time.Time) (*LeaderBoard, error) { +func NewNonPersonalized(cfg config.NonPersonalizedConfig, n int, timestamp time.Time) (*NonPersonalized, error) { // Compile score expression scoreFunc, err := expr.Compile(cfg.Score, expr.Env(map[string]any{ "item": data.Item{}, @@ -64,7 +66,8 @@ func NewLeaderBoard(cfg config.NonPersonalizedConfig, n int, timestamp time.Time return nil, errors.New("filter function must return bool") } } - return &LeaderBoard{ + return &NonPersonalized{ + name: cfg.Name, timestamp: timestamp, scoreFunc: scoreFunc, filterFunc: filterFunc, @@ -72,21 +75,26 @@ func NewLeaderBoard(cfg config.NonPersonalizedConfig, n int, timestamp time.Time }, nil } -func NewLatest(n int, timestamp time.Time) *LeaderBoard { - return lo.Must(NewLeaderBoard(config.NonPersonalizedConfig{ +func NewLatest(n int, timestamp time.Time) *NonPersonalized { + return lo.Must(NewNonPersonalized(config.NonPersonalizedConfig{ Name: "latest", Score: "item.Timestamp.Unix()", }, n, timestamp)) } -func NewPopular(n int, timestamp time.Time) *LeaderBoard { - return lo.Must(NewLeaderBoard(config.NonPersonalizedConfig{ - Name: "popular", - Score: "len(feedback)", +func NewPopular(window time.Duration, n int, timestamp time.Time) *NonPersonalized { + var filter string + if window > 0 { + filter = fmt.Sprintf("(now() - item.Timestamp).Nanoseconds() < %d", window.Nanoseconds()) + } + return lo.Must(NewNonPersonalized(config.NonPersonalizedConfig{ + Name: "popular", + Score: "len(feedback)", + Filter: filter, }, n, timestamp)) } -func (l *LeaderBoard) Push(item data.Item, feedback []data.Feedback) { +func (l *NonPersonalized) Push(item data.Item, feedback []data.Feedback) { // Skip hidden items if item.IsHidden { return @@ -141,7 +149,15 @@ func (l *LeaderBoard) Push(item data.Item, feedback []data.Feedback) { }, score) } -func (l *LeaderBoard) PopAll() []cache.Score { +func (l *NonPersonalized) PopAll() []cache.Score { scores, _ := l.heap.PopAll() return scores } + +func (l *NonPersonalized) Name() string { + return l.name +} + +func (l *NonPersonalized) Timestamp() time.Time { + return l.timestamp +} diff --git a/logics/leaderboard_test.go b/logics/non_personalized_test.go similarity index 53% rename from logics/leaderboard_test.go rename to logics/non_personalized_test.go index c32fd7e4a..6583b60fa 100644 --- a/logics/leaderboard_test.go +++ b/logics/non_personalized_test.go @@ -41,13 +41,10 @@ func TestLatest(t *testing.T) { func TestPopular(t *testing.T) { timestamp := time.Now() - popular := NewPopular(10, timestamp) + popular := NewPopular(0, 10, timestamp) for i := 0; i < 100; i++ { item := data.Item{ItemId: strconv.Itoa(i)} - var feedback []data.Feedback - for j := 0; j < i; j++ { - feedback = append(feedback, data.Feedback{FeedbackKey: data.FeedbackKey{UserId: strconv.Itoa(j)}, Timestamp: timestamp}) - } + feedback := make([]data.Feedback, i) popular.Push(item, feedback) } scores := popular.PopAll() @@ -58,9 +55,37 @@ func TestPopular(t *testing.T) { } } +func TestPopularWindow(t *testing.T) { + // Create popular recommender + timestamp := time.Now() + popular := NewPopular(time.Hour, 10, timestamp) + + // Add items + for i := 0; i < 100; i++ { + item := data.Item{ItemId: strconv.Itoa(i), Timestamp: timestamp.Add(time.Second - time.Hour)} + feedback := make([]data.Feedback, i) + popular.Push(item, feedback) + } + + // Add outdated items + for i := 100; i < 110; i++ { + item := data.Item{ItemId: strconv.Itoa(i), Timestamp: timestamp.Add(-time.Hour)} + feedback := make([]data.Feedback, i) + popular.Push(item, feedback) + } + + // Check result + scores := popular.PopAll() + assert.Len(t, scores, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, strconv.Itoa(99-i), scores[i].Id) + assert.Equal(t, float64(99-i), scores[i].Score) + } +} + func TestFilter(t *testing.T) { timestamp := time.Now() - latest, err := NewLeaderBoard(config.NonPersonalizedConfig{ + latest, err := NewNonPersonalized(config.NonPersonalizedConfig{ Name: "latest", Score: "item.Timestamp.Unix()", Filter: "!item.IsHidden", @@ -96,3 +121,64 @@ func TestHidden(t *testing.T) { assert.Equal(t, timestamp, scores[i].Timestamp) } } + +func TestMostStarredWeekly(t *testing.T) { + // Create non-personalized recommender + timestamp := time.Now() + mostStarredWeekly, err := NewNonPersonalized(config.NonPersonalizedConfig{ + Name: "most_starred_weekly", + Score: "count(feedback, .FeedbackType == 'star')", + Filter: "(now() - item.Timestamp).Hours() < 168", + }, 10, timestamp) + assert.NoError(t, err) + + // Add items + for i := 0; i < 100; i++ { + item := data.Item{ItemId: strconv.Itoa(i), Timestamp: timestamp.Add(-167 * time.Hour)} + var feedback []data.Feedback + for j := 0; j < i; j++ { + feedback = append(feedback, data.Feedback{ + FeedbackKey: data.FeedbackKey{ + FeedbackType: "star", + UserId: strconv.Itoa(j), + ItemId: strconv.Itoa(i), + }, + Timestamp: timestamp, + }) + feedback = append(feedback, data.Feedback{ + FeedbackKey: data.FeedbackKey{ + FeedbackType: "like", + UserId: strconv.Itoa(j), + ItemId: strconv.Itoa(i), + }, + Timestamp: timestamp, + }) + } + mostStarredWeekly.Push(item, feedback) + } + + // Add outdated items + for i := 100; i < 110; i++ { + item := data.Item{ItemId: strconv.Itoa(i), Timestamp: timestamp.Add(-168 * time.Hour)} + var feedback []data.Feedback + for j := 0; j < i; j++ { + feedback = append(feedback, data.Feedback{ + FeedbackKey: data.FeedbackKey{ + FeedbackType: "star", + UserId: strconv.Itoa(j), + ItemId: strconv.Itoa(i), + }, + Timestamp: timestamp.Add(-time.Hour * 169), + }) + } + mostStarredWeekly.Push(item, feedback) + } + + // Check result + scores := mostStarredWeekly.PopAll() + assert.Len(t, scores, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, strconv.Itoa(99-i), scores[i].Id) + assert.Equal(t, float64(99-i), scores[i].Score) + } +} From b48e7b88c3dad7701725584bec590c10f97d77ed Mon Sep 17 00:00:00 2001 From: Zhenghao Zhang Date: Sun, 8 Dec 2024 10:09:25 +0800 Subject: [PATCH 12/19] remove leaderboard to non-personalized --- master/rest.go | 6 +-- master/tasks.go | 114 ++++++++++++++++--------------------------- master/tasks_test.go | 18 ++++--- 3 files changed, 57 insertions(+), 81 deletions(-) diff --git a/master/rest.go b/master/rest.go index dc00389c4..25bc46c4c 100644 --- a/master/rest.go +++ b/master/rest.go @@ -1360,11 +1360,11 @@ func writeError(response http.ResponseWriter, httpStatus int, message string) { } } -func (s *Master) checkAdmin(request *http.Request) bool { - if s.Config.Master.AdminAPIKey == "" { +func (m *Master) checkAdmin(request *http.Request) bool { + if m.Config.Master.AdminAPIKey == "" { return true } - if request.FormValue("X-API-Key") == s.Config.Master.AdminAPIKey { + if request.FormValue("X-API-Key") == m.Config.Master.AdminAPIKey { return true } return false diff --git a/master/tasks.go b/master/tasks.go index c14824853..16b46c054 100644 --- a/master/tasks.go +++ b/master/tasks.go @@ -17,6 +17,7 @@ package master import ( "context" "fmt" + "github.com/zhenghaoz/gorse/logics" "sort" "strings" "sync" @@ -49,7 +50,6 @@ import ( const ( PositiveFeedbackRate = "PositiveFeedbackRate" - TaskLoadDataset = "Load dataset" TaskFindItemNeighbors = "Find neighbors of items" TaskFindUserNeighbors = "Find neighbors of users" TaskFitRankingModel = "Fit collaborative filtering model" @@ -73,45 +73,49 @@ func (m *Master) runLoadDatasetTask() error { ctx, span := m.tracer.Start(context.Background(), "Load Dataset", 1) defer span.End() + // Build non-personalized recommenders initialStartTime := time.Now() + nonPersonalizedRecommenders := []*logics.NonPersonalized{ + logics.NewLatest(m.Config.Recommend.CacheSize, initialStartTime), + logics.NewPopular(m.Config.Recommend.Popular.PopularWindow, m.Config.Recommend.CacheSize, initialStartTime), + } + for _, cfg := range m.Config.Recommend.NonPersonalized { + recommender, err := logics.NewNonPersonalized(cfg, m.Config.Recommend.CacheSize, initialStartTime) + if err != nil { + return errors.Trace(err) + } + nonPersonalizedRecommenders = append(nonPersonalizedRecommenders, recommender) + } + log.Logger().Info("load dataset", zap.Strings("positive_feedback_types", m.Config.Recommend.DataSource.PositiveFeedbackTypes), zap.Strings("read_feedback_types", m.Config.Recommend.DataSource.ReadFeedbackTypes), zap.Uint("item_ttl", m.Config.Recommend.DataSource.ItemTTL), zap.Uint("feedback_ttl", m.Config.Recommend.DataSource.PositiveFeedbackTTL)) evaluator := NewOnlineEvaluator() - rankingDataset, clickDataset, latestItems, popularItems, err := m.LoadDataFromDatabase(ctx, m.DataClient, + rankingDataset, clickDataset, err := m.LoadDataFromDatabase(ctx, m.DataClient, m.Config.Recommend.DataSource.PositiveFeedbackTypes, m.Config.Recommend.DataSource.ReadFeedbackTypes, m.Config.Recommend.DataSource.ItemTTL, m.Config.Recommend.DataSource.PositiveFeedbackTTL, - evaluator) + evaluator, nonPersonalizedRecommenders) if err != nil { return errors.Trace(err) } - // save popular items to cache - if err = m.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, popularItems.ToSlice()); err != nil { - log.Logger().Error("failed to cache popular items", zap.Error(err)) - } - if err = m.CacheClient.DeleteScores(ctx, []string{cache.NonPersonalized}, - cache.ScoreCondition{Subset: proto.String(cache.Popular), Before: &popularItems.Timestamp}); err != nil { - log.Logger().Error("failed to reclaim outdated items", zap.Error(err)) - } - if err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdatePopularItemsTime), time.Now())); err != nil { - log.Logger().Error("failed to write latest update popular items time", zap.Error(err)) - } - - // save the latest items to cache - if err = m.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, latestItems.ToSlice()); err != nil { - log.Logger().Error("failed to cache latest items", zap.Error(err)) - } - if err = m.CacheClient.DeleteScores(ctx, []string{cache.NonPersonalized}, - cache.ScoreCondition{Subset: proto.String(cache.Latest), Before: &latestItems.Timestamp}); err != nil { - log.Logger().Error("failed to reclaim outdated items", zap.Error(err)) - } - if err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateLatestItemsTime), time.Now())); err != nil { - log.Logger().Error("failed to write latest update latest items time", zap.Error(err)) + // save non-personalized recommenders to cache + for _, recommender := range nonPersonalizedRecommenders { + scores := recommender.PopAll() + if err = m.CacheClient.AddScores(ctx, cache.NonPersonalized, recommender.Name(), scores); err != nil { + log.Logger().Error("failed to cache non-personalized recommenders", zap.Error(err)) + } + if err = m.CacheClient.DeleteScores(ctx, []string{cache.NonPersonalized}, + cache.ScoreCondition{ + Subset: proto.String(recommender.Name()), + Before: lo.ToPtr(recommender.Timestamp()), + }); err != nil { + log.Logger().Error("failed to reclaim outdated items", zap.Error(err)) + } } // write statistics to database @@ -1396,12 +1400,17 @@ func (t *CacheGarbageCollectionTask) run(ctx context.Context, j *task.JobsAlloca } // LoadDataFromDatabase loads dataset from data store. -func (m *Master) LoadDataFromDatabase(ctx context.Context, database data.Database, posFeedbackTypes, readTypes []string, itemTTL, positiveFeedbackTTL uint, evaluator *OnlineEvaluator) ( - rankingDataset *ranking.DataSet, clickDataset *click.Dataset, latestItems *cache.DocumentAggregator, popularItems *cache.DocumentAggregator, err error) { +func (m *Master) LoadDataFromDatabase( + ctx context.Context, + database data.Database, + posFeedbackTypes, readTypes []string, + itemTTL, positiveFeedbackTTL uint, + evaluator *OnlineEvaluator, + nonPersonalizedRecommenders []*logics.NonPersonalized, +) (rankingDataset *ranking.DataSet, clickDataset *click.Dataset, err error) { newCtx, span := progress.Start(ctx, "LoadDataFromDatabase", 4) defer span.End() - startLoadTime := time.Now() // setup time limit var feedbackTimeLimit data.ScanOption var itemTimeLimit *time.Time @@ -1419,10 +1428,6 @@ func (m *Master) LoadDataFromDatabase(ctx context.Context, database data.Databas } rankingDataset = ranking.NewMapIndexDataset() - // create filers for latest items - latestItemsFilters := make(map[string]*heap.TopKFilter[string, float64]) - latestItemsFilters[""] = heap.NewTopKFilter[string, float64](m.Config.Recommend.CacheSize) - // STEP 1: pull users userLabelCount := make(map[string]int) userLabelFirst := make(map[string]int32) @@ -1465,7 +1470,7 @@ func (m *Master) LoadDataFromDatabase(ctx context.Context, database data.Databas } } if err = <-errChan; err != nil { - return nil, nil, nil, nil, errors.Trace(err) + return nil, nil, errors.Trace(err) } rankingDataset.NumUserLabels = userLabelIndex.Len() log.Logger().Debug("pulled users from database", @@ -1519,19 +1524,11 @@ func (m *Master) LoadDataFromDatabase(ctx context.Context, database data.Databas } if item.IsHidden { // set hidden flag rankingDataset.HiddenItems[itemIndex] = true - } else if !item.Timestamp.IsZero() { // add items to the latest items filter - latestItemsFilters[""].Push(item.ItemId, float64(item.Timestamp.Unix())) - for _, category := range item.Categories { - if _, exist := latestItemsFilters[category]; !exist { - latestItemsFilters[category] = heap.NewTopKFilter[string, float64](m.Config.Recommend.CacheSize) - } - latestItemsFilters[category].Push(item.ItemId, float64(item.Timestamp.Unix())) - } } } } if err = <-errChan; err != nil { - return nil, nil, nil, nil, errors.Trace(err) + return nil, nil, errors.Trace(err) } rankingDataset.NumItemLabels = itemLabelIndex.Len() log.Logger().Debug("pulled items from database", @@ -1597,7 +1594,7 @@ func (m *Master) LoadDataFromDatabase(ctx context.Context, database data.Databas return nil }) if err != nil { - return nil, nil, nil, nil, errors.Trace(err) + return nil, nil, errors.Trace(err) } log.Logger().Debug("pulled positive feedback from database", zap.Int("n_positive_feedback", posFeedbackCount), @@ -1647,7 +1644,7 @@ func (m *Master) LoadDataFromDatabase(ctx context.Context, database data.Databas return nil }) if err != nil { - return nil, nil, nil, nil, errors.Trace(err) + return nil, nil, errors.Trace(err) } log.Logger().Debug("pulled negative feedback from database", zap.Int("n_negative_feedback", int(negativeFeedbackCount)), @@ -1697,32 +1694,5 @@ func (m *Master) LoadDataFromDatabase(ctx context.Context, database data.Databas zap.Int("n_valid_negative", clickDataset.NegativeCount), zap.Duration("used_time", time.Since(start))) LoadDatasetStepSecondsVec.WithLabelValues("create_ranking_dataset").Set(time.Since(start).Seconds()) - - // collect latest items - latestItems = cache.NewDocumentAggregator(startLoadTime) - for category, latestItemsFilter := range latestItemsFilters { - items, scores := latestItemsFilter.PopAll() - latestItems.Add(category, items, scores) - } - - // collect popular items - popularItemFilters := make(map[string]*heap.TopKFilter[string, float64]) - popularItemFilters[""] = heap.NewTopKFilter[string, float64](m.Config.Recommend.CacheSize) - for itemIndex, val := range popularCount { - itemId := rankingDataset.ItemIndex.ToName(int32(itemIndex)) - popularItemFilters[""].Push(itemId, float64(val)) - for _, category := range rankingDataset.ItemCategories[itemIndex] { - if _, exist := popularItemFilters[category]; !exist { - popularItemFilters[category] = heap.NewTopKFilter[string, float64](m.Config.Recommend.CacheSize) - } - popularItemFilters[category].Push(itemId, float64(val)) - } - } - popularItems = cache.NewDocumentAggregator(startLoadTime) - for category, popularItemFilter := range popularItemFilters { - items, scores := popularItemFilter.PopAll() - popularItems.Add(category, items, scores) - } - - return rankingDataset, clickDataset, latestItems, popularItems, nil + return rankingDataset, clickDataset, nil } diff --git a/master/tasks_test.go b/master/tasks_test.go index 3e5143858..d7a168fbe 100644 --- a/master/tasks_test.go +++ b/master/tasks_test.go @@ -81,7 +81,8 @@ func (s *MasterTestSuite) TestFindItemNeighborsBruteForce() { } // load mock dataset - dataset, _, _, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) + dataset, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, + nil, 0, 0, NewOnlineEvaluator(), nil) s.NoError(err) s.rankingTrainSet = dataset @@ -186,7 +187,8 @@ func (s *MasterTestSuite) TestFindItemNeighborsIVF() { } // load mock dataset - dataset, _, _, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) + dataset, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, + nil, 0, 0, NewOnlineEvaluator(), nil) s.NoError(err) s.rankingTrainSet = dataset @@ -253,7 +255,8 @@ func (s *MasterTestSuite) TestFindItemNeighborsIVF_ZeroIDF() { {FeedbackKey: data.FeedbackKey{FeedbackType: "FeedbackType", UserId: "0", ItemId: "1"}}, }, true, true, true) s.NoError(err) - dataset, _, _, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) + dataset, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, + nil, 0, 0, NewOnlineEvaluator(), nil) s.NoError(err) s.rankingTrainSet = dataset @@ -313,7 +316,8 @@ func (s *MasterTestSuite) TestFindUserNeighborsBruteForce() { s.NoError(err) err = s.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) s.NoError(err) - dataset, _, _, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) + dataset, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, + nil, 0, 0, NewOnlineEvaluator(), nil) s.NoError(err) s.rankingTrainSet = dataset @@ -393,7 +397,8 @@ func (s *MasterTestSuite) TestFindUserNeighborsIVF() { s.NoError(err) err = s.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) s.NoError(err) - dataset, _, _, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) + dataset, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, + nil, 0, 0, NewOnlineEvaluator(), nil) s.NoError(err) s.rankingTrainSet = dataset @@ -452,7 +457,8 @@ func (s *MasterTestSuite) TestFindUserNeighborsIVF_ZeroIDF() { {FeedbackKey: data.FeedbackKey{FeedbackType: "FeedbackType", UserId: "1", ItemId: "0"}}, }, true, true, true) s.NoError(err) - dataset, _, _, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) + dataset, _, err := s.LoadDataFromDatabase(context.Background(), s.DataClient, []string{"FeedbackType"}, + nil, 0, 0, NewOnlineEvaluator(), nil) s.NoError(err) s.rankingTrainSet = dataset From 4bbb6c0ca27675cafc60097b748438f1545c6dfb Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Mon, 16 Dec 2024 20:27:54 +0800 Subject: [PATCH 13/19] support with begin/end item id --- protocol/data_store.pb.go | 59 ++++++++++++++++++++++++----------- protocol/data_store.proto | 8 +++-- storage/data/database.go | 16 ++++++++++ storage/data/database_test.go | 2 ++ storage/data/mongodb.go | 10 ++++++ storage/data/proxy.go | 8 +++++ storage/data/sql.go | 6 ++++ worker/worker.go | 8 ++--- worker/worker_test.go | 14 ++++----- 9 files changed, 99 insertions(+), 32 deletions(-) diff --git a/protocol/data_store.pb.go b/protocol/data_store.pb.go index 726e9aea5..906291f77 100644 --- a/protocol/data_store.pb.go +++ b/protocol/data_store.pb.go @@ -14,8 +14,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.2 -// protoc v5.29.0 +// protoc-gen-go v1.35.1 +// protoc v5.28.3 // source: data_store.proto package protocol @@ -180,9 +180,11 @@ type ScanOptions struct { BeginUserId *string `protobuf:"bytes,1,opt,name=begin_user_id,json=beginUserId,proto3,oneof" json:"begin_user_id,omitempty"` EndUserId *string `protobuf:"bytes,2,opt,name=end_user_id,json=endUserId,proto3,oneof" json:"end_user_id,omitempty"` - BeginTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=begin_time,json=beginTime,proto3,oneof" json:"begin_time,omitempty"` - EndTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=end_time,json=endTime,proto3,oneof" json:"end_time,omitempty"` - FeedbackTypes []string `protobuf:"bytes,5,rep,name=feedback_types,json=feedbackTypes,proto3" json:"feedback_types,omitempty"` + BeginItemId *string `protobuf:"bytes,3,opt,name=begin_item_id,json=beginItemId,proto3,oneof" json:"begin_item_id,omitempty"` + EndItemId *string `protobuf:"bytes,4,opt,name=end_item_id,json=endItemId,proto3,oneof" json:"end_item_id,omitempty"` + BeginTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=begin_time,json=beginTime,proto3,oneof" json:"begin_time,omitempty"` + EndTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=end_time,json=endTime,proto3,oneof" json:"end_time,omitempty"` + FeedbackTypes []string `protobuf:"bytes,7,rep,name=feedback_types,json=feedbackTypes,proto3" json:"feedback_types,omitempty"` } func (x *ScanOptions) Reset() { @@ -229,6 +231,20 @@ func (x *ScanOptions) GetEndUserId() string { return "" } +func (x *ScanOptions) GetBeginItemId() string { + if x != nil && x.BeginItemId != nil { + return *x.BeginItemId + } + return "" +} + +func (x *ScanOptions) GetEndItemId() string { + if x != nil && x.EndItemId != nil { + return *x.EndItemId + } + return "" +} + func (x *ScanOptions) GetBeginTime() *timestamppb.Timestamp { if x != nil { return x.BeginTime @@ -2073,25 +2089,32 @@ var file_data_store_proto_rawDesc = []byte{ 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x69, 0x73, 0x5f, 0x68, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x0a, 0x0a, - 0x08, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0xbc, 0x02, 0x0a, 0x0b, 0x53, 0x63, + 0x08, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0xac, 0x03, 0x0a, 0x0b, 0x53, 0x63, 0x61, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x55, 0x73, 0x65, 0x72, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x23, 0x0a, 0x0b, 0x65, 0x6e, 0x64, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x09, 0x65, 0x6e, 0x64, 0x55, 0x73, - 0x65, 0x72, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x3e, 0x0a, 0x0a, 0x62, 0x65, 0x67, 0x69, 0x6e, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x02, 0x52, 0x09, 0x62, 0x65, 0x67, 0x69, 0x6e, - 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x65, 0x72, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x65, 0x67, 0x69, 0x6e, + 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, + 0x52, 0x0b, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x88, 0x01, 0x01, + 0x12, 0x23, 0x0a, 0x0b, 0x65, 0x6e, 0x64, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x03, 0x52, 0x09, 0x65, 0x6e, 0x64, 0x49, 0x74, 0x65, 0x6d, + 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x3e, 0x0a, 0x0a, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, - 0x88, 0x01, 0x01, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, - 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, - 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, - 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x0d, 0x0a, 0x0b, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x04, 0x52, 0x09, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x54, 0x69, + 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x48, 0x05, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, + 0x01, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, + 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, 0x65, 0x67, + 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x65, + 0x6e, 0x64, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, + 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, + 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x3f, 0x0a, 0x17, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, diff --git a/protocol/data_store.proto b/protocol/data_store.proto index 22a73d8af..19f8829dd 100644 --- a/protocol/data_store.proto +++ b/protocol/data_store.proto @@ -38,9 +38,11 @@ message ItemPatch { message ScanOptions { optional string begin_user_id = 1; optional string end_user_id = 2; - optional google.protobuf.Timestamp begin_time = 3; - optional google.protobuf.Timestamp end_time = 4; - repeated string feedback_types = 5; + optional string begin_item_id = 3; + optional string end_item_id = 4; + optional google.protobuf.Timestamp begin_time = 5; + optional google.protobuf.Timestamp end_time = 6; + repeated string feedback_types = 7; } message BatchInsertItemsRequest { diff --git a/storage/data/database.go b/storage/data/database.go index 911ef3bfd..544227eca 100644 --- a/storage/data/database.go +++ b/storage/data/database.go @@ -174,6 +174,8 @@ func (sorter feedbackSorter) Swap(i, j int) { type ScanOptions struct { BeginUserId *string EndUserId *string + BeginItemId *string + EndItemId *string BeginTime *time.Time EndTime *time.Time FeedbackTypes []string @@ -195,6 +197,20 @@ func WithEndUserId(userId string) ScanOption { } } +// WithBeginItemId sets the beginning item id. The beginning item id is included in the result. +func WithBeginItemId(itemId string) ScanOption { + return func(options *ScanOptions) { + options.BeginItemId = &itemId + } +} + +// WithEndItemId sets the end item id. The end item id is included in the result. +func WithEndItemId(itemId string) ScanOption { + return func(options *ScanOptions) { + options.EndItemId = &itemId + } +} + // WithBeginTime sets the begin time. The begin time is included in the result. func WithBeginTime(t time.Time) ScanOption { return func(options *ScanOptions) { diff --git a/storage/data/database_test.go b/storage/data/database_test.go index ac9fcad1a..5d2a8d5d1 100644 --- a/storage/data/database_test.go +++ b/storage/data/database_test.go @@ -271,6 +271,8 @@ func (suite *baseTestSuite) TestFeedback() { suite.Empty(feedbackFromStream) feedbackFromStream = suite.getFeedbackStream(ctx, 3, WithBeginUserId("1"), WithEndUserId("3"), WithEndTime(time.Now()), WithFeedbackTypes(positiveFeedbackType)) suite.Equal(feedback[1:4], feedbackFromStream) + feedbackFromStream = suite.getFeedbackStream(ctx, 3, WithBeginItemId("2"), WithEndItemId("6"), WithEndTime(time.Now()), WithFeedbackTypes(positiveFeedbackType)) + suite.Equal(feedback[1:4], feedbackFromStream) // Get items err = suite.Database.Optimize() suite.NoError(err) diff --git a/storage/data/mongodb.go b/storage/data/mongodb.go index 3848e6b0a..1f953cb73 100644 --- a/storage/data/mongodb.go +++ b/storage/data/mongodb.go @@ -726,6 +726,16 @@ func (db *MongoDB) GetFeedbackStream(ctx context.Context, batchSize int, scanOpt } filter["feedbackkey.userid"] = userIdConditions } + if scan.BeginItemId != nil || scan.EndItemId != nil { + itemIdConditions := bson.M{} + if scan.BeginItemId != nil { + itemIdConditions["$gte"] = *scan.BeginItemId + } + if scan.EndItemId != nil { + itemIdConditions["$lte"] = *scan.EndItemId + } + filter["feedbackkey.itemid"] = itemIdConditions + } r, err := c.Find(ctx, filter, opt) if err != nil { diff --git a/storage/data/proxy.go b/storage/data/proxy.go index 12eaf29c0..38989281c 100644 --- a/storage/data/proxy.go +++ b/storage/data/proxy.go @@ -432,6 +432,12 @@ func (p *ProxyServer) GetFeedbackStream(in *protocol.GetFeedbackStreamRequest, s if in.ScanOptions.EndUserId != nil { opts = append(opts, WithEndUserId(*in.ScanOptions.EndUserId)) } + if in.ScanOptions.BeginItemId != nil { + opts = append(opts, WithBeginItemId(*in.ScanOptions.BeginItemId)) + } + if in.ScanOptions.EndItemId != nil { + opts = append(opts, WithEndItemId(*in.ScanOptions.EndItemId)) + } feedbackChan, errChan := p.database.GetFeedbackStream(stream.Context(), int(in.BatchSize), opts...) for feedback := range feedbackChan { pbFeedback := make([]*protocol.Feedback, len(feedback)) @@ -930,6 +936,8 @@ func (p ProxyClient) GetFeedbackStream(ctx context.Context, batchSize int, optio pbOptions := &protocol.ScanOptions{ BeginUserId: o.BeginUserId, EndUserId: o.EndUserId, + BeginItemId: o.BeginItemId, + EndItemId: o.EndItemId, FeedbackTypes: o.FeedbackTypes, } if o.BeginTime != nil { diff --git a/storage/data/sql.go b/storage/data/sql.go index caf5a85fc..9b27a99fa 100644 --- a/storage/data/sql.go +++ b/storage/data/sql.go @@ -976,6 +976,12 @@ func (d *SQLDatabase) GetFeedbackStream(ctx context.Context, batchSize int, scan if scan.EndUserId != nil { tx.Where("user_id <= ?", scan.EndUserId) } + if scan.BeginItemId != nil { + tx.Where("item_id >= ?", scan.BeginItemId) + } + if scan.EndItemId != nil { + tx.Where("item_id <= ?", scan.EndItemId) + } result, err := tx.Rows() if err != nil { errChan <- errors.Trace(err) diff --git a/worker/worker.go b/worker/worker.go index 899336896..d5554242e 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -736,7 +736,7 @@ func (w *Worker) Recommend(users []data.User) { if w.Config.Recommend.Offline.EnableLatestRecommend { localStartTime := time.Now() for _, category := range append([]string{""}, itemCategories...) { - latestItems, err := w.CacheClient.SearchScores(ctx, cache.LatestItems, "", []string{category}, 0, w.Config.Recommend.CacheSize) + latestItems, err := w.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Latest, []string{category}, 0, w.Config.Recommend.CacheSize) if err != nil { log.Logger().Error("failed to load latest items", zap.Error(err)) return errors.Trace(err) @@ -756,7 +756,7 @@ func (w *Worker) Recommend(users []data.User) { if w.Config.Recommend.Offline.EnablePopularRecommend { localStartTime := time.Now() for _, category := range append([]string{""}, itemCategories...) { - popularItems, err := w.CacheClient.SearchScores(ctx, cache.PopularItems, "", []string{category}, 0, w.Config.Recommend.CacheSize) + popularItems, err := w.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Popular, []string{category}, 0, w.Config.Recommend.CacheSize) if err != nil { log.Logger().Error("failed to load popular items", zap.Error(err)) return errors.Trace(err) @@ -1045,12 +1045,12 @@ func (w *Worker) exploreRecommend(exploitRecommend []cache.Score, excludeSet map exploreLatestThreshold += threshold } // load popular items - popularItems, err := w.CacheClient.SearchScores(ctx, cache.PopularItems, "", []string{category}, 0, w.Config.Recommend.CacheSize) + popularItems, err := w.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Popular, []string{category}, 0, w.Config.Recommend.CacheSize) if err != nil { return nil, errors.Trace(err) } // load the latest items - latestItems, err := w.CacheClient.SearchScores(ctx, cache.LatestItems, "", []string{category}, 0, w.Config.Recommend.CacheSize) + latestItems, err := w.CacheClient.SearchScores(ctx, cache.NonPersonalized, cache.Latest, []string{category}, 0, w.Config.Recommend.CacheSize) if err != nil { return nil, errors.Trace(err) } diff --git a/worker/worker_test.go b/worker/worker_test.go index 5a2babac0..0bcbc322f 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -442,7 +442,7 @@ func (suite *WorkerTestSuite) TestRecommendPopular() { suite.Config.Recommend.Offline.EnableColRecommend = false suite.Config.Recommend.Offline.EnablePopularRecommend = true // insert popular items - err := suite.CacheClient.AddScores(ctx, cache.PopularItems, "", []cache.Score{ + err := suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, []cache.Score{ {Id: "11", Score: 11, Categories: []string{""}}, {Id: "10", Score: 10, Categories: []string{""}}, {Id: "9", Score: 9, Categories: []string{""}}, @@ -491,7 +491,7 @@ func (suite *WorkerTestSuite) TestRecommendLatest() { suite.Config.Recommend.Offline.EnableColRecommend = false suite.Config.Recommend.Offline.EnableLatestRecommend = true // insert latest items - err := suite.CacheClient.AddScores(ctx, cache.LatestItems, "", []cache.Score{ + err := suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, []cache.Score{ {Id: "11", Score: 11, Categories: []string{""}}, {Id: "10", Score: 10, Categories: []string{""}}, {Id: "9", Score: 9, Categories: []string{""}}, @@ -539,7 +539,7 @@ func (suite *WorkerTestSuite) TestRecommendColdStart() { suite.Config.Recommend.Offline.EnableColRecommend = true suite.Config.Recommend.Offline.EnableLatestRecommend = true // insert latest items - err := suite.CacheClient.AddScores(ctx, cache.LatestItems, "", []cache.Score{ + err := suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, []cache.Score{ {Id: "11", Score: 11, Categories: []string{""}}, {Id: "10", Score: 10, Categories: []string{""}}, {Id: "9", Score: 9, Categories: []string{""}}, @@ -591,10 +591,10 @@ func (suite *WorkerTestSuite) TestExploreRecommend() { ctx := context.Background() suite.Config.Recommend.Offline.ExploreRecommend = map[string]float64{"popular": 0.3, "latest": 0.3} // insert popular items - err := suite.CacheClient.AddScores(ctx, cache.PopularItems, "", []cache.Score{{Id: "popular", Score: 0, Categories: []string{""}, Timestamp: time.Now()}}) + err := suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, []cache.Score{{Id: "popular", Score: 0, Categories: []string{""}, Timestamp: time.Now()}}) suite.NoError(err) // insert latest items - err = suite.CacheClient.AddScores(ctx, cache.LatestItems, "", []cache.Score{{Id: "latest", Score: 0, Categories: []string{""}, Timestamp: time.Now()}}) + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Latest, []cache.Score{{Id: "latest", Score: 0, Categories: []string{""}, Timestamp: time.Now()}}) suite.NoError(err) recommend, err := suite.exploreRecommend([]cache.Score{ @@ -940,7 +940,7 @@ func (suite *WorkerTestSuite) TestReplacement_ClickThroughRate() { err = suite.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().AddDate(-1, 0, 0))) suite.NoError(err) // insert popular items - err = suite.CacheClient.AddScores(ctx, cache.PopularItems, "", []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, []cache.Score{ {Id: "7", Score: 10, Categories: []string{""}}, {Id: "6", Score: 9, Categories: []string{""}}, {Id: "5", Score: 8, Categories: []string{""}}, @@ -1005,7 +1005,7 @@ func (suite *WorkerTestSuite) TestReplacement_CollaborativeFiltering() { err = suite.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().AddDate(-1, 0, 0))) suite.NoError(err) // insert popular items - err = suite.CacheClient.AddScores(ctx, cache.PopularItems, "", []cache.Score{ + err = suite.CacheClient.AddScores(ctx, cache.NonPersonalized, cache.Popular, []cache.Score{ {Id: "7", Score: 10, Categories: []string{""}}, {Id: "6", Score: 9, Categories: []string{""}}, {Id: "5", Score: 8, Categories: []string{""}}}) From c44162a7161875fbbcdc69220906aed6650918ae Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Mon, 16 Dec 2024 21:26:01 +0800 Subject: [PATCH 14/19] Fix test --- logics/non_personalized.go | 50 +++++++++++++++++++++++++++++--------- master/tasks.go | 9 +++++++ master/tasks_test.go | 3 ++- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/logics/non_personalized.go b/logics/non_personalized.go index 5a19a4ffd..38a7ed947 100644 --- a/logics/non_personalized.go +++ b/logics/non_personalized.go @@ -27,6 +27,7 @@ import ( "github.com/zhenghaoz/gorse/storage/data" "go.uber.org/zap" "reflect" + "sort" "time" ) @@ -35,7 +36,8 @@ type NonPersonalized struct { timestamp time.Time scoreFunc *vm.Program filterFunc *vm.Program - heap *heap.TopKFilter[cache.Score, float64] + heapSize int + heaps map[string]*heap.TopKFilter[string, float64] } func NewNonPersonalized(cfg config.NonPersonalizedConfig, n int, timestamp time.Time) (*NonPersonalized, error) { @@ -66,12 +68,16 @@ func NewNonPersonalized(cfg config.NonPersonalizedConfig, n int, timestamp time. return nil, errors.New("filter function must return bool") } } + // Initialize heap + heaps := make(map[string]*heap.TopKFilter[string, float64]) + heaps[""] = heap.NewTopKFilter[string, float64](n) return &NonPersonalized{ name: cfg.Name, timestamp: timestamp, scoreFunc: scoreFunc, filterFunc: filterFunc, - heap: heap.NewTopKFilter[cache.Score, float64](n), + heapSize: n, + heaps: heaps, }, nil } @@ -140,18 +146,40 @@ func (l *NonPersonalized) Push(item data.Item, feedback []data.Feedback) { log.Logger().Error("score function must return float64", zap.Any("result", result)) return } - l.heap.Push(cache.Score{ - Id: item.ItemId, - Score: score, - IsHidden: item.IsHidden, - Categories: item.Categories, - Timestamp: l.timestamp, - }, score) + // Add to heap + l.heaps[""].Push(item.ItemId, score) + for _, group := range item.Categories { + if _, exist := l.heaps[group]; !exist { + l.heaps[group] = heap.NewTopKFilter[string, float64](l.heapSize) + } + l.heaps[group].Push(item.ItemId, score) + } } func (l *NonPersonalized) PopAll() []cache.Score { - scores, _ := l.heap.PopAll() - return scores + scores := make(map[string]*cache.Score) + for category, h := range l.heaps { + names, values := h.PopAll() + for i, name := range names { + if _, exist := scores[name]; !exist { + scores[name] = &cache.Score{ + Id: name, + Score: values[i], + Categories: []string{category}, + Timestamp: l.timestamp, + } + } else { + scores[name].Categories = append(scores[name].Categories, category) + } + } + } + result := lo.MapToSlice(scores, func(_ string, v *cache.Score) cache.Score { + return *v + }) + sort.Slice(result, func(i, j int) bool { + return result[i].Score > result[j].Score + }) + return result } func (l *NonPersonalized) Name() string { diff --git a/master/tasks.go b/master/tasks.go index 16b46c054..696338cc4 100644 --- a/master/tasks.go +++ b/master/tasks.go @@ -1525,6 +1525,15 @@ func (m *Master) LoadDataFromDatabase( if item.IsHidden { // set hidden flag rankingDataset.HiddenItems[itemIndex] = true } + // TODO: Refactor + // add item to non-personalized recommenders + feedback, err := database.GetItemFeedback(newCtx, item.ItemId, posFeedbackTypes...) + if err != nil { + return nil, nil, errors.Trace(err) + } + for _, recommender := range nonPersonalizedRecommenders { + recommender.Push(item, feedback) + } } } if err = <-errChan; err != nil { diff --git a/master/tasks_test.go b/master/tasks_test.go index d7a168fbe..9e2da0c5c 100644 --- a/master/tasks_test.go +++ b/master/tasks_test.go @@ -532,7 +532,8 @@ func (s *MasterTestSuite) TestLoadDataFromDatabase() { UserId: strconv.Itoa(j), FeedbackType: "positive", }, - Timestamp: time.Now(), + // TODO: Refactor + Timestamp: time.Now().Add(-time.Second), }) } // negative feedback From c835aac96a28ab760d1f449991ec1d610c2e004c Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Mon, 16 Dec 2024 14:02:04 +0000 Subject: [PATCH 15/19] Fix tests --- storage/cache/mongodb.go | 4 ++-- storage/data/database_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/cache/mongodb.go b/storage/cache/mongodb.go index 82fa4c01e..85b321df8 100644 --- a/storage/cache/mongodb.go +++ b/storage/cache/mongodb.go @@ -381,7 +381,7 @@ func (m MongoDB) UpdateScores(ctx context.Context, collections []string, subset "id": id, } if subset != nil { - filter["subset"] = subset + filter["subset"] = *subset } update := bson.D{} if patch.IsHidden != nil { @@ -393,7 +393,7 @@ func (m MongoDB) UpdateScores(ctx context.Context, collections []string, subset if patch.Score != nil { update = append(update, bson.E{Key: "$set", Value: bson.M{"score": *patch.Score}}) } - _, err := m.client.Database(m.dbName).Collection(m.DocumentTable()).UpdateMany(ctx, subset, update) + _, err := m.client.Database(m.dbName).Collection(m.DocumentTable()).UpdateMany(ctx, filter, update) return errors.Trace(err) } diff --git a/storage/data/database_test.go b/storage/data/database_test.go index 5d2a8d5d1..592f42fa2 100644 --- a/storage/data/database_test.go +++ b/storage/data/database_test.go @@ -272,7 +272,7 @@ func (suite *baseTestSuite) TestFeedback() { feedbackFromStream = suite.getFeedbackStream(ctx, 3, WithBeginUserId("1"), WithEndUserId("3"), WithEndTime(time.Now()), WithFeedbackTypes(positiveFeedbackType)) suite.Equal(feedback[1:4], feedbackFromStream) feedbackFromStream = suite.getFeedbackStream(ctx, 3, WithBeginItemId("2"), WithEndItemId("6"), WithEndTime(time.Now()), WithFeedbackTypes(positiveFeedbackType)) - suite.Equal(feedback[1:4], feedbackFromStream) + suite.ElementsMatch(feedback[1:4], feedbackFromStream) // Get items err = suite.Database.Optimize() suite.NoError(err) From 2b5d4bb168f2a5da914ac1b002f21f9a8e11a5f1 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Mon, 16 Dec 2024 14:16:44 +0000 Subject: [PATCH 16/19] Fix tests --- storage/cache/sql.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/storage/cache/sql.go b/storage/cache/sql.go index 89880fd7c..d4b8ad76b 100644 --- a/storage/cache/sql.go +++ b/storage/cache/sql.go @@ -484,9 +484,12 @@ func (db *SQLDatabase) UpdateScores(ctx context.Context, collections []string, s if patch.Score == nil && patch.IsHidden == nil && patch.Categories == nil { return nil } - tx := db.gormDB.WithContext(ctx).Model(&PostgresDocument{}). - Where("collection in (?) and id = ? and (subset = ? or ? is null)", - collections, id, subset, subset) + tx := db.gormDB.WithContext(ctx).Model(&PostgresDocument{}) + if subset != nil { + tx = tx.Where("collection in (?) and id = ? and subset = ?", collections, id, subset) + } else { + tx = tx.Where("collection in (?) and id = ?", collections, id) + } if patch.Score != nil { tx = tx.Update("score", *patch.Score) } From cce588e5eb90e57d0e4ce86ae5692dd71d74b611 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Wed, 18 Dec 2024 21:25:12 +0800 Subject: [PATCH 17/19] proto: add order_by_item_id --- protocol/data_store.pb.go | 582 +++++++++++++++++++------------------- protocol/data_store.proto | 1 + 2 files changed, 297 insertions(+), 286 deletions(-) diff --git a/protocol/data_store.pb.go b/protocol/data_store.pb.go index 906291f77..aa0fba3d4 100644 --- a/protocol/data_store.pb.go +++ b/protocol/data_store.pb.go @@ -185,6 +185,7 @@ type ScanOptions struct { BeginTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=begin_time,json=beginTime,proto3,oneof" json:"begin_time,omitempty"` EndTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=end_time,json=endTime,proto3,oneof" json:"end_time,omitempty"` FeedbackTypes []string `protobuf:"bytes,7,rep,name=feedback_types,json=feedbackTypes,proto3" json:"feedback_types,omitempty"` + OrderByItemId bool `protobuf:"varint,8,opt,name=order_by_item_id,json=orderByItemId,proto3" json:"order_by_item_id,omitempty"` } func (x *ScanOptions) Reset() { @@ -266,6 +267,13 @@ func (x *ScanOptions) GetFeedbackTypes() []string { return nil } +func (x *ScanOptions) GetOrderByItemId() bool { + if x != nil { + return x.OrderByItemId + } + return false +} + type BatchInsertItemsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2089,7 +2097,7 @@ var file_data_store_proto_rawDesc = []byte{ 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x69, 0x73, 0x5f, 0x68, 0x69, 0x64, 0x64, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x0a, 0x0a, - 0x08, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0xac, 0x03, 0x0a, 0x0b, 0x53, 0x63, + 0x08, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0xd5, 0x03, 0x0a, 0x0b, 0x53, 0x63, 0x61, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x55, 0x73, 0x65, 0x72, 0x49, 0x64, 0x88, @@ -2110,297 +2118,299 @@ var file_data_store_proto_rawDesc = []byte{ 0x61, 0x6d, 0x70, 0x48, 0x05, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, - 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, 0x65, 0x67, - 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x65, - 0x6e, 0x64, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, - 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, - 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x42, 0x0d, 0x0a, 0x0b, - 0x5f, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, - 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x3f, 0x0a, 0x17, 0x42, 0x61, 0x74, 0x63, - 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, - 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x1a, 0x0a, 0x18, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x31, 0x0a, 0x14, 0x42, 0x61, 0x74, 0x63, 0x68, 0x47, 0x65, - 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, - 0x08, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x07, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x73, 0x22, 0x3d, 0x0a, 0x15, 0x42, 0x61, 0x74, 0x63, - 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, 0x6d, - 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x2c, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, - 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, - 0x74, 0x65, 0x6d, 0x49, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, - 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x0a, 0x0e, 0x47, - 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, - 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x22, 0x43, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, - 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x04, 0x69, 0x74, 0x65, - 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x48, 0x00, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x88, - 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x22, 0x57, 0x0a, 0x11, 0x4d, - 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x05, 0x70, 0x61, 0x74, - 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, 0x63, 0x68, 0x52, 0x05, 0x70, - 0x61, 0x74, 0x63, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, - 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x72, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, - 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x0c, 0x0a, 0x01, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x01, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x50, - 0x0a, 0x10, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, - 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x10, 0x6f, 0x72, 0x64, 0x65, + 0x72, 0x5f, 0x62, 0x79, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0d, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x49, + 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x75, 0x73, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x75, 0x73, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x69, 0x74, + 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x69, 0x74, + 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x22, 0x3f, 0x0a, 0x17, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, + 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, + 0x6d, 0x73, 0x22, 0x1a, 0x0a, 0x18, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, + 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x31, + 0x0a, 0x14, 0x42, 0x61, 0x74, 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, + 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, + 0x73, 0x22, 0x3d, 0x0a, 0x15, 0x42, 0x61, 0x74, 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, + 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, + 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, - 0x22, 0x58, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, - 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, - 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, - 0x6d, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, - 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x3f, 0x0a, 0x17, 0x42, 0x61, - 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x55, 0x73, 0x65, 0x72, 0x52, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x22, 0x1a, 0x0a, 0x18, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2c, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, - 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, - 0x73, 0x65, 0x72, 0x49, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, - 0x73, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x0a, 0x0e, 0x47, - 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, - 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x22, 0x43, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, - 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x04, 0x75, 0x73, 0x65, - 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x48, 0x00, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x88, - 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x22, 0x57, 0x0a, 0x11, 0x4d, - 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x05, 0x70, 0x61, 0x74, - 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x50, 0x61, 0x74, 0x63, 0x68, 0x52, 0x05, 0x70, - 0x61, 0x74, 0x63, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x37, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, - 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x0c, 0x0a, 0x01, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x01, 0x6e, 0x22, 0x50, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, - 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, - 0x24, 0x0a, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x52, 0x05, - 0x75, 0x73, 0x65, 0x72, 0x73, 0x22, 0x8f, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, - 0x72, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, - 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, - 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x75, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x55, 0x73, - 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, - 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, - 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x78, - 0x0a, 0x1d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, - 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, - 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, - 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x36, 0x0a, 0x1e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, - 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x22, 0xac, 0x01, 0x0a, 0x1a, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, - 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x2e, 0x0a, 0x08, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x65, 0x65, - 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x08, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, - 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, - 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, - 0x6d, 0x12, 0x1c, 0x0a, 0x09, 0x6f, 0x76, 0x65, 0x72, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6f, 0x76, 0x65, 0x72, 0x77, 0x72, 0x69, 0x74, 0x65, 0x22, - 0x1d, 0x0a, 0x1b, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x65, - 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xd3, - 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x0c, 0x0a, - 0x01, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x62, - 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x62, 0x65, 0x67, - 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x25, 0x0a, - 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, - 0x79, 0x70, 0x65, 0x73, 0x22, 0x5d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, - 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, + 0x22, 0x2c, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x22, 0x14, + 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x22, + 0x43, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x27, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, 0x6d, + 0x48, 0x00, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, + 0x69, 0x74, 0x65, 0x6d, 0x22, 0x57, 0x0a, 0x11, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, + 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, + 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, + 0x49, 0x64, 0x12, 0x29, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, + 0x6d, 0x50, 0x61, 0x74, 0x63, 0x68, 0x52, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x22, 0x14, 0x0a, + 0x12, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x72, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x0c, + 0x0a, 0x01, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x6e, 0x12, 0x39, 0x0a, 0x0a, + 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x62, 0x65, + 0x67, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x50, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x75, 0x72, - 0x73, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x08, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x2e, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x08, 0x66, 0x65, 0x65, 0x64, 0x62, - 0x61, 0x63, 0x6b, 0x22, 0x35, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, - 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x09, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x3d, 0x0a, 0x15, 0x47, 0x65, - 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x73, - 0x65, 0x72, 0x52, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x22, 0x70, 0x0a, 0x14, 0x47, 0x65, 0x74, - 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, 0x7a, 0x65, - 0x12, 0x39, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, + 0x73, 0x6f, 0x72, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, + 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x58, 0x0a, 0x16, 0x47, 0x65, 0x74, + 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, + 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x79, + 0x70, 0x65, 0x73, 0x22, 0x3f, 0x0a, 0x17, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, + 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, + 0x0a, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x52, 0x05, 0x75, + 0x73, 0x65, 0x72, 0x73, 0x22, 0x1a, 0x0a, 0x18, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, + 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x2c, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x22, 0x14, + 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x22, + 0x43, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x27, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, 0x72, + 0x48, 0x00, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, + 0x75, 0x73, 0x65, 0x72, 0x22, 0x57, 0x0a, 0x11, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, + 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x29, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, + 0x72, 0x50, 0x61, 0x74, 0x63, 0x68, 0x52, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x22, 0x14, 0x0a, + 0x12, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x37, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x0c, + 0x0a, 0x01, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x6e, 0x22, 0x50, 0x0a, 0x10, + 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x24, 0x0a, 0x05, 0x75, 0x73, 0x65, 0x72, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x52, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x22, 0x8f, + 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, + 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x3d, 0x0a, 0x15, 0x47, - 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, - 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x73, 0x0a, 0x18, 0x47, 0x65, - 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, - 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x74, 0x63, - 0x68, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x38, 0x0a, 0x0c, 0x73, 0x63, 0x61, 0x6e, 0x5f, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x53, 0x63, 0x61, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x52, 0x0b, 0x73, 0x63, 0x61, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, - 0x4b, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x08, - 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, - 0x63, 0x6b, 0x52, 0x08, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x32, 0xc7, 0x0d, 0x0a, - 0x09, 0x44, 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x37, 0x0a, 0x04, 0x50, 0x69, - 0x6e, 0x67, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x69, - 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x10, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, - 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, - 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x52, 0x0a, 0x0d, 0x42, 0x61, 0x74, 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, - 0x73, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, - 0x65, 0x6d, 0x12, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x40, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x12, - 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, - 0x79, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, - 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x08, - 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, - 0x62, 0x61, 0x63, 0x6b, 0x12, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x10, 0x42, 0x61, 0x74, 0x63, 0x68, - 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x12, 0x21, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, - 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, - 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, - 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x40, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, 0x65, 0x72, 0x12, - 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, - 0x79, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x08, - 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x12, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x46, 0x65, 0x65, 0x64, - 0x62, 0x61, 0x63, 0x6b, 0x12, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x55, 0x73, - 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x24, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, - 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x16, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, - 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, - 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, + 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x22, 0x75, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, + 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, + 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, + 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, + 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x78, 0x0a, 0x1d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, - 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, - 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x13, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, - 0x65, 0x72, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, - 0x72, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x47, 0x65, - 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x55, - 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x54, - 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, - 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, - 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, - 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x30, 0x01, 0x12, 0x60, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, - 0x61, 0x63, 0x6b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, + 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x69, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, + 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0d, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, + 0x73, 0x22, 0x36, 0x0a, 0x1e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, + 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xac, 0x01, 0x0a, 0x1a, 0x42, 0x61, + 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, + 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x08, 0x66, 0x65, 0x65, 0x64, + 0x62, 0x61, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x08, + 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x65, + 0x72, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, + 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x73, + 0x65, 0x72, 0x74, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, + 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x1c, 0x0a, 0x09, 0x6f, 0x76, + 0x65, 0x72, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6f, + 0x76, 0x65, 0x72, 0x77, 0x72, 0x69, 0x74, 0x65, 0x22, 0x1d, 0x0a, 0x1b, 0x42, 0x61, 0x74, 0x63, + 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xd3, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x46, + 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x0c, 0x0a, 0x01, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x01, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x12, + 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, + 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, + 0x63, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, + 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x5d, 0x0a, + 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x08, + 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, + 0x63, 0x6b, 0x52, 0x08, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x22, 0x35, 0x0a, 0x14, + 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, + 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, + 0x69, 0x7a, 0x65, 0x22, 0x3d, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x05, + 0x75, 0x73, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x52, 0x05, 0x75, 0x73, 0x65, + 0x72, 0x73, 0x22, 0x70, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, + 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, + 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x74, 0x69, 0x6d, + 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x22, 0x3d, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, + 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, + 0x65, 0x6d, 0x73, 0x22, 0x73, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, + 0x63, 0x6b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x38, + 0x0a, 0x0c, 0x73, 0x63, 0x61, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, + 0x53, 0x63, 0x61, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0b, 0x73, 0x63, 0x61, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4b, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x46, + 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x08, 0x66, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, + 0x6b, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x2e, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x08, 0x66, 0x65, 0x65, + 0x64, 0x62, 0x61, 0x63, 0x6b, 0x32, 0xc7, 0x0d, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x12, 0x37, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x15, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x50, 0x69, + 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x10, + 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, + 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, + 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, + 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0d, 0x42, 0x61, 0x74, + 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x47, 0x65, 0x74, 0x49, 0x74, + 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, + 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x1b, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, + 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x49, + 0x74, 0x65, 0x6d, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x4d, 0x6f, + 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, + 0x73, 0x12, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, + 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, + 0x74, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x20, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, + 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, + 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x5b, 0x0a, 0x10, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, + 0x73, 0x65, 0x72, 0x73, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, + 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x55, 0x73, + 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, + 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x55, + 0x73, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x4d, 0x6f, + 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x2e, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x55, 0x73, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, + 0x73, 0x12, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, + 0x55, 0x73, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, + 0x74, 0x55, 0x73, 0x65, 0x72, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x20, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, + 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, + 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x5c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, + 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, + 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, - 0x62, 0x61, 0x63, 0x6b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x7a, 0x68, 0x65, 0x6e, 0x67, 0x68, 0x61, 0x6f, 0x7a, 0x2f, 0x67, - 0x6f, 0x72, 0x73, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, + 0x0a, 0x16, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, + 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, + 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x28, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x55, 0x73, 0x65, 0x72, 0x49, 0x74, 0x65, 0x6d, 0x46, 0x65, 0x65, 0x64, 0x62, + 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, + 0x13, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x65, 0x65, 0x64, + 0x62, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, + 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, + 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x73, 0x65, 0x72, + 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, + 0x63, 0x6b, 0x12, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, + 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, + 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, + 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, + 0x74, 0x55, 0x73, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x49, 0x74, + 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x60, 0x0a, + 0x11, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x12, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x47, 0x65, + 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x65, 0x64, 0x62, 0x61, 0x63, 0x6b, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, + 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x7a, 0x68, + 0x65, 0x6e, 0x67, 0x68, 0x61, 0x6f, 0x7a, 0x2f, 0x67, 0x6f, 0x72, 0x73, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/protocol/data_store.proto b/protocol/data_store.proto index 19f8829dd..87ce3b22f 100644 --- a/protocol/data_store.proto +++ b/protocol/data_store.proto @@ -43,6 +43,7 @@ message ScanOptions { optional google.protobuf.Timestamp begin_time = 5; optional google.protobuf.Timestamp end_time = 6; repeated string feedback_types = 7; + bool order_by_item_id = 8; } message BatchInsertItemsRequest { From 6424518740192efd7fe115d7302d998d42602c10 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Wed, 18 Dec 2024 13:29:18 +0000 Subject: [PATCH 18/19] support order by item id --- storage/data/database.go | 8 ++++++++ storage/data/database_test.go | 4 ++-- storage/data/mongodb.go | 3 +++ storage/data/proxy.go | 11 ++++++++--- storage/data/sql.go | 11 ++++++++--- 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/storage/data/database.go b/storage/data/database.go index 544227eca..e4bf66b34 100644 --- a/storage/data/database.go +++ b/storage/data/database.go @@ -179,6 +179,7 @@ type ScanOptions struct { BeginTime *time.Time EndTime *time.Time FeedbackTypes []string + OrderByItemId bool } type ScanOption func(options *ScanOptions) @@ -232,6 +233,13 @@ func WithFeedbackTypes(feedbackTypes ...string) ScanOption { } } +// WithOrderByItemId sets the order by item id. +func WithOrderByItemId() ScanOption { + return func(options *ScanOptions) { + options.OrderByItemId = true + } +} + func NewScanOptions(opts ...ScanOption) ScanOptions { options := ScanOptions{} for _, opt := range opts { diff --git a/storage/data/database_test.go b/storage/data/database_test.go index 592f42fa2..24b127f1f 100644 --- a/storage/data/database_test.go +++ b/storage/data/database_test.go @@ -271,8 +271,8 @@ func (suite *baseTestSuite) TestFeedback() { suite.Empty(feedbackFromStream) feedbackFromStream = suite.getFeedbackStream(ctx, 3, WithBeginUserId("1"), WithEndUserId("3"), WithEndTime(time.Now()), WithFeedbackTypes(positiveFeedbackType)) suite.Equal(feedback[1:4], feedbackFromStream) - feedbackFromStream = suite.getFeedbackStream(ctx, 3, WithBeginItemId("2"), WithEndItemId("6"), WithEndTime(time.Now()), WithFeedbackTypes(positiveFeedbackType)) - suite.ElementsMatch(feedback[1:4], feedbackFromStream) + feedbackFromStream = suite.getFeedbackStream(ctx, 3, WithBeginItemId("2"), WithEndItemId("6"), WithEndTime(time.Now()), WithFeedbackTypes(positiveFeedbackType), WithOrderByItemId()) + suite.Equal([]Feedback{feedback[3], feedback[2], feedback[1]}, feedbackFromStream) // Get items err = suite.Database.Optimize() suite.NoError(err) diff --git a/storage/data/mongodb.go b/storage/data/mongodb.go index 1f953cb73..e658fd381 100644 --- a/storage/data/mongodb.go +++ b/storage/data/mongodb.go @@ -736,6 +736,9 @@ func (db *MongoDB) GetFeedbackStream(ctx context.Context, batchSize int, scanOpt } filter["feedbackkey.itemid"] = itemIdConditions } + if scan.OrderByItemId { + opt.SetSort(bson.D{{"feedbackkey.itemid", 1}}) + } r, err := c.Find(ctx, filter, opt) if err != nil { diff --git a/storage/data/proxy.go b/storage/data/proxy.go index 38989281c..761c9a69d 100644 --- a/storage/data/proxy.go +++ b/storage/data/proxy.go @@ -17,14 +17,15 @@ package data import ( "context" "encoding/json" + "io" + "net" + "time" + "github.com/juju/errors" "github.com/samber/lo" "github.com/zhenghaoz/gorse/protocol" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" - "io" - "net" - "time" ) type ProxyServer struct { @@ -438,6 +439,9 @@ func (p *ProxyServer) GetFeedbackStream(in *protocol.GetFeedbackStreamRequest, s if in.ScanOptions.EndItemId != nil { opts = append(opts, WithEndItemId(*in.ScanOptions.EndItemId)) } + if in.ScanOptions.OrderByItemId { + opts = append(opts, WithOrderByItemId()) + } feedbackChan, errChan := p.database.GetFeedbackStream(stream.Context(), int(in.BatchSize), opts...) for feedback := range feedbackChan { pbFeedback := make([]*protocol.Feedback, len(feedback)) @@ -939,6 +943,7 @@ func (p ProxyClient) GetFeedbackStream(ctx context.Context, batchSize int, optio BeginItemId: o.BeginItemId, EndItemId: o.EndItemId, FeedbackTypes: o.FeedbackTypes, + OrderByItemId: o.OrderByItemId, } if o.BeginTime != nil { pbOptions.BeginTime = timestamppb.New(*o.BeginTime) diff --git a/storage/data/sql.go b/storage/data/sql.go index 9b27a99fa..c4eb6ba02 100644 --- a/storage/data/sql.go +++ b/storage/data/sql.go @@ -958,9 +958,9 @@ func (d *SQLDatabase) GetFeedbackStream(ctx context.Context, batchSize int, scan defer close(feedbackChan) defer close(errChan) // send query - tx := d.gormDB.WithContext(ctx).Table(d.FeedbackTable()). - Select("feedback_type, user_id, item_id, time_stamp, comment"). - Order("feedback_type, user_id, item_id") + tx := d.gormDB.WithContext(ctx). + Table(d.FeedbackTable()). + Select("feedback_type, user_id, item_id, time_stamp, comment") if len(scan.FeedbackTypes) > 0 { tx.Where("feedback_type IN ?", scan.FeedbackTypes) } @@ -982,6 +982,11 @@ func (d *SQLDatabase) GetFeedbackStream(ctx context.Context, batchSize int, scan if scan.EndItemId != nil { tx.Where("item_id <= ?", scan.EndItemId) } + if scan.OrderByItemId { + tx.Order("item_id") + } else { + tx.Order("feedback_type, user_id, item_id") + } result, err := tx.Rows() if err != nil { errChan <- errors.Trace(err) From cf14a42c9d21f89dcf54b7a6125776355555dcee Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Wed, 18 Dec 2024 21:41:49 +0800 Subject: [PATCH 19/19] implement NP --- master/tasks.go | 64 +++++++++++++++++++++++++++++--------------- master/tasks_test.go | 3 +-- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/master/tasks.go b/master/tasks.go index 696338cc4..b7f133d58 100644 --- a/master/tasks.go +++ b/master/tasks.go @@ -1481,13 +1481,15 @@ func (m *Master) LoadDataFromDatabase( span.Add(1) // STEP 2: pull items + var items []data.Item itemLabelCount := make(map[string]int) itemLabelFirst := make(map[string]int32) itemLabelIndex := base.NewMapIndex() start = time.Now() itemChan, errChan := database.GetItemStream(newCtx, batchSize, itemTimeLimit) - for items := range itemChan { - for _, item := range items { + for batchItems := range itemChan { + items = append(items, batchItems...) + for _, item := range batchItems { rankingDataset.AddItem(item.ItemId) itemIndex := rankingDataset.ItemIndex.ToNumber(item.ItemId) if len(rankingDataset.ItemFeatures) == int(itemIndex) { @@ -1525,15 +1527,6 @@ func (m *Master) LoadDataFromDatabase( if item.IsHidden { // set hidden flag rankingDataset.HiddenItems[itemIndex] = true } - // TODO: Refactor - // add item to non-personalized recommenders - feedback, err := database.GetItemFeedback(newCtx, item.ItemId, posFeedbackTypes...) - if err != nil { - return nil, nil, errors.Trace(err) - } - for _, recommender := range nonPersonalizedRecommenders { - recommender.Push(item, feedback) - } } } if err = <-errChan; err != nil { @@ -1554,22 +1547,26 @@ func (m *Master) LoadDataFromDatabase( positiveSet[i] = mapset.NewSet[int32]() } - // split user groups - users := rankingDataset.UserIndex.GetNames() - sort.Strings(users) - userGroups := parallel.Split(users, m.Config.Master.NumJobs) + // split item groups + sort.Slice(items, func(i, j int) bool { + return items[i].ItemId < items[j].ItemId + }) + itemGroups := parallel.Split(items, m.Config.Master.NumJobs) // STEP 3: pull positive feedback var mu sync.Mutex var posFeedbackCount int start = time.Now() - err = parallel.Parallel(len(userGroups), m.Config.Master.NumJobs, func(_, userIndex int) error { + err = parallel.Parallel(len(itemGroups), m.Config.Master.NumJobs, func(_, i int) error { + var itemFeedback []data.Feedback + var itemGroupIndex int feedbackChan, errChan := database.GetFeedbackStream(newCtx, batchSize, - data.WithBeginUserId(userGroups[userIndex][0]), - data.WithEndUserId(userGroups[userIndex][len(userGroups[userIndex])-1]), + data.WithBeginItemId(itemGroups[i][0].ItemId), + data.WithEndItemId(itemGroups[i][len(itemGroups[i])-1].ItemId), feedbackTimeLimit, data.WithEndTime(*m.Config.Now()), - data.WithFeedbackTypes(posFeedbackTypes...)) + data.WithFeedbackTypes(posFeedbackTypes...), + data.WithOrderByItemId()) for feedback := range feedbackChan { for _, f := range feedback { // convert user and item id to index @@ -1595,6 +1592,29 @@ func (m *Master) LoadDataFromDatabase( // insert feedback to evaluator evaluator.Positive(f.FeedbackType, userIndex, itemIndex, f.Timestamp) mu.Unlock() + + // append item feedback + if len(itemFeedback) == 0 || itemFeedback[len(itemFeedback)-1].ItemId == f.ItemId { + itemFeedback = append(itemFeedback, f) + } else { + // add item to non-personalized recommenders + for _, recommender := range nonPersonalizedRecommenders { + recommender.Push(itemGroups[i][itemGroupIndex], itemFeedback) + } + itemFeedback = itemFeedback[:0] + itemFeedback = append(itemFeedback, f) + } + // find item group index + for itemGroupIndex = 0; itemGroupIndex < len(itemGroups[i]); itemGroupIndex++ { + if itemGroups[i][itemGroupIndex].ItemId == f.ItemId { + break + } + } + } + + // add item to non-personalized recommenders + for _, recommender := range nonPersonalizedRecommenders { + recommender.Push(itemGroups[i][itemGroupIndex], itemFeedback) } } if err = <-errChan; err != nil { @@ -1620,10 +1640,10 @@ func (m *Master) LoadDataFromDatabase( // STEP 4: pull negative feedback start = time.Now() var negativeFeedbackCount float64 - err = parallel.Parallel(len(userGroups), m.Config.Master.NumJobs, func(_, userIndex int) error { + err = parallel.Parallel(len(itemGroups), m.Config.Master.NumJobs, func(_, i int) error { feedbackChan, errChan := database.GetFeedbackStream(newCtx, batchSize, - data.WithBeginUserId(userGroups[userIndex][0]), - data.WithEndUserId(userGroups[userIndex][len(userGroups[userIndex])-1]), + data.WithBeginItemId(itemGroups[i][0].ItemId), + data.WithEndItemId(itemGroups[i][len(itemGroups[i])-1].ItemId), feedbackTimeLimit, data.WithEndTime(*m.Config.Now()), data.WithFeedbackTypes(readTypes...)) diff --git a/master/tasks_test.go b/master/tasks_test.go index 9e2da0c5c..d7a168fbe 100644 --- a/master/tasks_test.go +++ b/master/tasks_test.go @@ -532,8 +532,7 @@ func (s *MasterTestSuite) TestLoadDataFromDatabase() { UserId: strconv.Itoa(j), FeedbackType: "positive", }, - // TODO: Refactor - Timestamp: time.Now().Add(-time.Second), + Timestamp: time.Now(), }) } // negative feedback