From ac09eefec1dcd7dca3c44c0fb0ece5dba0d81c9f Mon Sep 17 00:00:00 2001 From: Dmitriy Matrenichev Date: Thu, 10 Oct 2024 22:59:11 +0300 Subject: [PATCH] chore: move from Codec to CodecV2 This should be protocol backward compatible change (and breaking change in the source code) which should speed-up the process of gRPC data encoding / decoding. Also unify our CodecV2 usage. For https://github.com/siderolabs/talos/issues/9404 Signed-off-by: Dmitriy Matrenichev --- client/go.mod | 7 +- client/go.sum | 14 ++- client/pkg/client/client.go | 1 + client/pkg/client/encoding.go | 54 -------- go.mod | 23 ++-- go.sum | 46 +++---- hack/zstd-dict/go.mod | 2 +- internal/backend/grpc/router/encoding.go | 118 ------------------ internal/backend/grpc/router/router.go | 4 +- internal/backend/grpc/router/server.go | 3 +- .../backend/grpc/router/talos_backend_test.go | 2 +- 11 files changed, 56 insertions(+), 218 deletions(-) delete mode 100644 client/pkg/client/encoding.go delete mode 100644 internal/backend/grpc/router/encoding.go diff --git a/client/go.mod b/client/go.mod index c16fdcf2..2a80339d 100644 --- a/client/go.mod +++ b/client/go.mod @@ -24,13 +24,14 @@ require ( github.com/hexops/gotextdiff v1.0.3 github.com/klauspost/compress v1.17.9 github.com/mattn/go-isatty v0.0.20 - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 + github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587 github.com/sergi/go-diff v1.3.1 github.com/siderolabs/gen v0.5.0 github.com/siderolabs/go-api-signature v0.3.6 github.com/siderolabs/go-kubeconfig v0.1.0 github.com/siderolabs/go-pointer v1.0.0 github.com/siderolabs/image-factory v0.5.0 + github.com/siderolabs/proto-codec v0.1.1 github.com/siderolabs/talos/pkg/machinery v1.8.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 @@ -38,7 +39,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.23.0 - google.golang.org/grpc v1.66.0 + google.golang.org/grpc v1.67.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v3 v3.0.1 k8s.io/client-go v0.31.0 @@ -95,7 +96,7 @@ require ( golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.24.0 // indirect + golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect diff --git a/client/go.sum b/client/go.sum index a67e5475..ad374e83 100644 --- a/client/go.sum +++ b/client/go.sum @@ -145,8 +145,8 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587 h1:xzZOeCMQLA/W198ZkdVdt4EKFKJtS26B773zNU377ZY= +github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -180,6 +180,8 @@ github.com/siderolabs/image-factory v0.5.0 h1:v1FXZLCcV6xu+6QpgvhDEICxVF7o2VxMjf github.com/siderolabs/image-factory v0.5.0/go.mod h1:npJwHOBsI+h+gKdezCyrs7ZHDmkgRnrAK2Cjk1nzv8A= github.com/siderolabs/net v0.4.0 h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I= github.com/siderolabs/net v0.4.0/go.mod h1:/ibG+Hm9HU27agp5r9Q3eZicEfjquzNzQNux5uEk0kM= +github.com/siderolabs/proto-codec v0.1.1 h1:4jiUwW/vaXTZ+YNgZDs37B4aj/1mzV/erIkzUUCRY9g= +github.com/siderolabs/proto-codec v0.1.1/go.mod h1:rIvmhKJG8+JwSCGPX+cQljpOMDmuHhLKPkt6KaFwEaU= github.com/siderolabs/protoenc v0.2.1 h1:BqxEmeWQeMpNP3R6WrPqDatX8sM/r4t97OP8mFmg6GA= github.com/siderolabs/protoenc v0.2.1/go.mod h1:StTHxjet1g11GpNAWiATgc8K0HMKiFSEVVFOa/H0otc= github.com/siderolabs/talos/pkg/machinery v1.8.0 h1:azhBj+Nm9oTgaFgcNaHU8TPS9Oi5OdV1ELNgFAVder8= @@ -267,8 +269,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -302,8 +304,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1: google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= +google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/pkg/client/client.go b/client/pkg/client/client.go index 39018c13..447b1d21 100644 --- a/client/pkg/client/client.go +++ b/client/pkg/client/client.go @@ -13,6 +13,7 @@ import ( "slices" "github.com/siderolabs/go-api-signature/pkg/client/auth" + _ "github.com/siderolabs/proto-codec/codec" // for encoding.CodecV2 "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" diff --git a/client/pkg/client/encoding.go b/client/pkg/client/encoding.go deleted file mode 100644 index 03729ac7..00000000 --- a/client/pkg/client/encoding.go +++ /dev/null @@ -1,54 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package client - -import ( - "fmt" - - "google.golang.org/grpc/encoding" - _ "google.golang.org/grpc/encoding/proto" // Register the proto codec before we replace it with ours. - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/protoadapt" -) - -// Name is the name registered for the proto compressor. -const Name = "proto" - -type vtprotoCodec struct{} - -func (vtprotoCodec) Marshal(v any) ([]byte, error) { - switch v := v.(type) { - case vtprotoMessage: - return v.MarshalVT() - case proto.Message: - return proto.Marshal(v) - case protoadapt.MessageV1: - return proto.Marshal(protoadapt.MessageV2Of(v)) - default: - return nil, fmt.Errorf("failed to marshal, message is %T, must satisfy the vtprotoMessage, proto.Message or protoadapt.MessageV1 ", v) - } -} - -func (vtprotoCodec) Unmarshal(data []byte, v any) error { - switch v := v.(type) { - case vtprotoMessage: - return v.UnmarshalVT(data) - case proto.Message: - return proto.Unmarshal(data, v) - case protoadapt.MessageV1: - return proto.Unmarshal(data, protoadapt.MessageV2Of(v)) - default: - return fmt.Errorf("failed to unmarshal, message is %T, must satisfy the vtprotoMessage, proto.Message or protoadapt.MessageV1", v) - } -} - -func (vtprotoCodec) Name() string { return Name } - -type vtprotoMessage interface { - MarshalVT() ([]byte, error) - UnmarshalVT([]byte) error -} - -func init() { encoding.RegisterCodec(vtprotoCodec{}) } diff --git a/go.mod b/go.mod index 2b388b0f..039ac133 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/siderolabs/omni go 1.23.0 replace ( + // ensure h2 ALPN is used + github.com/siderolabs/crypto => github.com/DmitriyMV/crypto v0.0.0-20241010155414-58b2f9291c7e // use nested module github.com/siderolabs/omni/client => ./client // forked go-yaml that introduces RawYAML interface, which can be used to populate YAML fields using bytes @@ -72,10 +74,11 @@ require ( github.com/siderolabs/go-retry v0.3.3 github.com/siderolabs/go-tail v0.1.1 github.com/siderolabs/go-talos-support v0.1.1 - github.com/siderolabs/grpc-proxy v0.4.1 + github.com/siderolabs/grpc-proxy v0.5.0 github.com/siderolabs/image-factory v0.5.0 github.com/siderolabs/kms-client v0.1.0 github.com/siderolabs/omni/client v0.39.1 + github.com/siderolabs/proto-codec v0.1.1 github.com/siderolabs/siderolink v0.3.9 github.com/siderolabs/talos/pkg/machinery v1.8.0 github.com/siderolabs/tcpproxy v0.1.0 @@ -90,14 +93,14 @@ require ( go.etcd.io/etcd/server/v3 v3.5.15 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.26.0 - golang.org/x/net v0.28.0 + golang.org/x/crypto v0.28.0 + golang.org/x/net v0.30.0 golang.org/x/sync v0.8.0 golang.org/x/tools v0.24.0 golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 - google.golang.org/grpc v1.66.0 - google.golang.org/protobuf v1.34.2 + google.golang.org/grpc v1.67.1 + google.golang.org/protobuf v1.35.1 gopkg.in/yaml.v3 v3.0.3 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.0 @@ -201,7 +204,7 @@ require ( github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -244,14 +247,14 @@ require ( golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.6.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index b991c61d..9b52cca6 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT filippo.io/age v1.2.0 h1:vRDp7pUMaAJzXNIWJVAZnEf/Dyi4Vu4wI8S1LBzufhE= filippo.io/age v1.2.0/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DmitriyMV/crypto v0.0.0-20241010155414-58b2f9291c7e h1:vKgVWVhipaPkXfnNZSOTSMZOHtb+cX1/JSXidkvsHOQ= +github.com/DmitriyMV/crypto v0.0.0-20241010155414-58b2f9291c7e/go.mod h1:hsR3tJ3aaeuhCChsLF4dBd9vlJVPvmhg4vvx2ez4aD4= github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/ProtonMail/go-crypto v1.1.0-alpha.5.0.20240827111422-b5837fa4476e h1:O1cSHAcGcbGEO66Qi2AIJeYmXO8iP4L/PNrbdN+RjJA= github.com/ProtonMail/go-crypto v1.1.0-alpha.5.0.20240827111422-b5837fa4476e/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= @@ -362,8 +364,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587 h1:xzZOeCMQLA/W198ZkdVdt4EKFKJtS26B773zNU377ZY= +github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -397,8 +399,6 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= -github.com/siderolabs/crypto v0.4.4 h1:Q6EDBMR2Ub2oAZW5Xl8lrKB27bM3Sn8Gkfw3rngco5U= -github.com/siderolabs/crypto v0.4.4/go.mod h1:hsR3tJ3aaeuhCChsLF4dBd9vlJVPvmhg4vvx2ez4aD4= github.com/siderolabs/discovery-api v0.1.4 h1:2fMEFSMiWaD1zDiBDY5md8VxItvL1rDQRSOfeXNjYKc= github.com/siderolabs/discovery-api v0.1.4/go.mod h1:kaBy+G42v2xd/uAF/NIe383sjNTBE2AhxPTyi9SZI0s= github.com/siderolabs/discovery-client v0.1.9 h1:yDzvts++Nf/2qczdDUfU5GAibkEIgz/eo9RPG/k/rOc= @@ -433,14 +433,16 @@ github.com/siderolabs/go-tail v0.1.1 h1:3XeJgd97OHyFAIE7nQEMcRhOfnv7DvXbu0BRKbtT github.com/siderolabs/go-tail v0.1.1/go.mod h1:IihAL39acadXHfb5fEAOKK2DaDFIrG2+VD3b2H/ziZ0= github.com/siderolabs/go-talos-support v0.1.1 h1:g51J0WQssQAycU/0cDliC2l4uX2H02yUs2+fa5pCvHg= github.com/siderolabs/go-talos-support v0.1.1/go.mod h1:o4woiYS+2J3djCQgyHZRVZQm8XpazQr+XPcTXAZvamo= -github.com/siderolabs/grpc-proxy v0.4.1 h1:UTYviMqb65oKjnH7dy5D+U4zMJ6iCTjAN6x6K/Ss120= -github.com/siderolabs/grpc-proxy v0.4.1/go.mod h1:QwQuLUpJrlN08kpP0m63oO/SEeoz0dEhU9ndlBafc0Y= +github.com/siderolabs/grpc-proxy v0.5.0 h1:fv3E7Hd9ufp1lHlumSB0SLlHtlFDlBH9rH//+3t0UZQ= +github.com/siderolabs/grpc-proxy v0.5.0/go.mod h1:HW3ROf5epUDUk8RGq2z8jdSNDK+wijp4hl1E7KAvELo= github.com/siderolabs/image-factory v0.5.0 h1:v1FXZLCcV6xu+6QpgvhDEICxVF7o2VxMjfU0MutkFbo= github.com/siderolabs/image-factory v0.5.0/go.mod h1:npJwHOBsI+h+gKdezCyrs7ZHDmkgRnrAK2Cjk1nzv8A= github.com/siderolabs/kms-client v0.1.0 h1:rCDWzcDDsNlp6zdyLngOuuhchVILn+vwUQy3tk6rQps= github.com/siderolabs/kms-client v0.1.0/go.mod h1:4UQkRhuEh3kaK7VhJxez4YyJLv6lPEff7g3Pa6Y9okg= github.com/siderolabs/net v0.4.0 h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I= github.com/siderolabs/net v0.4.0/go.mod h1:/ibG+Hm9HU27agp5r9Q3eZicEfjquzNzQNux5uEk0kM= +github.com/siderolabs/proto-codec v0.1.1 h1:4jiUwW/vaXTZ+YNgZDs37B4aj/1mzV/erIkzUUCRY9g= +github.com/siderolabs/proto-codec v0.1.1/go.mod h1:rIvmhKJG8+JwSCGPX+cQljpOMDmuHhLKPkt6KaFwEaU= github.com/siderolabs/protoenc v0.2.1 h1:BqxEmeWQeMpNP3R6WrPqDatX8sM/r4t97OP8mFmg6GA= github.com/siderolabs/protoenc v0.2.1/go.mod h1:StTHxjet1g11GpNAWiATgc8K0HMKiFSEVVFOa/H0otc= github.com/siderolabs/siderolink v0.3.9 h1:lvHFCu+CdfUyMk90g1Zt5r7n1Dw3jhXMxyzXmQ0776o= @@ -553,8 +555,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= @@ -588,8 +590,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= @@ -628,8 +630,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -637,8 +639,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -647,8 +649,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -686,18 +688,18 @@ google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d h1:/hmn0Ku5kWij/kj google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/hack/zstd-dict/go.mod b/hack/zstd-dict/go.mod index 4088a4f8..bac38e3f 100644 --- a/hack/zstd-dict/go.mod +++ b/hack/zstd-dict/go.mod @@ -6,6 +6,7 @@ require ( github.com/klauspost/compress v1.17.9 github.com/mittwald/go-helm-client v0.12.13 go.uber.org/zap v1.26.0 + gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.15.4 ) @@ -128,7 +129,6 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.30.3 // indirect k8s.io/apiextensions-apiserver v0.30.3 // indirect k8s.io/apimachinery v0.30.3 // indirect diff --git a/internal/backend/grpc/router/encoding.go b/internal/backend/grpc/router/encoding.go deleted file mode 100644 index 3d21df01..00000000 --- a/internal/backend/grpc/router/encoding.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) 2024 Sidero Labs, Inc. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. - -package router - -import ( - "fmt" - - "google.golang.org/grpc/encoding" - _ "google.golang.org/grpc/encoding/proto" // Register the proto codec before we replace it with ours. - "google.golang.org/grpc/mem" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/protoadapt" -) - -// Name is the name registered for the proto compressor. -const Name = "proto" - -type vtprotoCodec struct{} - -func (c vtprotoCodec) Marshal(v any) (mem.BufferSlice, error) { - size, err := getSize(v) - if err != nil { - return nil, err - } - - if mem.IsBelowBufferPoolingThreshold(size) { - buf, err := marshal(v) - if err != nil { - return nil, err - } - - return mem.BufferSlice{mem.SliceBuffer(buf)}, nil - } - - pool := mem.DefaultBufferPool() - - buf := pool.Get(size) - if err := marshalAppend((*buf)[:size], v); err != nil { - pool.Put(buf) - - return nil, err - } - - return mem.BufferSlice{mem.NewBuffer(buf, pool)}, nil -} - -func getSize(v any) (int, error) { - switch v := v.(type) { - case vtprotoMessage: - return v.SizeVT(), nil - case proto.Message: - return proto.Size(v), nil - case protoadapt.MessageV1: - return proto.Size(protoadapt.MessageV2Of(v)), nil - default: - return -1, fmt.Errorf("failed to get size, message is %T, must satisfy the vtprotoMessage, proto.Message or protoadapt.MessageV1 ", v) - } -} - -func marshal(v any) ([]byte, error) { - switch v := v.(type) { - case vtprotoMessage: - return v.MarshalVT() - case proto.Message: - return proto.Marshal(v) - case protoadapt.MessageV1: - return proto.Marshal(protoadapt.MessageV2Of(v)) - default: - return nil, fmt.Errorf("failed to marshal, message is %T, must satisfy the vtprotoMessage, proto.Message or protoadapt.MessageV1 ", v) - } -} - -func marshalAppend(dst []byte, v any) error { - takeErr := func(_ any, e error) error { return e } - - switch v := v.(type) { - case vtprotoMessage: - return takeErr(v.MarshalToSizedBufferVT(dst)) - case proto.Message: - return takeErr((proto.MarshalOptions{}).MarshalAppend(dst, v)) - case protoadapt.MessageV1: - return takeErr((proto.MarshalOptions{}).MarshalAppend(dst[:0], protoadapt.MessageV2Of(v))) - default: - return fmt.Errorf("failed to marshal-append, message is %T, must satisfy the vtprotoMessage, proto.Message or protoadapt.MessageV1 ", v) - } -} - -func (c vtprotoCodec) Unmarshal(data mem.BufferSlice, v any) error { - buf := data.MaterializeToBuffer(mem.DefaultBufferPool()) - defer buf.Free() - - switch v := v.(type) { - case vtprotoMessage: - return v.UnmarshalVT(buf.ReadOnlyData()) - case proto.Message: - return proto.Unmarshal(buf.ReadOnlyData(), v) - case protoadapt.MessageV1: - return proto.Unmarshal(buf.ReadOnlyData(), protoadapt.MessageV2Of(v)) - default: - return fmt.Errorf("failed to unmarshal, message is %T, must satisfy the vtprotoMessage, proto.Message or protoadapt.MessageV1", v) - } -} - -func (c vtprotoCodec) Name() string { return Name } - -func (vtprotoCodec) OldName() string { return Name } - -type vtprotoMessage interface { - MarshalToSizedBufferVT([]byte) (int, error) - MarshalVT() ([]byte, error) - UnmarshalVT([]byte) error - SizeVT() int -} - -func init() { encoding.RegisterCodecV2(vtprotoCodec{}) } diff --git a/internal/backend/grpc/router/router.go b/internal/backend/grpc/router/router.go index 341315b4..92eae76e 100644 --- a/internal/backend/grpc/router/router.go +++ b/internal/backend/grpc/router/router.go @@ -93,7 +93,7 @@ func NewRouter( grpc.WithDefaultCallOptions( // we are proxying requests to ourselves, so we don't need to impose a limit grpc.MaxCallRecvMsgSize(math.MaxInt32), - grpc.ForceCodec(proxy.Codec()), + grpc.ForceCodecV2(proxy.Codec()), ), ) if err != nil { @@ -292,7 +292,7 @@ func (r *Router) getConn(ctx context.Context, contextName string) (*grpc.ClientC MinConnectTimeout: 20 * time.Second, }), grpc.WithTransportCredentials(creds), - grpc.WithDefaultCallOptions(grpc.ForceCodec(proxy.Codec())), + grpc.WithDefaultCallOptions(grpc.ForceCodecV2(proxy.Codec())), grpc.WithSharedWriteBuffer(true), } diff --git a/internal/backend/grpc/router/server.go b/internal/backend/grpc/router/server.go index 4c10a527..91e46663 100644 --- a/internal/backend/grpc/router/server.go +++ b/internal/backend/grpc/router/server.go @@ -11,6 +11,7 @@ import ( grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" "github.com/siderolabs/grpc-proxy/proxy" + _ "github.com/siderolabs/proto-codec/codec" // for encoding.CodecV2 "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/grpc" @@ -28,7 +29,7 @@ type Director interface { func NewServer(router Director, options ...grpc.ServerOption) *grpc.Server { opts := append( []grpc.ServerOption{ - grpc.ForceServerCodec(proxy.Codec()), + grpc.ForceServerCodecV2(proxy.Codec()), grpc.UnknownServiceHandler( proxy.TransparentHandler( router.Director, diff --git a/internal/backend/grpc/router/talos_backend_test.go b/internal/backend/grpc/router/talos_backend_test.go index 8100efdb..82defcd8 100644 --- a/internal/backend/grpc/router/talos_backend_test.go +++ b/internal/backend/grpc/router/talos_backend_test.go @@ -205,7 +205,7 @@ func dial(serverEndpoint string) (*grpc.ClientConn, error) { MinConnectTimeout: 20 * time.Second, }), grpc.WithTransportCredentials(creds), - grpc.WithDefaultCallOptions(grpc.ForceCodec(proxy.Codec())), + grpc.WithDefaultCallOptions(grpc.ForceCodecV2(proxy.Codec())), } return grpc.NewClient(serverEndpoint, opts...)