From 5aadad5a9f0b59336ed48771f9ff388d7d1c9114 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Fri, 29 Aug 2025 11:04:49 +1000 Subject: [PATCH 01/13] checker --- gcp/workers/record_checker/go.mod | 49 ++++ gcp/workers/record_checker/go.sum | 182 ++++++++++++++ gcp/workers/record_checker/record_checker.go | 248 +++++++++++++++++++ go/go.mod | 38 +++ go/go.sum | 166 +++++++++++++ go/models/models.go | 27 ++ go/models/validate/run_validate.sh | 2 + go/models/validate/validate.go | 51 ++++ go/models/validate/validate.py | 44 ++++ 9 files changed, 807 insertions(+) create mode 100644 gcp/workers/record_checker/go.mod create mode 100644 gcp/workers/record_checker/go.sum create mode 100644 gcp/workers/record_checker/record_checker.go create mode 100644 go/go.mod create mode 100644 go/go.sum create mode 100644 go/models/models.go create mode 100755 go/models/validate/run_validate.sh create mode 100644 go/models/validate/validate.go create mode 100644 go/models/validate/validate.py diff --git a/gcp/workers/record_checker/go.mod b/gcp/workers/record_checker/go.mod new file mode 100644 index 00000000000..6b96fa4fc7f --- /dev/null +++ b/gcp/workers/record_checker/go.mod @@ -0,0 +1,49 @@ +module github.com/google/osv.dev/gcp/workers/record_checker + +go 1.23.5 + +replace github.com/google/osv.dev/go => ../../../go + +require ( + cloud.google.com/go/datastore v1.20.0 + cloud.google.com/go/logging v1.11.0 + cloud.google.com/go/pubsub v1.44.0 + cloud.google.com/go/storage v1.43.0 + github.com/google/osv.dev/go v0.0.0-00010101000000-000000000000 + google.golang.org/api v0.203.0 +) + +require ( + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.9 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.5.2 // indirect + cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/longrunning v0.6.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect +) diff --git a/gcp/workers/record_checker/go.sum b/gcp/workers/record_checker/go.sum new file mode 100644 index 00000000000..a98b7430bfd --- /dev/null +++ b/gcp/workers/record_checker/go.sum @@ -0,0 +1,182 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ= +cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= +cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM= +cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew= +cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= +cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY= +cloud.google.com/go/kms v1.20.0/go.mod h1:/dMbFF1tLLFnQV44AoI2GlotbjowyUfgVwezxW291fM= +cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= +cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= +cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= +cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= +cloud.google.com/go/pubsub v1.44.0 h1:pLaMJVDTlnUDIKT5L0k53YyLszfBbGoUBo/IqDK/fEI= +cloud.google.com/go/pubsub v1.44.0/go.mod h1:BD4a/kmE8OePyHoa1qAHEw1rMzXX+Pc8Se54T/8mc3I= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= +go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU= +google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/gcp/workers/record_checker/record_checker.go b/gcp/workers/record_checker/record_checker.go new file mode 100644 index 00000000000..404e8fc17af --- /dev/null +++ b/gcp/workers/record_checker/record_checker.go @@ -0,0 +1,248 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + "time" + + "cloud.google.com/go/datastore" + "cloud.google.com/go/logging" + "cloud.google.com/go/pubsub" + "cloud.google.com/go/storage" + "github.com/google/osv.dev/go/models" + "google.golang.org/api/iterator" +) + +// appEnv holds the clients and other environment-specific resources. +type appEnv struct { + logger *logging.Logger + bucket *storage.BucketHandle + ds *datastore.Client + topic *pubsub.Topic +} + +// setup initializes the application environment. +func setup(ctx context.Context) (*appEnv, error) { + projectID, ok := os.LookupEnv("GOOGLE_CLOUD_PROJECT") + if !ok { + return nil, errors.New("GOOGLE_CLOUD_PROJECT must be set") + } + + logger, err := setupLogging(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("failed to setup logging: %w", err) + } + + bucketName, ok := os.LookupEnv("OSV_VULNERABILITIES_BUCKET") + if !ok { + err := errors.New("OSV_VULNERABILITIES_BUCKET must be set") + logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) + return nil, err + } + stClient, err := storage.NewClient(ctx) + if err != nil { + err = fmt.Errorf("failed to create storage client: %w", err) + logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) + return nil, err + } + bucket := stClient.Bucket(bucketName) + + dsClient, err := datastore.NewClient(ctx, projectID) + if err != nil { + err = fmt.Errorf("failed to create datastore client: %w", err) + logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) + return nil, err + } + + pubsubClient, err := pubsub.NewClient(ctx, projectID) + if err != nil { + err = fmt.Errorf("failed to create pubsub client: %w", err) + logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) + return nil, err + } + topic := pubsubClient.Topic("failed-tasks") + + return &appEnv{ + logger: logger, + bucket: bucket, + ds: dsClient, + topic: topic, + }, nil +} + +func main() { + ctx := context.Background() + env, err := setup(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to setup application: %v\n", err) + os.Exit(1) + } + + rcData, err := getRecordCheckerData(ctx, env.ds) + if err != nil { + env.logger.Log(logging.Entry{ + Severity: logging.Critical, + Payload: fmt.Sprintf("failed to get prior run data: %v", err)}) + os.Exit(1) + } + + var wg sync.WaitGroup + ch := make(chan checkRecordResult) + + var newInvalid []string + var resultsWg sync.WaitGroup + resultsWg.Add(1) + go func() { + defer resultsWg.Done() + newInvalid = handleResults(ctx, env.logger, env.topic, ch) + }() + + for _, id := range rcData.InvalidRecords { + wg.Add(1) + go func(id string) { + defer wg.Done() + checkRecord(ctx, env.logger, env.ds, env.bucket, id, nil, ch) + }(id) + } + runStartTime := time.Now().UTC() + query := datastore.NewQuery("Vulnerability") + if rcData.LastRun != nil { + query = query.FilterField("modified", ">", *rcData.LastRun) + } + it := env.ds.Run(ctx, query) + for { + var vuln models.Vulnerability + key, err := it.Next(&vuln) + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + env.logger.Log(logging.Entry{ + Severity: logging.Critical, + Payload: fmt.Sprintf("failed to query vulnerabilities: %v", err), + }) + os.Exit(1) + } + wg.Add(1) + go func(v models.Vulnerability, name string) { + defer wg.Done() + checkRecord(ctx, env.logger, env.ds, env.bucket, name, &v, ch) + }(vuln, key.Name) + } + + wg.Wait() + close(ch) + resultsWg.Wait() + + rcData = recordCheckerData{ + LastRun: &runStartTime, + InvalidRecords: newInvalid, + } + _, err = env.ds.Put(ctx, datastore.NameKey("JobData", "record_checker", nil), &rcData) + if err != nil { + env.logger.Log(logging.Entry{ + Severity: logging.Error, + Payload: fmt.Sprintf("failed to store run data: %v", err), + }) + } +} + +func handleResults(ctx context.Context, logger *logging.Logger, topic *pubsub.Topic, ch <-chan checkRecordResult) []string { + var newInvalid []string + for result := range ch { + if result.Err != nil { + logger.Log(logging.Entry{ + Severity: logging.Error, + Payload: result.Err, + }) + } + if result.NeedsRetry { + newInvalid = append(newInvalid, result.ID) + msg := pubsub.Message{ + Attributes: map[string]string{"type": "gcs_missing", "id": result.ID}, + } + logger.Log(logging.Entry{ + Severity: logging.Info, + Payload: fmt.Sprintf("Publishing gcs_missing for %s", result.ID), + }) + _, err := topic.Publish(ctx, &msg).Get(ctx) + if err != nil { + logger.Log(logging.Entry{ + Severity: logging.Error, + Payload: fmt.Sprintf("failed to publish message for %s: %v", result.ID, err), + }) + } + } + } + return newInvalid +} + +func setupLogging(ctx context.Context, projectID string) (*logging.Logger, error) { + client, err := logging.NewClient(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("failed to create logging client: %w", err) + } + return client.Logger("record_checker"), nil +} + +type recordCheckerData struct { + LastRun *time.Time `datastore:"last_run"` + InvalidRecords []string `datastore:"invalid_records"` +} + +func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordCheckerData, error) { + infoKey := datastore.NameKey("JobData", "record_checker", nil) + var data recordCheckerData + err := cl.Get(ctx, infoKey, &data) + if errors.Is(err, datastore.ErrNoSuchEntity) { + err = nil + } + return data, err +} + +type checkRecordResult struct { + ID string + NeedsRetry bool + Err error +} + +func checkRecord(ctx context.Context, logger *logging.Logger, cl *datastore.Client, bucket *storage.BucketHandle, id string, vuln *models.Vulnerability, out chan<- checkRecordResult) { + res := checkRecordResult{ID: id} + // Send the result when the function returns. + defer func() { out <- res }() + + if vuln == nil { + key := datastore.NameKey("Vulnerability", id, nil) + var fetchedVuln models.Vulnerability + if err := cl.Get(ctx, key, &fetchedVuln); err != nil { + if errors.Is(err, datastore.ErrNoSuchEntity) { + // This is a permanent error, don't retry. + res.Err = fmt.Errorf("vulnerability %s not found in datastore: %w", id, err) + } else { + // This is likely a transient error, retry. + res.NeedsRetry = true + res.Err = fmt.Errorf("failed to get vulnerability %s from datastore: %w", id, err) + } + return + } + vuln = &fetchedVuln + } + + obj := bucket.Object(fmt.Sprintf("all/pb/%s.pb", id)) + attrs, err := obj.Attrs(ctx) + if err != nil { + res.NeedsRetry = true + if !errors.Is(err, storage.ErrObjectNotExist) { + // Log the error if it's not the expected "not found" error. + res.Err = fmt.Errorf("failed to get GCS attributes for %s: %w", id, err) + } + return + } + + if attrs.CustomTime.Before(vuln.Modified) { + res.NeedsRetry = true + } +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 00000000000..180afd91914 --- /dev/null +++ b/go/go.mod @@ -0,0 +1,38 @@ +module github.com/google/osv.dev/go + +go 1.23.5 + +require cloud.google.com/go/datastore v1.20.0 + +require ( + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.9 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.5.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/api v0.203.0 // indirect + google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect +) diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 00000000000..bfa11a2a2ef --- /dev/null +++ b/go/go.sum @@ -0,0 +1,166 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ= +cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= +cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM= +cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU= +google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= +google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/go/models/models.go b/go/models/models.go new file mode 100644 index 00000000000..3bf1c818a25 --- /dev/null +++ b/go/models/models.go @@ -0,0 +1,27 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import "time" + +type Vulnerability struct { + SourceID string `datastore:"source_id"` + Modified time.Time `datastore:"modified"` + IsWithdrawn bool `datastore:"is_withdrawn"` + ModifiedRaw time.Time `datastore:"modified_raw"` + AliasRaw []string `datastore:"alias_raw"` + RelatedRaw []string `datastore:"related_raw"` + UpstreamRaw []string `datastore:"upstream_raw"` +} diff --git a/go/models/validate/run_validate.sh b/go/models/validate/run_validate.sh new file mode 100755 index 00000000000..d5eedce6cc6 --- /dev/null +++ b/go/models/validate/run_validate.sh @@ -0,0 +1,2 @@ +#!/bin/bash +poetry run python validate.py \ No newline at end of file diff --git a/go/models/validate/validate.go b/go/models/validate/validate.go new file mode 100644 index 00000000000..fe13d18e2b3 --- /dev/null +++ b/go/models/validate/validate.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "cloud.google.com/go/datastore" + "github.com/google/osv.dev/go/models" +) + +func main() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, os.Getenv("GOOGLE_CLOUD_PROJECT")) + if err != nil { + fmt.Printf("(Go) Failed creating Datastore client: %v\n", err) + os.Exit(1) + } + + readRecords(ctx, client) + writeRecords(ctx, client) +} + +func readRecords(ctx context.Context, client *datastore.Client) { + fmt.Println("(Go) Getting Vulnerability") + key := datastore.NameKey("Vulnerability", "CVE-123-456", nil) + var vulnerability models.Vulnerability + if err := client.Get(ctx, key, &vulnerability); err != nil { + fmt.Printf("(Go) Failed getting Vulnerability: %v\n", err) + os.Exit(1) + } +} + +func writeRecords(ctx context.Context, client *datastore.Client) { + fmt.Println("(Go) Writing Vulnerability") + key := datastore.NameKey("Vulnerability", "CVE-987-654", nil) + vulnerability := models.Vulnerability{ + SourceID: "test:path/to/CVE-987-654", + Modified: time.Date(2025, time.December, 31, 23, 59, 59, 0, time.UTC), + IsWithdrawn: false, + ModifiedRaw: time.Date(2025, time.December, 1, 23, 59, 59, 0, time.UTC), + AliasRaw: []string{"OSV-987-654", "TEST-987-654"}, + RelatedRaw: []string{"CVE-999-999"}, + UpstreamRaw: []string{"CVE-987-000", "OSV-987-000"}, + } + if _, err := client.Put(ctx, key, &vulnerability); err != nil { + fmt.Printf("(Go) Failed writing Vulnerability %v: %v\n", key, err) + os.Exit(1) + } +} diff --git a/go/models/validate/validate.py b/go/models/validate/validate.py new file mode 100644 index 00000000000..c2aec253ddc --- /dev/null +++ b/go/models/validate/validate.py @@ -0,0 +1,44 @@ +import datetime +import subprocess +import sys + +from google.cloud import ndb + +import osv.tests +from osv import Vulnerability + +def main() -> int: + # Populate the examples from Python + print('(Python) Putting Vulnerability') + Vulnerability( + id='CVE-123-456', + source_id='test:path/to/CVE-123-456.json', + modified=datetime.datetime(2025, 1, 2, 3, 4, 5, tzinfo=datetime.UTC), + is_withdrawn=False, + modified_raw=datetime.datetime(2025, 1, 1, 1, 1, 1, tzinfo=datetime.UTC), + alias_raw=['OSV-123-456', 'TEST-123-456'], + related_raw=['CVE-000-000', 'CVE-111-111'], + upstream_raw=['CVE-123-000', 'OSV-123-000'], + ).put() + + # Run Go program to read the Python-created entities in Go. + # And write Go entities. + result = subprocess.run(['go', 'run', './validate.go']) + if result.returncode != 0: + return result.returncode + + # Read the Go-created entities in Python. + print('(Python) Getting Vulnerability') + if Vulnerability.get_by_id('CVE-987-654') is None: + return 1 + return 0 + + +if __name__ == '__main__': + try: + osv.tests.start_datastore_emulator() + with ndb.Client().context(): + ret = main() + finally: + osv.tests.stop_emulator() + sys.exit(ret) \ No newline at end of file From bdb154fb9a019297ad71dc2b95a365d2a0697979 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Wed, 10 Sep 2025 11:16:48 +1000 Subject: [PATCH 02/13] clean clean --- gcp/workers/record_checker/record_checker.go | 246 ++++++++++--------- 1 file changed, 136 insertions(+), 110 deletions(-) diff --git a/gcp/workers/record_checker/record_checker.go b/gcp/workers/record_checker/record_checker.go index 404e8fc17af..738b7f4a5bc 100644 --- a/gcp/workers/record_checker/record_checker.go +++ b/gcp/workers/record_checker/record_checker.go @@ -16,6 +16,127 @@ import ( "google.golang.org/api/iterator" ) +const ( + pubsubTopic = "failed-tasks" + numWorkers = 50 +) + +var recordCheckerKey = datastore.NameKey("JobData", "record_checker", nil) + +// run is the main application logic +func run(ctx context.Context, env *appEnv) error { + rcData, err := getRecordCheckerData(ctx, env.ds) + if err != nil { + return fmt.Errorf("failed to get prior run data: %w", err) + } + + tasksChan := make(chan struct { + ID string + Vuln *models.Vulnerability + }) + resultsChan := make(chan checkRecordResult) + + // Start the results handler + var resultsWg sync.WaitGroup + resultsWg.Add(1) + var newInvalid []string + go func() { + defer resultsWg.Done() + for result := range resultsChan { + if handleResult(ctx, env.logger, env.topic, result) { + newInvalid = append(newInvalid, result.ID) + } + } + }() + + // Start the worker pool + var workerWg sync.WaitGroup + for range numWorkers { + workerWg.Add(1) + go func() { + defer workerWg.Done() + for task := range tasksChan { + checkRecord(ctx, env.logger, env.ds, env.bucket, task.ID, task.Vuln, resultsChan) + } + }() + } + + // Queue all invalid records from the previous run. + for _, id := range rcData.InvalidRecords { + tasksChan <- struct { + ID string + Vuln *models.Vulnerability + }{ID: id} + } + + // Queue all new records. + runStartTime := time.Now().UTC() + query := datastore.NewQuery("Vulnerability") + if rcData.LastRun != nil { + query = query.FilterField("modified", ">", *rcData.LastRun) + } + it := env.ds.Run(ctx, query) + for { + var vuln models.Vulnerability + key, err := it.Next(&vuln) + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return fmt.Errorf("failed to query vulnerabilities: %w", err) + } + tasksChan <- struct { + ID string + Vuln *models.Vulnerability + }{ID: key.Name, Vuln: &vuln} + } + // Wait for all tasks to finish processing + close(tasksChan) + workerWg.Wait() + close(resultsChan) + resultsWg.Wait() + + // Update the record checker run data + rcData = recordCheckerData{ + LastRun: &runStartTime, + InvalidRecords: newInvalid, + } + _, err = env.ds.Put(ctx, recordCheckerKey, &rcData) + if err != nil { + return fmt.Errorf("failed to store run data: %w", err) + } + + return nil +} + +func handleResult(ctx context.Context, logger *logging.Logger, topic *pubsub.Topic, result checkRecordResult) bool { + wasInvalid := false + if result.Err != nil { + logger.Log(logging.Entry{ + Severity: logging.Error, + Payload: fmt.Sprintf("failed to process record %s: %v", result.ID, result.Err), + }) + } + if result.NeedsRetry { + wasInvalid = true + msg := pubsub.Message{ + Attributes: map[string]string{"type": "gcs_missing", "id": result.ID}, + } + logger.Log(logging.Entry{ + Severity: logging.Info, + Payload: fmt.Sprintf("Publishing gcs_missing for %s", result.ID), + }) + _, err := topic.Publish(ctx, &msg).Get(ctx) + if err != nil { + logger.Log(logging.Entry{ + Severity: logging.Error, + Payload: fmt.Sprintf("failed to publish message for %s: %v", result.ID, err), + }) + } + } + return wasInvalid +} + // appEnv holds the clients and other environment-specific resources. type appEnv struct { logger *logging.Logger @@ -63,7 +184,7 @@ func setup(ctx context.Context) (*appEnv, error) { logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) return nil, err } - topic := pubsubClient.Topic("failed-tasks") + topic := pubsubClient.Topic(pubsubTopic) return &appEnv{ logger: logger, @@ -73,113 +194,6 @@ func setup(ctx context.Context) (*appEnv, error) { }, nil } -func main() { - ctx := context.Background() - env, err := setup(ctx) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to setup application: %v\n", err) - os.Exit(1) - } - - rcData, err := getRecordCheckerData(ctx, env.ds) - if err != nil { - env.logger.Log(logging.Entry{ - Severity: logging.Critical, - Payload: fmt.Sprintf("failed to get prior run data: %v", err)}) - os.Exit(1) - } - - var wg sync.WaitGroup - ch := make(chan checkRecordResult) - - var newInvalid []string - var resultsWg sync.WaitGroup - resultsWg.Add(1) - go func() { - defer resultsWg.Done() - newInvalid = handleResults(ctx, env.logger, env.topic, ch) - }() - - for _, id := range rcData.InvalidRecords { - wg.Add(1) - go func(id string) { - defer wg.Done() - checkRecord(ctx, env.logger, env.ds, env.bucket, id, nil, ch) - }(id) - } - runStartTime := time.Now().UTC() - query := datastore.NewQuery("Vulnerability") - if rcData.LastRun != nil { - query = query.FilterField("modified", ">", *rcData.LastRun) - } - it := env.ds.Run(ctx, query) - for { - var vuln models.Vulnerability - key, err := it.Next(&vuln) - if errors.Is(err, iterator.Done) { - break - } - if err != nil { - env.logger.Log(logging.Entry{ - Severity: logging.Critical, - Payload: fmt.Sprintf("failed to query vulnerabilities: %v", err), - }) - os.Exit(1) - } - wg.Add(1) - go func(v models.Vulnerability, name string) { - defer wg.Done() - checkRecord(ctx, env.logger, env.ds, env.bucket, name, &v, ch) - }(vuln, key.Name) - } - - wg.Wait() - close(ch) - resultsWg.Wait() - - rcData = recordCheckerData{ - LastRun: &runStartTime, - InvalidRecords: newInvalid, - } - _, err = env.ds.Put(ctx, datastore.NameKey("JobData", "record_checker", nil), &rcData) - if err != nil { - env.logger.Log(logging.Entry{ - Severity: logging.Error, - Payload: fmt.Sprintf("failed to store run data: %v", err), - }) - } -} - -func handleResults(ctx context.Context, logger *logging.Logger, topic *pubsub.Topic, ch <-chan checkRecordResult) []string { - var newInvalid []string - for result := range ch { - if result.Err != nil { - logger.Log(logging.Entry{ - Severity: logging.Error, - Payload: result.Err, - }) - } - if result.NeedsRetry { - newInvalid = append(newInvalid, result.ID) - msg := pubsub.Message{ - Attributes: map[string]string{"type": "gcs_missing", "id": result.ID}, - } - logger.Log(logging.Entry{ - Severity: logging.Info, - Payload: fmt.Sprintf("Publishing gcs_missing for %s", result.ID), - }) - _, err := topic.Publish(ctx, &msg).Get(ctx) - if err != nil { - logger.Log(logging.Entry{ - Severity: logging.Error, - Payload: fmt.Sprintf("failed to publish message for %s: %v", result.ID, err), - }) - } - } - } - return newInvalid -} - func setupLogging(ctx context.Context, projectID string) (*logging.Logger, error) { client, err := logging.NewClient(ctx, projectID) if err != nil { @@ -194,9 +208,8 @@ type recordCheckerData struct { } func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordCheckerData, error) { - infoKey := datastore.NameKey("JobData", "record_checker", nil) var data recordCheckerData - err := cl.Get(ctx, infoKey, &data) + err := cl.Get(ctx, recordCheckerKey, &data) if errors.Is(err, datastore.ErrNoSuchEntity) { err = nil } @@ -246,3 +259,16 @@ func checkRecord(ctx context.Context, logger *logging.Logger, cl *datastore.Clie res.NeedsRetry = true } } + +func main() { + ctx := context.Background() + env, err := setup(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to setup application: %v\n", err) + os.Exit(1) + } + if err := run(ctx, env); err != nil { + fmt.Fprintf(os.Stderr, "failed to run application: %v\n", err) + os.Exit(1) + } +} From 5176ad0772d2b15c3cfe85e4b9e53f0d8185d4e7 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Wed, 10 Sep 2025 13:11:03 +1000 Subject: [PATCH 03/13] break up jobdata --- gcp/workers/record_checker/record_checker.go | 91 +++++++++++++++----- 1 file changed, 69 insertions(+), 22 deletions(-) diff --git a/gcp/workers/record_checker/record_checker.go b/gcp/workers/record_checker/record_checker.go index 738b7f4a5bc..eda525f82c8 100644 --- a/gcp/workers/record_checker/record_checker.go +++ b/gcp/workers/record_checker/record_checker.go @@ -19,9 +19,71 @@ import ( const ( pubsubTopic = "failed-tasks" numWorkers = 50 + + jobDataKind = "JobData" + JobDataLastRun = "record_checker_last_run" + JobDataInvalidRecords = "record_checker_invalid_records" ) -var recordCheckerKey = datastore.NameKey("JobData", "record_checker", nil) +type jobDataLastRunEntity struct { + Value *time.Time `datastore:"value,noindex"` +} +type jobDataInvalidRecordsEntity struct { + Value []string `datastore:"value,noindex"` +} + +type recordCheckerData struct { + lastRun *time.Time + invalidRecords []string +} + +func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordCheckerData, error) { + var data recordCheckerData + + lastRunKey := datastore.NameKey(jobDataKind, JobDataLastRun, nil) + var lr jobDataLastRunEntity + err := cl.Get(ctx, lastRunKey, &lr) + if err != nil && !errors.Is(err, datastore.ErrNoSuchEntity) { + return data, err + } + if err == nil { + data.lastRun = lr.Value + } + + invalidRecordsKey := datastore.NameKey(jobDataKind, JobDataInvalidRecords, nil) + var ir jobDataInvalidRecordsEntity + err = cl.Get(ctx, invalidRecordsKey, &ir) + if err != nil && !errors.Is(err, datastore.ErrNoSuchEntity) { + return data, err + } + if err == nil { + data.invalidRecords = ir.Value + } + + return data, nil +} + +func writeRecordCheckData(ctx context.Context, cl *datastore.Client, data recordCheckerData) error { + _, err := cl.RunInTransaction(ctx, func(tx *datastore.Transaction) error { + lastRunKey := datastore.NameKey(jobDataKind, JobDataLastRun, nil) + lr := jobDataLastRunEntity{Value: data.lastRun} + _, err := tx.Put(lastRunKey, &lr) + if err != nil { + return err + } + + invalidRecordsKey := datastore.NameKey(jobDataKind, JobDataInvalidRecords, nil) + ir := jobDataInvalidRecordsEntity{Value: data.invalidRecords} + _, err = tx.Put(invalidRecordsKey, &ir) + if err != nil { + return err + } + + return nil + }) + + return err +} // run is the main application logic func run(ctx context.Context, env *appEnv) error { @@ -62,7 +124,7 @@ func run(ctx context.Context, env *appEnv) error { } // Queue all invalid records from the previous run. - for _, id := range rcData.InvalidRecords { + for _, id := range rcData.invalidRecords { tasksChan <- struct { ID string Vuln *models.Vulnerability @@ -72,8 +134,8 @@ func run(ctx context.Context, env *appEnv) error { // Queue all new records. runStartTime := time.Now().UTC() query := datastore.NewQuery("Vulnerability") - if rcData.LastRun != nil { - query = query.FilterField("modified", ">", *rcData.LastRun) + if rcData.lastRun != nil { + query = query.FilterField("modified", ">", *rcData.lastRun) } it := env.ds.Run(ctx, query) for { @@ -98,11 +160,10 @@ func run(ctx context.Context, env *appEnv) error { // Update the record checker run data rcData = recordCheckerData{ - LastRun: &runStartTime, - InvalidRecords: newInvalid, + lastRun: &runStartTime, + invalidRecords: newInvalid, } - _, err = env.ds.Put(ctx, recordCheckerKey, &rcData) - if err != nil { + if err := writeRecordCheckData(ctx, env.ds, rcData); err != nil { return fmt.Errorf("failed to store run data: %w", err) } @@ -202,20 +263,6 @@ func setupLogging(ctx context.Context, projectID string) (*logging.Logger, error return client.Logger("record_checker"), nil } -type recordCheckerData struct { - LastRun *time.Time `datastore:"last_run"` - InvalidRecords []string `datastore:"invalid_records"` -} - -func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordCheckerData, error) { - var data recordCheckerData - err := cl.Get(ctx, recordCheckerKey, &data) - if errors.Is(err, datastore.ErrNoSuchEntity) { - err = nil - } - return data, err -} - type checkRecordResult struct { ID string NeedsRetry bool From f178b46dce8da691cda4da4cb862896ec065e1e3 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Wed, 10 Sep 2025 13:23:42 +1000 Subject: [PATCH 04/13] lont --- go/models/validate/validate.py | 39 +++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/go/models/validate/validate.py b/go/models/validate/validate.py index c2aec253ddc..5293063f5a9 100644 --- a/go/models/validate/validate.py +++ b/go/models/validate/validate.py @@ -1,3 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Script to ensure datastore definitions in Python and Go are compatible.""" import datetime import subprocess import sys @@ -7,26 +21,27 @@ import osv.tests from osv import Vulnerability + def main() -> int: # Populate the examples from Python print('(Python) Putting Vulnerability') Vulnerability( - id='CVE-123-456', - source_id='test:path/to/CVE-123-456.json', - modified=datetime.datetime(2025, 1, 2, 3, 4, 5, tzinfo=datetime.UTC), - is_withdrawn=False, - modified_raw=datetime.datetime(2025, 1, 1, 1, 1, 1, tzinfo=datetime.UTC), - alias_raw=['OSV-123-456', 'TEST-123-456'], - related_raw=['CVE-000-000', 'CVE-111-111'], - upstream_raw=['CVE-123-000', 'OSV-123-000'], + id='CVE-123-456', + source_id='test:path/to/CVE-123-456.json', + modified=datetime.datetime(2025, 1, 2, 3, 4, 5, tzinfo=datetime.UTC), + is_withdrawn=False, + modified_raw=datetime.datetime(2025, 1, 1, 1, 1, 1, tzinfo=datetime.UTC), + alias_raw=['OSV-123-456', 'TEST-123-456'], + related_raw=['CVE-000-000', 'CVE-111-111'], + upstream_raw=['CVE-123-000', 'OSV-123-000'], ).put() - + # Run Go program to read the Python-created entities in Go. # And write Go entities. - result = subprocess.run(['go', 'run', './validate.go']) + result = subprocess.run(['go', 'run', './validate.go'], check=False) if result.returncode != 0: return result.returncode - + # Read the Go-created entities in Python. print('(Python) Getting Vulnerability') if Vulnerability.get_by_id('CVE-987-654') is None: @@ -41,4 +56,4 @@ def main() -> int: ret = main() finally: osv.tests.stop_emulator() - sys.exit(ret) \ No newline at end of file + sys.exit(ret) From 7720a1146694b94b924258e2e0061de24830bae2 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Thu, 11 Sep 2025 13:27:39 +1000 Subject: [PATCH 05/13] see you later, emulator --- go/models/validate/validate.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/go/models/validate/validate.py b/go/models/validate/validate.py index 5293063f5a9..d692f967f16 100644 --- a/go/models/validate/validate.py +++ b/go/models/validate/validate.py @@ -50,10 +50,6 @@ def main() -> int: if __name__ == '__main__': - try: - osv.tests.start_datastore_emulator() - with ndb.Client().context(): - ret = main() - finally: - osv.tests.stop_emulator() + with osv.tests.datastore_emulator(), ndb.Client().context(): + ret = main() sys.exit(ret) From e112f5b37b2a3e53806ac8878204f924ac0b0d66 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Thu, 11 Sep 2025 14:17:48 +1000 Subject: [PATCH 06/13] I like to move it --- gcp/workers/record_checker/go.mod | 49 ----- gcp/workers/record_checker/go.sum | 182 ------------------ .../cmd}/record_checker/record_checker.go | 0 go/go.mod | 12 +- go/go.sum | 16 ++ .../{ => internal}/validate/run_validate.sh | 0 go/models/{ => internal}/validate/validate.go | 0 go/models/{ => internal}/validate/validate.py | 0 8 files changed, 26 insertions(+), 233 deletions(-) delete mode 100644 gcp/workers/record_checker/go.mod delete mode 100644 gcp/workers/record_checker/go.sum rename {gcp/workers => go/cmd}/record_checker/record_checker.go (100%) rename go/models/{ => internal}/validate/run_validate.sh (100%) rename go/models/{ => internal}/validate/validate.go (100%) rename go/models/{ => internal}/validate/validate.py (100%) diff --git a/gcp/workers/record_checker/go.mod b/gcp/workers/record_checker/go.mod deleted file mode 100644 index 6b96fa4fc7f..00000000000 --- a/gcp/workers/record_checker/go.mod +++ /dev/null @@ -1,49 +0,0 @@ -module github.com/google/osv.dev/gcp/workers/record_checker - -go 1.23.5 - -replace github.com/google/osv.dev/go => ../../../go - -require ( - cloud.google.com/go/datastore v1.20.0 - cloud.google.com/go/logging v1.11.0 - cloud.google.com/go/pubsub v1.44.0 - cloud.google.com/go/storage v1.43.0 - github.com/google/osv.dev/go v0.0.0-00010101000000-000000000000 - google.golang.org/api v0.203.0 -) - -require ( - cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/auth v0.9.9 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect - cloud.google.com/go/compute/metadata v0.5.2 // indirect - cloud.google.com/go/iam v1.2.1 // indirect - cloud.google.com/go/longrunning v0.6.1 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/s2a-go v0.1.8 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.29.0 // indirect - go.opentelemetry.io/otel/metric v1.29.0 // indirect - go.opentelemetry.io/otel/trace v1.29.0 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect - golang.org/x/time v0.7.0 // indirect - google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.35.1 // indirect -) diff --git a/gcp/workers/record_checker/go.sum b/gcp/workers/record_checker/go.sum deleted file mode 100644 index a98b7430bfd..00000000000 --- a/gcp/workers/record_checker/go.sum +++ /dev/null @@ -1,182 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= -cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= -cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ= -cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= -cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= -cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= -cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= -cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= -cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM= -cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew= -cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= -cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= -cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY= -cloud.google.com/go/kms v1.20.0/go.mod h1:/dMbFF1tLLFnQV44AoI2GlotbjowyUfgVwezxW291fM= -cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= -cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= -cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= -cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= -cloud.google.com/go/pubsub v1.44.0 h1:pLaMJVDTlnUDIKT5L0k53YyLszfBbGoUBo/IqDK/fEI= -cloud.google.com/go/pubsub v1.44.0/go.mod h1:BD4a/kmE8OePyHoa1qAHEw1rMzXX+Pc8Se54T/8mc3I= -cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= -cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= -github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= -github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= -github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= -go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= -go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= -go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= -go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= -go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= -go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU= -google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE= -google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE= -google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= -google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= -google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/gcp/workers/record_checker/record_checker.go b/go/cmd/record_checker/record_checker.go similarity index 100% rename from gcp/workers/record_checker/record_checker.go rename to go/cmd/record_checker/record_checker.go diff --git a/go/go.mod b/go/go.mod index 180afd91914..8178085a5d7 100644 --- a/go/go.mod +++ b/go/go.mod @@ -2,18 +2,27 @@ module github.com/google/osv.dev/go go 1.23.5 -require cloud.google.com/go/datastore v1.20.0 +require ( + cloud.google.com/go/datastore v1.20.0 + cloud.google.com/go/logging v1.11.0 + cloud.google.com/go/pubsub v1.44.0 + cloud.google.com/go/storage v1.43.0 + google.golang.org/api v0.203.0 +) require ( cloud.google.com/go v0.116.0 // indirect cloud.google.com/go/auth v0.9.9 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.2 // indirect + cloud.google.com/go/iam v1.2.1 // indirect + cloud.google.com/go/longrunning v0.6.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/s2a-go v0.1.8 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect go.opencensus.io v0.24.0 // indirect @@ -29,7 +38,6 @@ require ( golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.7.0 // indirect - google.golang.org/api v0.203.0 // indirect google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect diff --git a/go/go.sum b/go/go.sum index bfa11a2a2ef..a98b7430bfd 100644 --- a/go/go.sum +++ b/go/go.sum @@ -9,6 +9,18 @@ cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixA cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM= cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew= +cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= +cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY= +cloud.google.com/go/kms v1.20.0/go.mod h1:/dMbFF1tLLFnQV44AoI2GlotbjowyUfgVwezxW291fM= +cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= +cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= +cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= +cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= +cloud.google.com/go/pubsub v1.44.0 h1:pLaMJVDTlnUDIKT5L0k53YyLszfBbGoUBo/IqDK/fEI= +cloud.google.com/go/pubsub v1.44.0/go.mod h1:BD4a/kmE8OePyHoa1qAHEw1rMzXX+Pc8Se54T/8mc3I= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -51,6 +63,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -71,6 +85,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= +go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= diff --git a/go/models/validate/run_validate.sh b/go/models/internal/validate/run_validate.sh similarity index 100% rename from go/models/validate/run_validate.sh rename to go/models/internal/validate/run_validate.sh diff --git a/go/models/validate/validate.go b/go/models/internal/validate/validate.go similarity index 100% rename from go/models/validate/validate.go rename to go/models/internal/validate/validate.go diff --git a/go/models/validate/validate.py b/go/models/internal/validate/validate.py similarity index 100% rename from go/models/validate/validate.py rename to go/models/internal/validate/validate.py From 1e23dc2fd4924ae2e917153d32768fb770424731 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Tue, 16 Sep 2025 14:17:26 +1000 Subject: [PATCH 07/13] loggogogogg --- go/cmd/record_checker/record_checker.go | 64 ++++------- go/go.mod | 16 ++- go/go.sum | 34 +++++- go/logger/localhandler.go | 69 +++++++++++ go/logger/logger.go | 146 ++++++++++++++++++++++++ 5 files changed, 280 insertions(+), 49 deletions(-) create mode 100644 go/logger/localhandler.go create mode 100644 go/logger/logger.go diff --git a/go/cmd/record_checker/record_checker.go b/go/cmd/record_checker/record_checker.go index eda525f82c8..8853dd27364 100644 --- a/go/cmd/record_checker/record_checker.go +++ b/go/cmd/record_checker/record_checker.go @@ -4,14 +4,15 @@ import ( "context" "errors" "fmt" + "log/slog" "os" "sync" "time" "cloud.google.com/go/datastore" - "cloud.google.com/go/logging" "cloud.google.com/go/pubsub" "cloud.google.com/go/storage" + "github.com/google/osv.dev/go/logger" "github.com/google/osv.dev/go/models" "google.golang.org/api/iterator" ) @@ -46,7 +47,9 @@ func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordChec if err != nil && !errors.Is(err, datastore.ErrNoSuchEntity) { return data, err } - if err == nil { + if errors.Is(err, datastore.ErrNoSuchEntity) { + logger.Info("no prior record checker last run time found") + } else { data.lastRun = lr.Value } @@ -56,7 +59,9 @@ func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordChec if err != nil && !errors.Is(err, datastore.ErrNoSuchEntity) { return data, err } - if err == nil { + if errors.Is(err, datastore.ErrNoSuchEntity) { + logger.Info("no prior record checker invalid records list found") + } else { data.invalidRecords = ir.Value } @@ -105,7 +110,7 @@ func run(ctx context.Context, env *appEnv) error { go func() { defer resultsWg.Done() for result := range resultsChan { - if handleResult(ctx, env.logger, env.topic, result) { + if handleResult(ctx, env.topic, result) { newInvalid = append(newInvalid, result.ID) } } @@ -118,13 +123,14 @@ func run(ctx context.Context, env *appEnv) error { go func() { defer workerWg.Done() for task := range tasksChan { - checkRecord(ctx, env.logger, env.ds, env.bucket, task.ID, task.Vuln, resultsChan) + checkRecord(ctx, env.ds, env.bucket, task.ID, task.Vuln, resultsChan) } }() } // Queue all invalid records from the previous run. for _, id := range rcData.invalidRecords { + logger.Debug("checking previously invalid record", slog.String("id", id)) tasksChan <- struct { ID string Vuln *models.Vulnerability @@ -147,6 +153,7 @@ func run(ctx context.Context, env *appEnv) error { if err != nil { return fmt.Errorf("failed to query vulnerabilities: %w", err) } + logger.Debug("checking newly modified record", slog.String("id", key.Name)) tasksChan <- struct { ID string Vuln *models.Vulnerability @@ -170,29 +177,20 @@ func run(ctx context.Context, env *appEnv) error { return nil } -func handleResult(ctx context.Context, logger *logging.Logger, topic *pubsub.Topic, result checkRecordResult) bool { +func handleResult(ctx context.Context, topic *pubsub.Topic, result checkRecordResult) bool { wasInvalid := false if result.Err != nil { - logger.Log(logging.Entry{ - Severity: logging.Error, - Payload: fmt.Sprintf("failed to process record %s: %v", result.ID, result.Err), - }) + logger.Error("failed to process record", slog.String("id", result.ID), slog.Any("err", result.Err)) } if result.NeedsRetry { wasInvalid = true msg := pubsub.Message{ Attributes: map[string]string{"type": "gcs_missing", "id": result.ID}, } - logger.Log(logging.Entry{ - Severity: logging.Info, - Payload: fmt.Sprintf("Publishing gcs_missing for %s", result.ID), - }) + logger.Info("publishing gcs_missing message", slog.String("id", result.ID)) _, err := topic.Publish(ctx, &msg).Get(ctx) if err != nil { - logger.Log(logging.Entry{ - Severity: logging.Error, - Payload: fmt.Sprintf("failed to publish message for %s: %v", result.ID, err), - }) + logger.Error("failed publishing message", slog.String("id", result.ID), slog.Any("err", err)) } } return wasInvalid @@ -200,7 +198,6 @@ func handleResult(ctx context.Context, logger *logging.Logger, topic *pubsub.Top // appEnv holds the clients and other environment-specific resources. type appEnv struct { - logger *logging.Logger bucket *storage.BucketHandle ds *datastore.Client topic *pubsub.Topic @@ -208,26 +205,20 @@ type appEnv struct { // setup initializes the application environment. func setup(ctx context.Context) (*appEnv, error) { + logger.InitGlobalLogger() projectID, ok := os.LookupEnv("GOOGLE_CLOUD_PROJECT") if !ok { return nil, errors.New("GOOGLE_CLOUD_PROJECT must be set") } - logger, err := setupLogging(ctx, projectID) - if err != nil { - return nil, fmt.Errorf("failed to setup logging: %w", err) - } - bucketName, ok := os.LookupEnv("OSV_VULNERABILITIES_BUCKET") if !ok { err := errors.New("OSV_VULNERABILITIES_BUCKET must be set") - logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) return nil, err } stClient, err := storage.NewClient(ctx) if err != nil { err = fmt.Errorf("failed to create storage client: %w", err) - logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) return nil, err } bucket := stClient.Bucket(bucketName) @@ -235,41 +226,30 @@ func setup(ctx context.Context) (*appEnv, error) { dsClient, err := datastore.NewClient(ctx, projectID) if err != nil { err = fmt.Errorf("failed to create datastore client: %w", err) - logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) return nil, err } pubsubClient, err := pubsub.NewClient(ctx, projectID) if err != nil { err = fmt.Errorf("failed to create pubsub client: %w", err) - logger.Log(logging.Entry{Severity: logging.Critical, Payload: err.Error()}) return nil, err } topic := pubsubClient.Topic(pubsubTopic) return &appEnv{ - logger: logger, bucket: bucket, ds: dsClient, topic: topic, }, nil } -func setupLogging(ctx context.Context, projectID string) (*logging.Logger, error) { - client, err := logging.NewClient(ctx, projectID) - if err != nil { - return nil, fmt.Errorf("failed to create logging client: %w", err) - } - return client.Logger("record_checker"), nil -} - type checkRecordResult struct { ID string NeedsRetry bool Err error } -func checkRecord(ctx context.Context, logger *logging.Logger, cl *datastore.Client, bucket *storage.BucketHandle, id string, vuln *models.Vulnerability, out chan<- checkRecordResult) { +func checkRecord(ctx context.Context, cl *datastore.Client, bucket *storage.BucketHandle, id string, vuln *models.Vulnerability, out chan<- checkRecordResult) { res := checkRecordResult{ID: id} // Send the result when the function returns. defer func() { out <- res }() @@ -311,11 +291,11 @@ func main() { ctx := context.Background() env, err := setup(ctx) if err != nil { - fmt.Fprintf(os.Stderr, "failed to setup application: %v\n", err) - os.Exit(1) + logger.Fatal("failed setting up environment", slog.Any("err", err)) } + logger.Info("starting record checker") if err := run(ctx, env); err != nil { - fmt.Fprintf(os.Stderr, "failed to run application: %v\n", err) - os.Exit(1) + logger.Fatal("failed running record checker", slog.Any("err", err)) } + logger.Info("record checker done") } diff --git a/go/go.mod b/go/go.mod index 8178085a5d7..c0fef81437f 100644 --- a/go/go.mod +++ b/go/go.mod @@ -4,9 +4,9 @@ go 1.23.5 require ( cloud.google.com/go/datastore v1.20.0 - cloud.google.com/go/logging v1.11.0 cloud.google.com/go/pubsub v1.44.0 cloud.google.com/go/storage v1.43.0 + github.com/charmbracelet/lipgloss v1.1.0 google.golang.org/api v0.203.0 ) @@ -16,7 +16,11 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.2 // indirect cloud.google.com/go/iam v1.2.1 // indirect - cloud.google.com/go/longrunning v0.6.1 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/x/ansi v0.8.0 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -25,6 +29,12 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect @@ -35,7 +45,7 @@ require ( golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.30.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.7.0 // indirect google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect diff --git a/go/go.sum b/go/go.sum index a98b7430bfd..f250566a24c 100644 --- a/go/go.sum +++ b/go/go.sum @@ -13,8 +13,6 @@ cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY= cloud.google.com/go/kms v1.20.0/go.mod h1:/dMbFF1tLLFnQV44AoI2GlotbjowyUfgVwezxW291fM= -cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= -cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= cloud.google.com/go/pubsub v1.44.0 h1:pLaMJVDTlnUDIKT5L0k53YyLszfBbGoUBo/IqDK/fEI= @@ -22,7 +20,19 @@ cloud.google.com/go/pubsub v1.44.0/go.mod h1:BD4a/kmE8OePyHoa1qAHEw1rMzXX+Pc8Se5 cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= +github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -74,9 +84,20 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gT github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -85,6 +106,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= @@ -106,6 +129,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -129,8 +154,9 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= diff --git a/go/logger/localhandler.go b/go/logger/localhandler.go new file mode 100644 index 00000000000..1673429dc62 --- /dev/null +++ b/go/logger/localhandler.go @@ -0,0 +1,69 @@ +package logger + +import ( + "context" + "fmt" + "io" + "log/slog" + "strings" + + "github.com/charmbracelet/lipgloss" +) + +var ( + levelColors = map[slog.Level]lipgloss.Color{ + slog.LevelDebug: lipgloss.Color("5"), // Purple + slog.LevelInfo: lipgloss.Color("4"), // Blue + slog.LevelWarn: lipgloss.Color("3"), // Yellow + slog.LevelError: lipgloss.Color("1"), // Red + } + levelStyle = lipgloss.NewStyle().Width(8).Bold(true) + messageStyle = lipgloss.NewStyle() + keyStyle = lipgloss.NewStyle(). + Foreground(lipgloss.Color("6")). // Cyan + Bold(true) + valueStyle = lipgloss.NewStyle() +) + +type localHandler struct { + w io.Writer +} + +func newLocalHandler(w io.Writer) *localHandler { + return &localHandler{ + w: w, + } +} + +func (h *localHandler) Enabled(_ context.Context, _ slog.Level) bool { + return true +} + +func (h *localHandler) Handle(_ context.Context, r slog.Record) error { + // INFO: message foo=bar + sb := &strings.Builder{} + fmt.Fprint(sb, levelStyle.Foreground(levelColors[r.Level]).Render(r.Level.String()+":")) + fmt.Fprint(sb, messageStyle.Render(r.Message)) + r.Attrs(func(a slog.Attr) bool { + keyStyle := keyStyle + if a.Key == "err" || a.Key == "error" { + // Make the error key bright red. + keyStyle = keyStyle.Foreground(lipgloss.Color("9")) + } + fmt.Fprint(sb, " "+keyStyle.Render(a.Key+"=")) + fmt.Fprint(sb, valueStyle.Render(fmt.Sprintf("%v", a.Value))) + + return true + }) + _, err := fmt.Fprintln(h.w, sb.String()) + + return err +} + +func (h *localHandler) WithAttrs(_ []slog.Attr) slog.Handler { + panic("not implemented") +} + +func (h *localHandler) WithGroup(_ string) slog.Handler { + panic("not implemented") +} diff --git a/go/logger/logger.go b/go/logger/logger.go new file mode 100644 index 00000000000..a26cb27e4ab --- /dev/null +++ b/go/logger/logger.go @@ -0,0 +1,146 @@ +// Package logger provides a slog logging wrapper that all packages within vulnfeeds should use to log output. +package logger + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "runtime" + "time" +) + +var slogLogger *slog.Logger + +// InitGlobalLogger initializes the global slog logger. +func InitGlobalLogger() { + if slogLogger != nil { + // Logger is already initialized. + return + } + + inGKE := os.Getenv("KUBERNETES_SERVICE_HOST") != "" + inCloudRun := os.Getenv("K_SERVICE") != "" + inCloud := inGKE || inCloudRun + var handler slog.Handler + if inCloud { + opts := &slog.HandlerOptions{ + // AddSource adds the source code position to the log output, which is invaluable for debugging. + // Google Cloud Logging will automatically parse this into the `sourceLocation` field. + AddSource: true, + // ReplaceAttr is used to customize log attributes. We use it here to make the output + // perfectly align with what Google Cloud Logging expects for structured logs. + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + // Remap the default "level" key to "severity" for Google Cloud Logging. + if a.Key == slog.LevelKey { + level := a.Value.Any().(slog.Level) + var levelStr string + switch level { + case slog.LevelDebug: + levelStr = "DEBUG" + case slog.LevelInfo: + levelStr = "INFO" + case slog.LevelWarn: + levelStr = "WARNING" + case slog.LevelError: + levelStr = "ERROR" + default: + levelStr = "DEFAULT" + } + + return slog.String("severity", levelStr) + } + // Remap the default "msg" key to "message" for better compatibility. + if a.Key == slog.MessageKey { + return slog.Attr{Key: "message", Value: a.Value} + } + // Remap the default "source" key to "sourceLocation", and trim file path to just file name. + if a.Key == slog.SourceKey { + source := a.Value.Any().(*slog.Source) + source.File = filepath.Base(source.File) + + return slog.Attr{Key: "sourceLocation", Value: slog.AnyValue(source)} + } + + return a + }, + } + + // A JSONHandler writing to stdout is the standard and correct way to log in GKE. + handler = slog.NewJSONHandler(os.Stdout, opts) + } else { + handler = newLocalHandler(os.Stdout) + } + slogLogger = slog.New(handler) +} + +func log(level slog.Level, msg string, a []any) { + var pcs [1]uintptr + runtime.Callers(3, pcs[:]) // skip [Callers, log, Info/Warn/etc] + r := slog.NewRecord(time.Now(), level, msg, pcs[0]) + r.Add(a...) + //nolint:errcheck + slogLogger.Handler().Handle(context.Background(), r) +} + +// Debug prints a Debug level log. +// +//nolint:contextcheck,nolintlint +func Debug(msg string, a ...any) { + if slogLogger == nil { + InitGlobalLogger() // Initialize with defaults if not already done. + } + log(slog.LevelDebug, msg, a) +} + +// Info prints an Info level log. +// +//nolint:contextcheck,nolintlint +func Info(msg string, a ...any) { + if slogLogger == nil { + InitGlobalLogger() // Initialize with defaults if not already done. + } + log(slog.LevelInfo, msg, a) +} + +// Warn prints a Warning level log. +// +//nolint:contextcheck,nolintlint +func Warn(msg string, a ...any) { + if slogLogger == nil { + InitGlobalLogger() // Initialize with defaults if not already done. + } + log(slog.LevelWarn, msg, a) +} + +// Error prints an Error level log. +// +//nolint:contextcheck,nolintlint +func Error(msg string, a ...any) { + if slogLogger == nil { + InitGlobalLogger() // Initialize with defaults if not already done. + } + log(slog.LevelError, msg, a) +} + +// Fatal prints an Error level log and then exits the program. +// +//nolint:contextcheck,nolintlint +func Fatal(msg string, a ...any) { + if slogLogger == nil { + InitGlobalLogger() // Initialize with defaults if not already done. + } + log(slog.LevelError, msg, a) + os.Exit(1) +} + +// Panic prints an Error level log and then panics. +// +//nolint:contextcheck,nolintlint +func Panic(msg string, a ...any) { + if slogLogger == nil { + InitGlobalLogger() // Initialize with defaults if not already done. + } + log(slog.LevelError, msg, a) + panic(msg) +} From e036d1ab428f28d367acd60ad392b6482e79eb16 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Tue, 16 Sep 2025 14:29:38 +1000 Subject: [PATCH 08/13] some tidying --- go/cmd/record_checker/record_checker.go | 76 ++++++++++++++----------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/go/cmd/record_checker/record_checker.go b/go/cmd/record_checker/record_checker.go index 8853dd27364..8f00ea516e7 100644 --- a/go/cmd/record_checker/record_checker.go +++ b/go/cmd/record_checker/record_checker.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "os" + "strconv" "sync" "time" @@ -19,7 +20,9 @@ import ( const ( pubsubTopic = "failed-tasks" - numWorkers = 50 + // defaultNumWorkers is the default number of concurrent workers to use. + // This can be overridden by setting the NUM_WORKERS environment variable. + defaultNumWorkers = 50 jobDataKind = "JobData" JobDataLastRun = "record_checker_last_run" @@ -44,11 +47,12 @@ func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordChec lastRunKey := datastore.NameKey(jobDataKind, JobDataLastRun, nil) var lr jobDataLastRunEntity err := cl.Get(ctx, lastRunKey, &lr) - if err != nil && !errors.Is(err, datastore.ErrNoSuchEntity) { - return data, err - } - if errors.Is(err, datastore.ErrNoSuchEntity) { - logger.Info("no prior record checker last run time found") + if err != nil { + if errors.Is(err, datastore.ErrNoSuchEntity) { + logger.Info("no prior record checker last run time found") + } else { + return data, fmt.Errorf("failed to get %s: %w", JobDataLastRun, err) + } } else { data.lastRun = lr.Value } @@ -56,11 +60,12 @@ func getRecordCheckerData(ctx context.Context, cl *datastore.Client) (recordChec invalidRecordsKey := datastore.NameKey(jobDataKind, JobDataInvalidRecords, nil) var ir jobDataInvalidRecordsEntity err = cl.Get(ctx, invalidRecordsKey, &ir) - if err != nil && !errors.Is(err, datastore.ErrNoSuchEntity) { - return data, err - } - if errors.Is(err, datastore.ErrNoSuchEntity) { - logger.Info("no prior record checker invalid records list found") + if err != nil { + if errors.Is(err, datastore.ErrNoSuchEntity) { + logger.Info("no prior record checker invalid records list found") + } else { + return data, fmt.Errorf("failed to get %s: %w", JobDataInvalidRecords, err) + } } else { data.invalidRecords = ir.Value } @@ -74,14 +79,14 @@ func writeRecordCheckData(ctx context.Context, cl *datastore.Client, data record lr := jobDataLastRunEntity{Value: data.lastRun} _, err := tx.Put(lastRunKey, &lr) if err != nil { - return err + return fmt.Errorf("failed to write %s: %w", JobDataLastRun, err) } invalidRecordsKey := datastore.NameKey(jobDataKind, JobDataInvalidRecords, nil) ir := jobDataInvalidRecordsEntity{Value: data.invalidRecords} _, err = tx.Put(invalidRecordsKey, &ir) if err != nil { - return err + return fmt.Errorf("failed to write %s: %w", JobDataInvalidRecords, err) } return nil @@ -90,6 +95,11 @@ func writeRecordCheckData(ctx context.Context, cl *datastore.Client, data record return err } +type checkTask struct { + ID string + Vuln *models.Vulnerability +} + // run is the main application logic func run(ctx context.Context, env *appEnv) error { rcData, err := getRecordCheckerData(ctx, env.ds) @@ -97,10 +107,7 @@ func run(ctx context.Context, env *appEnv) error { return fmt.Errorf("failed to get prior run data: %w", err) } - tasksChan := make(chan struct { - ID string - Vuln *models.Vulnerability - }) + tasksChan := make(chan checkTask) resultsChan := make(chan checkRecordResult) // Start the results handler @@ -118,7 +125,7 @@ func run(ctx context.Context, env *appEnv) error { // Start the worker pool var workerWg sync.WaitGroup - for range numWorkers { + for range env.numWorkers { workerWg.Add(1) go func() { defer workerWg.Done() @@ -131,10 +138,7 @@ func run(ctx context.Context, env *appEnv) error { // Queue all invalid records from the previous run. for _, id := range rcData.invalidRecords { logger.Debug("checking previously invalid record", slog.String("id", id)) - tasksChan <- struct { - ID string - Vuln *models.Vulnerability - }{ID: id} + tasksChan <- checkTask{ID: id} } // Queue all new records. @@ -154,10 +158,7 @@ func run(ctx context.Context, env *appEnv) error { return fmt.Errorf("failed to query vulnerabilities: %w", err) } logger.Debug("checking newly modified record", slog.String("id", key.Name)) - tasksChan <- struct { - ID string - Vuln *models.Vulnerability - }{ID: key.Name, Vuln: &vuln} + tasksChan <- checkTask{ID: key.Name, Vuln: &vuln} } // Wait for all tasks to finish processing close(tasksChan) @@ -198,9 +199,10 @@ func handleResult(ctx context.Context, topic *pubsub.Topic, result checkRecordRe // appEnv holds the clients and other environment-specific resources. type appEnv struct { - bucket *storage.BucketHandle - ds *datastore.Client - topic *pubsub.Topic + bucket *storage.BucketHandle + ds *datastore.Client + topic *pubsub.Topic + numWorkers int } // setup initializes the application environment. @@ -236,10 +238,20 @@ func setup(ctx context.Context) (*appEnv, error) { } topic := pubsubClient.Topic(pubsubTopic) + numWorkers := defaultNumWorkers + if numWorkersStr, ok := os.LookupEnv("NUM_WORKERS"); ok { + if i, err := strconv.Atoi(numWorkersStr); err == nil { + numWorkers = i + } else { + logger.Warn("invalid NUM_WORKERS value, using default", slog.String("value", numWorkersStr)) + } + } + return &appEnv{ - bucket: bucket, - ds: dsClient, - topic: topic, + bucket: bucket, + ds: dsClient, + topic: topic, + numWorkers: numWorkers, }, nil } From dc81715f9cd584de4b7f928b431401144fa8df77 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Tue, 16 Sep 2025 14:55:23 +1000 Subject: [PATCH 09/13] make it get deployed --- deployment/build-and-stage.yaml | 18 ++++++++++- .../oss-vdb-test/kustomization.yaml | 1 + .../oss-vdb-test/record-checker.yaml | 30 ++++++++++++++++++ go/cmd/recordchecker/Dockerfile | 31 +++++++++++++++++++ .../recordchecker.go} | 0 5 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 deployment/clouddeploy/gke-workers/environments/oss-vdb-test/record-checker.yaml create mode 100644 go/cmd/recordchecker/Dockerfile rename go/cmd/{record_checker/record_checker.go => recordchecker/recordchecker.go} (100%) diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index a6aaa4be504..ed33ddf8681 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -107,6 +107,20 @@ steps: args: ['push', '--all-tags', 'gcr.io/oss-vdb/recoverer'] waitFor: ['build-recoverer', 'cloud-build-queue'] +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/oss-vdb/record-checker:latest || exit 0'] + id: 'pull-record-checker' + waitFor: ['setup'] +- name: gcr.io/cloud-builders/docker + args: ['build', '-t', 'gcr.io/oss-vdb/record-checker:latest', '-t', 'gcr.io/oss-vdb/record-checker:$COMMIT_SHA', '.'] + dir: 'go' + id: 'build-record-checker' + waitFor: ['pull-record-checker'] +- name: gcr.io/cloud-builders/docker + args: ['push', '--all-tags', 'gcr.io/oss-vdb/record-checker'] + waitFor: ['build-record-checker', 'cloud-build-queue'] + # Build/push staging-api-test images to gcr.io/oss-vdb-test. - name: gcr.io/cloud-builders/docker args: ['build', '-t', 'gcr.io/oss-vdb-test/staging-api-test:latest', '-t', 'gcr.io/oss-vdb-test/staging-api-test:$COMMIT_SHA', '.'] @@ -316,7 +330,8 @@ steps: nvd-cve-osv=gcr.io/oss-vdb/nvd-cve-osv:$COMMIT_SHA,\ nvd-mirror=gcr.io/oss-vdb/nvd-mirror:$COMMIT_SHA,\ recoverer=gcr.io/oss-vdb/recoverer:$COMMIT_SHA,\ - cve5-to-osv=gcr.io/oss-vdb/cve5-to-osv:$COMMIT_SHA" + cve5-to-osv=gcr.io/oss-vdb/cve5-to-osv:$COMMIT_SHA,\ + record-checker=gcr.io/oss-vdb/record-checker:$COMMIT_SHA" ] dir: deployment/clouddeploy/gke-workers @@ -374,3 +389,4 @@ images: - 'gcr.io/oss-vdb-test/osv-linter:$COMMIT_SHA' - 'gcr.io/oss-vdb/recoverer:$COMMIT_SHA' - 'gcr.io/oss-vdb/cve5-to-osv:$COMMIT_SHA' +- 'gcr.io/oss-vdb/record-checker:$COMMIT_SHA' diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml index de86477dac3..53000a8477e 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml @@ -3,6 +3,7 @@ resources: - staging-api-test.yaml - osv-linter.yaml - cve5-to-osv.yaml +- record-checker.yaml patches: - path: workers.yaml - path: scaler.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/record-checker.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/record-checker.yaml new file mode 100644 index 00000000000..1e996d51f9d --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/record-checker.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: record-checker + labels: + cronLastSuccessfulTimeMins: "90" +spec: + schedule: "10/15 * * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: record-checker + image: record-checker + env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb-test + - name: OSV_VULNERABILITIES_BUCKET + value: osv-test-vulnerabilities + imagePullPolicy: Always + resources: + requests: + cpu: "1" + memory: "1G" + limits: + cpu: "1" + memory: "2G" + restartPolicy: Never diff --git a/go/cmd/recordchecker/Dockerfile b/go/cmd/recordchecker/Dockerfile new file mode 100644 index 00000000000..08261093b58 --- /dev/null +++ b/go/cmd/recordchecker/Dockerfile @@ -0,0 +1,31 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.25.0-alpine@sha256:f18a072054848d87a8077455f0ac8a25886f2397f88bfdd222d6fafbb5bba440 AS build + +WORKDIR /src + +COPY ./go.mod /src/go.mod +COPY ./go.sum /src/go.sum +RUN go mod download && go mod verify + + +COPY ./ /src/ +RUN CGO_ENABLED=0 go build -o recordchecker ./cmd/recordchecker/ + +FROM gcr.io/distroless/static-debian12@sha256:87bce11be0af225e4ca761c40babb06d6d559f5767fbf7dc3c47f0f1a466b92c + +COPY --from=build /src/recordchecker / + +ENTRYPOINT ["/recordchecker"] diff --git a/go/cmd/record_checker/record_checker.go b/go/cmd/recordchecker/recordchecker.go similarity index 100% rename from go/cmd/record_checker/record_checker.go rename to go/cmd/recordchecker/recordchecker.go From dd095091a25434f9c21976658d4324a76c21c2f8 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Tue, 16 Sep 2025 15:02:16 +1000 Subject: [PATCH 10/13] remember to actually run tests --- run_tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_tests.sh b/run_tests.sh index 31cddff3d0c..344999c61bb 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -11,3 +11,6 @@ poetry run python -m unittest osv.models_test # Run all osv.ecosystems tests poetry run python -m unittest discover osv/ecosystems/ "*_test.py" . + +# Run the validation for the go/python datastore models +cd ./go/models/internal/validate/ && ./run_validate.sh From 27b0d62ef6855f6023e6393c907505989abdd687 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Tue, 16 Sep 2025 15:14:25 +1000 Subject: [PATCH 11/13] vulnerabilities? in my osv.dev? more likely than you'd think --- go/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index c0fef81437f..2ed18ab0c3f 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,6 +1,6 @@ module github.com/google/osv.dev/go -go 1.23.5 +go 1.24.6 require ( cloud.google.com/go/datastore v1.20.0 From 96ac7f7356b722c132a13fa21d40a1fc5e77e1cd Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Thu, 18 Sep 2025 16:15:34 +1000 Subject: [PATCH 12/13] comments --- go/.golangci.yaml | 144 ++++++++++++++++++++++++ go/cmd/recordchecker/recordchecker.go | 65 +++++------ go/models/internal/validate/validate.go | 3 + 3 files changed, 180 insertions(+), 32 deletions(-) create mode 100644 go/.golangci.yaml diff --git a/go/.golangci.yaml b/go/.golangci.yaml new file mode 100644 index 00000000000..c66c97b37a0 --- /dev/null +++ b/go/.golangci.yaml @@ -0,0 +1,144 @@ +version: "2" + +linters: + default: all + # prettier-ignore + disable: + - forbidigo + - paralleltest + - tparallel + - cyclop # + - depguard # Too annoying + - err113 # will re-add later (another-rex) + - exhaustruct # overkill (g-rath) + - forcetypeassert # too hard (g-rath) + - funlen # + - funcorder # + - gochecknoglobals # disagree with, for non changing variables (another-rex) + - gocognit # + - goconst # not everything should be a constant + - gocyclo # + - godot # comments are fine without full stops (g-rath) + - godox # to-do comments are fine (g-rath) + - ireturn # disagree with, sort of (g-rath) + - lll # line length is hard (g-rath) + - maintidx # + - mnd # not every number is magic (g-rath) + - nestif # + - noctx # Most of these don't need a context + - noinlineerr # + - nonamedreturns # disagree with, for now (another-rex) + - tagliatelle # we're parsing data from external sources (g-rath) + - testpackage # will re-add later (another-rex) + - varnamelen # maybe later (g-rath) + - wrapcheck # too difficult, will re-add later (another-rex) + - wsl # disagree with, for now (g-rath) + - wsl_v5 # disagree with, for now (g-rath) + settings: + exhaustive: + default-signifies-exhaustive: true + gocritic: + disabled-checks: + - ifElseChain + nlreturn: + block-size: 2 + revive: + rules: + - name: increment-decrement + disabled: true + - name: blank-imports + disabled: false + - name: context-as-argument + disabled: false + - name: context-keys-type + disabled: false + - name: dot-imports + disabled: false + - name: empty-block + disabled: false + - name: error-naming + disabled: false + - name: error-return + disabled: false + - name: error-strings + disabled: false + - name: errorf + disabled: false + - name: exported + disabled: false + arguments: + # TODO: get these all enabled + - "check-private-receivers" + # - "check-public-interface" + - "disable-checks-on-constants" + - "disable-checks-on-functions" + - "disable-checks-on-methods" + - "disable-checks-on-types" + - "disable-checks-on-variables" + - name: import-alias-naming + disabled: false + - name: import-shadowing + disabled: false + - name: indent-error-flow + disabled: false + - name: package-comments + disabled: false + - name: range + disabled: false + - name: receiver-naming + disabled: false + - name: redefines-builtin-id + disabled: false + - name: redundant-test-main-exit + disabled: false + - name: superfluous-else + disabled: false + - name: time-naming + disabled: false + - name: unexported-return + disabled: false + - name: unreachable-code + disabled: false + - name: unused-parameter + disabled: false + - name: use-any + disabled: false + - name: var-declaration + disabled: false + - name: var-naming + disabled: false + arguments: + - [] # AllowList + - [] # DenyList + - - skip-package-name-checks: true + exclusions: + generated: lax + presets: + - common-false-positives + - legacy + - std-error-handling + rules: + - path: _test\.go + linters: + - dupl + - path-except: _test\.go + text: use `testutility.GetCurrentWorkingDirectory` + paths: + - third_party$ + - builtin$ + - examples$ + +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/go/cmd/recordchecker/recordchecker.go b/go/cmd/recordchecker/recordchecker.go index 8f00ea516e7..7dfd9b59ad2 100644 --- a/go/cmd/recordchecker/recordchecker.go +++ b/go/cmd/recordchecker/recordchecker.go @@ -96,8 +96,8 @@ func writeRecordCheckData(ctx context.Context, cl *datastore.Client, data record } type checkTask struct { - ID string - Vuln *models.Vulnerability + id string + vuln *models.Vulnerability } // run is the main application logic @@ -113,12 +113,12 @@ func run(ctx context.Context, env *appEnv) error { // Start the results handler var resultsWg sync.WaitGroup resultsWg.Add(1) - var newInvalid []string + var newInvalids []string go func() { defer resultsWg.Done() for result := range resultsChan { if handleResult(ctx, env.topic, result) { - newInvalid = append(newInvalid, result.ID) + newInvalids = append(newInvalids, result.id) } } }() @@ -130,7 +130,7 @@ func run(ctx context.Context, env *appEnv) error { go func() { defer workerWg.Done() for task := range tasksChan { - checkRecord(ctx, env.ds, env.bucket, task.ID, task.Vuln, resultsChan) + resultsChan <- checkRecord(ctx, env.ds, env.bucket, task.id, task.vuln) } }() } @@ -138,7 +138,7 @@ func run(ctx context.Context, env *appEnv) error { // Queue all invalid records from the previous run. for _, id := range rcData.invalidRecords { logger.Debug("checking previously invalid record", slog.String("id", id)) - tasksChan <- checkTask{ID: id} + tasksChan <- checkTask{id: id} } // Queue all new records. @@ -158,7 +158,7 @@ func run(ctx context.Context, env *appEnv) error { return fmt.Errorf("failed to query vulnerabilities: %w", err) } logger.Debug("checking newly modified record", slog.String("id", key.Name)) - tasksChan <- checkTask{ID: key.Name, Vuln: &vuln} + tasksChan <- checkTask{id: key.Name, vuln: &vuln} } // Wait for all tasks to finish processing close(tasksChan) @@ -169,7 +169,7 @@ func run(ctx context.Context, env *appEnv) error { // Update the record checker run data rcData = recordCheckerData{ lastRun: &runStartTime, - invalidRecords: newInvalid, + invalidRecords: newInvalids, } if err := writeRecordCheckData(ctx, env.ds, rcData); err != nil { return fmt.Errorf("failed to store run data: %w", err) @@ -178,23 +178,24 @@ func run(ctx context.Context, env *appEnv) error { return nil } +// handleResult handles logging and sending pub/sub message to the recoverer. +// Returns true if a pub/sub message was sent to the recoverer, +// to indicate that we need to verify that the recoverer fixes the problem on the next run. func handleResult(ctx context.Context, topic *pubsub.Topic, result checkRecordResult) bool { - wasInvalid := false - if result.Err != nil { - logger.Error("failed to process record", slog.String("id", result.ID), slog.Any("err", result.Err)) + if result.err != nil { + logger.Error("failed to process record", slog.String("id", result.id), slog.Any("err", result.err)) } - if result.NeedsRetry { - wasInvalid = true + if result.needsRetry { msg := pubsub.Message{ - Attributes: map[string]string{"type": "gcs_missing", "id": result.ID}, + Attributes: map[string]string{"type": "gcs_missing", "id": result.id}, } - logger.Info("publishing gcs_missing message", slog.String("id", result.ID)) + logger.Info("publishing gcs_missing message", slog.String("id", result.id)) _, err := topic.Publish(ctx, &msg).Get(ctx) if err != nil { - logger.Error("failed publishing message", slog.String("id", result.ID), slog.Any("err", err)) + logger.Error("failed publishing message", slog.String("id", result.id), slog.Any("err", err)) } } - return wasInvalid + return result.needsRetry } // appEnv holds the clients and other environment-specific resources. @@ -256,15 +257,13 @@ func setup(ctx context.Context) (*appEnv, error) { } type checkRecordResult struct { - ID string - NeedsRetry bool - Err error + id string + needsRetry bool + err error } -func checkRecord(ctx context.Context, cl *datastore.Client, bucket *storage.BucketHandle, id string, vuln *models.Vulnerability, out chan<- checkRecordResult) { - res := checkRecordResult{ID: id} - // Send the result when the function returns. - defer func() { out <- res }() +func checkRecord(ctx context.Context, cl *datastore.Client, bucket *storage.BucketHandle, id string, vuln *models.Vulnerability) checkRecordResult { + res := checkRecordResult{id: id} if vuln == nil { key := datastore.NameKey("Vulnerability", id, nil) @@ -272,13 +271,13 @@ func checkRecord(ctx context.Context, cl *datastore.Client, bucket *storage.Buck if err := cl.Get(ctx, key, &fetchedVuln); err != nil { if errors.Is(err, datastore.ErrNoSuchEntity) { // This is a permanent error, don't retry. - res.Err = fmt.Errorf("vulnerability %s not found in datastore: %w", id, err) + res.err = fmt.Errorf("vulnerability %s not found in datastore: %w", id, err) } else { // This is likely a transient error, retry. - res.NeedsRetry = true - res.Err = fmt.Errorf("failed to get vulnerability %s from datastore: %w", id, err) + res.needsRetry = true + res.err = fmt.Errorf("failed to get vulnerability %s from datastore: %w", id, err) } - return + return res } vuln = &fetchedVuln } @@ -286,17 +285,19 @@ func checkRecord(ctx context.Context, cl *datastore.Client, bucket *storage.Buck obj := bucket.Object(fmt.Sprintf("all/pb/%s.pb", id)) attrs, err := obj.Attrs(ctx) if err != nil { - res.NeedsRetry = true + res.needsRetry = true if !errors.Is(err, storage.ErrObjectNotExist) { // Log the error if it's not the expected "not found" error. - res.Err = fmt.Errorf("failed to get GCS attributes for %s: %w", id, err) + res.err = fmt.Errorf("failed to get GCS attributes for %s: %w", id, err) } - return + return res } if attrs.CustomTime.Before(vuln.Modified) { - res.NeedsRetry = true + res.needsRetry = true } + + return res } func main() { diff --git a/go/models/internal/validate/validate.go b/go/models/internal/validate/validate.go index fe13d18e2b3..0f1b652d73e 100644 --- a/go/models/internal/validate/validate.go +++ b/go/models/internal/validate/validate.go @@ -12,6 +12,9 @@ import ( func main() { ctx := context.Background() + // Note: this does not communicate with GCP. + // The Python code that runs this sets up the datastore emulator + // with the relevant environment variables necessary. client, err := datastore.NewClient(ctx, os.Getenv("GOOGLE_CLOUD_PROJECT")) if err != nil { fmt.Printf("(Go) Failed creating Datastore client: %v\n", err) From 219045203122803b0af860f379fec988b5063857 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Thu, 18 Sep 2025 16:20:43 +1000 Subject: [PATCH 13/13] lint --- go/cmd/recordchecker/recordchecker.go | 4 ++++ go/models/internal/validate/validate.go | 2 ++ go/models/models.go | 1 + 3 files changed, 7 insertions(+) diff --git a/go/cmd/recordchecker/recordchecker.go b/go/cmd/recordchecker/recordchecker.go index 7dfd9b59ad2..dec17f7952c 100644 --- a/go/cmd/recordchecker/recordchecker.go +++ b/go/cmd/recordchecker/recordchecker.go @@ -1,3 +1,4 @@ +// Package main runs the record checker, checking that each Vulnerability in Datastore has an up-to-date corresponding GCS object. package main import ( @@ -195,6 +196,7 @@ func handleResult(ctx context.Context, topic *pubsub.Topic, result checkRecordRe logger.Error("failed publishing message", slog.String("id", result.id), slog.Any("err", err)) } } + return result.needsRetry } @@ -277,6 +279,7 @@ func checkRecord(ctx context.Context, cl *datastore.Client, bucket *storage.Buck res.needsRetry = true res.err = fmt.Errorf("failed to get vulnerability %s from datastore: %w", id, err) } + return res } vuln = &fetchedVuln @@ -290,6 +293,7 @@ func checkRecord(ctx context.Context, cl *datastore.Client, bucket *storage.Buck // Log the error if it's not the expected "not found" error. res.err = fmt.Errorf("failed to get GCS attributes for %s: %w", id, err) } + return res } diff --git a/go/models/internal/validate/validate.go b/go/models/internal/validate/validate.go index 0f1b652d73e..c3d2cda9b1b 100644 --- a/go/models/internal/validate/validate.go +++ b/go/models/internal/validate/validate.go @@ -1,3 +1,5 @@ +// Package main reads datastore records that were created by the validate.py python script. +// This should not be run outside of the validate.py script. package main import ( diff --git a/go/models/models.go b/go/models/models.go index 3bf1c818a25..d8f1bc8df03 100644 --- a/go/models/models.go +++ b/go/models/models.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package models contains definitions of Datastore entities for OSV.dev. package models import "time"