diff --git a/go.mod b/go.mod index af38b7b2a..f0468639a 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module github.com/goccy/bigquery-emulator go 1.17 require ( - cloud.google.com/go/bigquery v1.43.0 + cloud.google.com/go/bigquery v1.44.0 + cloud.google.com/go/storage v1.28.1 github.com/GoogleCloudPlatform/golang-samples/bigquery v0.0.0-20221115172052-07ffb99455e8 github.com/apache/arrow/go/v10 v10.0.0 + github.com/fsouza/fake-gcs-server v1.43.0 github.com/go-playground/validator/v10 v10.11.0 github.com/goccy/go-json v0.10.0 github.com/goccy/go-yaml v1.9.5 @@ -19,17 +21,18 @@ require ( github.com/segmentio/parquet-go v0.0.0-20221020201645-63215c8128ff go.uber.org/zap v1.21.0 golang.org/x/sync v0.1.0 - google.golang.org/api v0.103.0 - google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66 - google.golang.org/grpc v1.50.1 + google.golang.org/api v0.105.0 + google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 + google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.1 ) require ( cloud.google.com/go v0.106.0 // indirect - cloud.google.com/go/compute v1.12.1 // indirect - cloud.google.com/go/compute/metadata v0.2.1 // indirect - cloud.google.com/go/iam v0.7.0 // indirect + cloud.google.com/go/compute v1.13.0 // indirect + cloud.google.com/go/compute/metadata v0.2.2 // indirect + cloud.google.com/go/iam v0.8.0 // indirect + cloud.google.com/go/pubsub v1.28.0 // indirect github.com/DataDog/go-hll v1.0.2 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/thrift v0.16.0 // indirect @@ -37,6 +40,7 @@ require ( github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dop251/goja v0.0.0-20221118162653-d4bf6fde1b86 // indirect github.com/fatih/color v1.10.0 // indirect + github.com/felixge/httpsnoop v1.0.2 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect @@ -44,8 +48,10 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/gorilla/handlers v1.5.1 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect @@ -59,7 +65,9 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/xattr v0.4.9 // indirect github.com/segmentio/encoding v0.3.5 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect @@ -70,7 +78,7 @@ require ( golang.org/x/net v0.1.0 // indirect golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect golang.org/x/sys v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/text v0.5.0 // indirect golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.11.0 // indirect diff --git a/go.sum b/go.sum index 80e0b7183..868654454 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,7 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= @@ -41,6 +42,7 @@ cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= @@ -83,8 +85,9 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.26.0/go.mod h1:zB9gvHgECb8KMHiAIVI8uwvlWVF4QDzVnUQP/CdPaQc= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= -cloud.google.com/go/bigquery v1.43.0 h1:u0fvz5ysJBe1jwUPI4LuPwAX+o+6fCUwf3ECeg6eDUQ= cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0 h1:Wi4dITi+cf9VYp4VH2T9O41w0kCW0uQTELq2Z6tukN0= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= @@ -113,11 +116,13 @@ cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLq cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0 h1:AYrLkB8NPdDRslNp4Jxmzrhdr03fUAIDbiGFjLWowoU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= -cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.2 h1:aWKAjYaBaOSrpKl57+jnS/3fJRQnxL7TvR/u1VVbt6k= +cloud.google.com/go/compute/metadata v0.2.2/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= @@ -149,6 +154,7 @@ cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.2.0/go.mod h1:FKd9dFEjRui5757lkOJ7z/eKtL74o5hsbY0o6Z0ozz8= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= @@ -171,12 +177,14 @@ cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9n cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/errorreporting v0.1.0/go.mod h1:cZSiBMvrnl0X13pD9DwKf9sQ8Eqy3EzHqkyKBZxiIrM= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= @@ -196,18 +204,22 @@ cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZ cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= -cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0 h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= @@ -216,6 +228,7 @@ cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEy cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= @@ -269,6 +282,11 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0 h1:XzabfdPx/+eNrsVVGLFgeUnQQKPGkMb8klRCeYK52is= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= @@ -323,6 +341,7 @@ cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5 cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= @@ -337,8 +356,9 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.24.0/go.mod h1:3xrJEFMXBsQLgxwThyjuD3aYlroL0TMRec1ypGUQ0KE= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.28.0 h1:DLrIZ6xkeZX6K70fU/boWx5INJumt6f+nwwWSHXzzGY= cloud.google.com/go/storage v1.28.0/go.mod h1:qlgZML35PXA3zoEnIkiPLY4/TOkUleufRlu6qmcf7sI= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= @@ -411,6 +431,7 @@ github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQm github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -454,8 +475,13 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= +github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsouza/fake-gcs-server v1.43.0 h1:8ScK+6/mpKp9QoICALyemgVySEoolMbO2TJMdxzgkr8= +github.com/fsouza/fake-gcs-server v1.43.0/go.mod h1:zYK0CjlFRY9gpWhukEGWzvGHMQ/g8HoGf/RzpKSqpbs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= @@ -570,7 +596,10 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -590,6 +619,8 @@ github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMd github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -653,6 +684,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -669,6 +702,8 @@ github.com/segmentio/encoding v0.3.5 h1:UZEiaZ55nlXGDL92scoVuw00RmiRCazIEmvPSbSv github.com/segmentio/encoding v0.3.5/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= github.com/segmentio/parquet-go v0.0.0-20221020201645-63215c8128ff h1:qLJHWKeFq31XCEKVS4uUlS7D3qGDiTjy/bPNn27m0pQ= github.com/segmentio/parquet-go v0.0.0-20221020201645-63215c8128ff/go.mod h1:SclLlCfB7c7CH0YerV+OtYmZExyK5rhVOd6UT90erVw= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -932,6 +967,7 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -939,6 +975,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -958,11 +995,14 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1092,8 +1132,9 @@ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= +google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1211,8 +1252,13 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= -google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66 h1:wx7sJ5GRBQLRcslTNcrTklsHhHevQvxgztW18txbbZM= google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc= +google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1247,8 +1293,9 @@ google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/server/handler.go b/server/handler.go index dba47828d..ce7b9f05b 100644 --- a/server/handler.go +++ b/server/handler.go @@ -12,15 +12,18 @@ import ( "mime" "mime/multipart" "net/http" + "os" "reflect" "strconv" "strings" "sync" "time" + "cloud.google.com/go/storage" "github.com/goccy/go-json" "go.uber.org/zap" bigqueryv2 "google.golang.org/api/bigquery/v2" + "google.golang.org/api/option" "github.com/goccy/bigquery-emulator/internal/connection" "github.com/goccy/bigquery-emulator/internal/logger" @@ -1046,12 +1049,310 @@ func (h *jobsInsertHandler) tableDefFromQueryResponse(tableID string, response * ) } +const ( + gcsEmulatorHostEnvName = "STORAGE_EMULATOR_HOST" + gcsURIPrefix = "gs://" +) + +func (h *jobsInsertHandler) importFromGCS(ctx context.Context, r *jobsInsertRequest) (*bigqueryv2.Job, error) { + var opts []option.ClientOption + if host := os.Getenv(gcsEmulatorHostEnvName); host != "" { + opts = append( + opts, + option.WithEndpoint(fmt.Sprintf("%s/storage/v1/", host)), + option.WithoutAuthentication(), + ) + } + client, err := storage.NewClient(ctx, opts...) + if err != nil { + return nil, err + } + startTime := time.Now() + for _, uri := range r.job.Configuration.Load.SourceUris { + if !strings.HasPrefix(uri, gcsURIPrefix) { + return nil, fmt.Errorf("load source uri must start with gs://") + } + uri = strings.TrimLeft(uri, gcsURIPrefix) + paths := strings.Split(uri, "/") + if len(paths) < 2 { + return nil, fmt.Errorf("unexpected gcs uri format %s", uri) + } + bucketName := paths[0] + objectPath := strings.Join(paths[1:], "/") + reader, err := client.Bucket(bucketName).Object(objectPath).NewReader(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get gcs object reader for %s: %w", uri, err) + } + if err := h.importFromGCSObject(ctx, r, reader); err != nil { + return nil, err + } + } + endTime := time.Now() + job := r.job + job.Kind = "bigquery#job" + job.Configuration.JobType = "LOAD" + job.SelfLink = fmt.Sprintf( + "http://%s/bigquery/v2/projects/%s/jobs/%s", + r.server.httpServer.Addr, + r.project.ID, + job.JobReference.JobId, + ) + job.Status = &bigqueryv2.JobStatus{State: "DONE"} + job.Statistics = &bigqueryv2.JobStatistics{ + CreationTime: startTime.Unix(), + StartTime: startTime.Unix(), + EndTime: endTime.Unix(), + } + conn, err := r.server.connMgr.Connection(ctx, r.project.ID, "") + if err != nil { + return nil, fmt.Errorf("failed to get connection: %w", err) + } + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("failed to start transaction: %w", err) + } + defer tx.RollbackIfNotCommitted() + if err := r.project.AddJob( + ctx, + tx.Tx(), + metadata.NewJob( + r.server.metaRepo, + r.project.ID, + job.JobReference.JobId, + job, + nil, + nil, + ), + ); err != nil { + return nil, fmt.Errorf("failed to add job: %w", err) + } + if !job.Configuration.DryRun { + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("failed to commit job: %w", err) + } + } + return job, nil +} + +func (h *jobsInsertHandler) importFromGCSObject(ctx context.Context, r *jobsInsertRequest, reader *storage.Reader) error { + defer func() { + _ = reader.Close() + }() + job := metadata.NewJob( + r.server.metaRepo, + r.project.ID, + r.job.JobReference.JobId, + r.job, + nil, + nil, + ) + if err := new(uploadContentHandler).Handle(ctx, &uploadContentRequest{ + server: r.server, + project: r.project, + job: job, + reader: reader, + }); err != nil { + return err + } + return nil +} + +func (h *jobsInsertHandler) exportToGCS(ctx context.Context, r *jobsInsertRequest) (*bigqueryv2.Job, error) { + var opts []option.ClientOption + if host := os.Getenv(gcsEmulatorHostEnvName); host != "" { + opts = append( + opts, + option.WithEndpoint(fmt.Sprintf("%s/storage/v1/", host)), + option.WithoutAuthentication(), + ) + } + client, err := storage.NewClient(ctx, opts...) + if err != nil { + return nil, err + } + startTime := time.Now() + conn, err := r.server.connMgr.Connection(ctx, r.project.ID, "") + if err != nil { + return nil, fmt.Errorf("failed to get connection: %w", err) + } + tx, err := conn.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("failed to start transaction: %w", err) + } + defer tx.RollbackIfNotCommitted() + extract := r.job.Configuration.Extract + sourceTable := extract.SourceTable + response, err := r.server.contentRepo.Query( + ctx, + tx, + sourceTable.ProjectId, + sourceTable.DatasetId, + fmt.Sprintf("SELECT * FROM `%s`", sourceTable.TableId), + nil, + ) + if err != nil { + return nil, err + } + for _, uri := range extract.DestinationUris { + if !strings.HasPrefix(uri, gcsURIPrefix) { + return nil, fmt.Errorf("destination uri must start with gs://") + } + uri = strings.TrimLeft(uri, gcsURIPrefix) + paths := strings.Split(uri, "/") + if len(paths) < 2 { + return nil, fmt.Errorf("unexpected gcs uri format %s", uri) + } + bucketName := paths[0] + objectPath := strings.Join(paths[1:], "/") + bucket := client.Bucket(bucketName) + _ = bucket.Create(ctx, r.project.ID, nil) // ignore "already exists" error. + writer := bucket.Object(objectPath).NewWriter(ctx) + if err := h.exportToGCSWithObject(ctx, response, extract, writer); err != nil { + return nil, err + } + } + endTime := time.Now() + job := r.job + job.Kind = "bigquery#job" + job.Configuration.JobType = "EXTRACT" + job.SelfLink = fmt.Sprintf( + "http://%s/bigquery/v2/projects/%s/jobs/%s", + r.server.httpServer.Addr, + r.project.ID, + job.JobReference.JobId, + ) + job.Status = &bigqueryv2.JobStatus{State: "DONE"} + job.Statistics = &bigqueryv2.JobStatistics{ + CreationTime: startTime.Unix(), + StartTime: startTime.Unix(), + EndTime: endTime.Unix(), + } + if err := r.project.AddJob( + ctx, + tx.Tx(), + metadata.NewJob( + r.server.metaRepo, + r.project.ID, + job.JobReference.JobId, + job, + nil, + nil, + ), + ); err != nil { + return nil, fmt.Errorf("failed to add job: %w", err) + } + if !job.Configuration.DryRun { + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("failed to commit job: %w", err) + } + } + return job, nil +} + +func (h *jobsInsertHandler) exportToGCSWithObject(ctx context.Context, response *internaltypes.QueryResponse, extract *bigqueryv2.JobConfigurationExtract, writer *storage.Writer) (e error) { + defer func() { + if err := writer.Close(); err != nil { + e = err + } + }() + switch extract.DestinationFormat { + case "CSV": + if len(response.Rows) == 0 { + if _, err := writer.Write(nil); err != nil { + return fmt.Errorf("failed to empty table data to gcs object: %w", err) + } + return nil + } + csvWriter := csv.NewWriter(writer) + var columns []string + for _, cell := range response.Rows[0].F { + columns = append(columns, cell.Name) + } + if extract.PrintHeader == nil { + if err := csvWriter.Write(columns); err != nil { + return fmt.Errorf("failed to encode csv columns: %w", err) + } + } + for _, row := range response.Rows { + data, err := row.Data() + if err != nil { + return fmt.Errorf("failed to get data from table row: %w", err) + } + var records []string + for _, col := range columns { + value := data[col] + if value == nil { + records = append(records, "") + continue + } + if v, ok := value.(string); ok { + records = append(records, v) + continue + } + jsonValue, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("failed to encode row value: %w", err) + } + records = append(records, string(jsonValue)) + } + if err := csvWriter.Write(records); err != nil { + return fmt.Errorf("failed to encode csv data: %w", err) + } + } + csvWriter.Flush() + if err := csvWriter.Error(); err != nil { + return fmt.Errorf("failed to encode csv data: %w", err) + } + case "NEWLINE_DELIMITED_JSON": + writer.ContentType = "application/json" + enc := json.NewEncoder(writer) + for _, row := range response.Rows { + data, err := row.Data() + if err != nil { + return fmt.Errorf("failed to get data from table row: %w", err) + } + if err := enc.Encode(data); err != nil { + return fmt.Errorf("failed to encode table data: %w", err) + } + } + case "PARQUET": + var opts []parquet.WriterOption + switch extract.Compression { + case "GZIP": + opts = append(opts, parquet.Compression(&parquet.Gzip)) + case "SNAPPY": + opts = append(opts, parquet.Compression(&parquet.Snappy)) + case "DEFLATE": + opts = append(opts, parquet.Compression(&parquet.Gzip)) + } + _ = opts + fallthrough + default: + return fmt.Errorf("failed to export to gcs: unsupported destination format %s", extract.DestinationFormat) + } + return nil +} + func (h *jobsInsertHandler) Handle(ctx context.Context, r *jobsInsertRequest) (*bigqueryv2.Job, error) { job := r.job if job.Configuration == nil { return nil, fmt.Errorf("unspecified job configuration") } if job.Configuration.Query == nil { + if job.Configuration.Load != nil && len(job.Configuration.Load.SourceUris) != 0 { + // load from google cloud storage + job, err := h.importFromGCS(ctx, r) + if err != nil { + return nil, fmt.Errorf("failed to import from gcs: %w", err) + } + return job, nil + } else if job.Configuration.Extract != nil && len(job.Configuration.Extract.DestinationUris) != 0 { + job, err := h.exportToGCS(ctx, r) + if err != nil { + return nil, fmt.Errorf("failed to export to gcs: %w", err) + } + return job, nil + } return nil, fmt.Errorf("unspecified job configuration query") } conn, err := r.server.connMgr.Connection(ctx, r.project.ID, "") diff --git a/server/server_test.go b/server/server_test.go index 78b8bb837..b2cf139e3 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -4,15 +4,20 @@ import ( "bytes" "compress/gzip" "context" + "encoding/csv" "fmt" "io" "math/big" "net/http" + "net/url" "path/filepath" + "strconv" "testing" "time" "cloud.google.com/go/bigquery" + "cloud.google.com/go/storage" + "github.com/fsouza/fake-gcs-server/fakestorage" "github.com/goccy/bigquery-emulator/server" "github.com/goccy/bigquery-emulator/types" "github.com/goccy/go-json" @@ -1346,6 +1351,329 @@ func TestLoadJSON(t *testing.T) { } } +func TestImportFromGCS(t *testing.T) { + const ( + projectID = "test" + datasetID = "dataset1" + tableID = "table_a" + publicHost = "127.0.0.1" + bucketName = "test-bucket" + sourceName = "path/to/data.json" + ) + + ctx := context.Background() + bqServer, err := server.New(server.TempStorage) + if err != nil { + t.Fatal(err) + } + project := types.NewProject( + projectID, + types.NewDataset( + datasetID, + types.NewTable( + tableID, + []*types.Column{ + types.NewColumn("id", types.INT64), + types.NewColumn("value", types.INT64), + }, + nil, + ), + ), + ) + if err := bqServer.Load(server.StructSource(project)); err != nil { + t.Fatal(err) + } + + testServer := bqServer.TestServer() + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for i := 0; i < 3; i++ { + if err := enc.Encode(map[string]interface{}{ + "id": i + 1, + "value": i + 10, + }); err != nil { + t.Fatal(err) + } + } + storageServer, err := fakestorage.NewServerWithOptions(fakestorage.Options{ + InitialObjects: []fakestorage.Object{ + { + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: bucketName, + Name: sourceName, + Size: int64(len(buf.Bytes())), + }, + Content: buf.Bytes(), + }, + }, + PublicHost: publicHost, + Scheme: "http", + }) + if err != nil { + t.Fatal(err) + } + + storageServerURL := storageServer.URL() + u, err := url.Parse(storageServerURL) + if err != nil { + t.Fatal(err) + } + storageEmulatorHost := fmt.Sprintf("http://%s:%s", publicHost, u.Port()) + t.Setenv("STORAGE_EMULATOR_HOST", storageEmulatorHost) + + defer func() { + testServer.Close() + bqServer.Stop(ctx) + storageServer.Stop() + }() + + client, err := bigquery.NewClient( + ctx, + projectID, + option.WithEndpoint(testServer.URL), + option.WithoutAuthentication(), + ) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + gcsSourceURL := fmt.Sprintf("gs://%s/%s", bucketName, sourceName) + gcsRef := bigquery.NewGCSReference(gcsSourceURL) + gcsRef.SourceFormat = bigquery.JSON + gcsRef.AutoDetect = true + loader := client.Dataset(datasetID).Table(tableID).LoaderFrom(gcsRef) + loader.WriteDisposition = bigquery.WriteTruncate + job, err := loader.Run(ctx) + if err != nil { + t.Fatal(err) + } + status, err := job.Wait(ctx) + if err != nil { + t.Fatal(err) + } + if status.Err() != nil { + t.Fatal(status.Err()) + } + + query := client.Query(fmt.Sprintf("SELECT * FROM %s.%s", datasetID, tableID)) + it, err := query.Read(ctx) + if err != nil { + t.Fatal(err) + } + + type row struct { + ID int64 + Value int64 + } + var rows []*row + for { + var r row + if err := it.Next(&r); err != nil { + if err == iterator.Done { + break + } + t.Fatal(err) + } + rows = append(rows, &r) + } + if diff := cmp.Diff([]*row{ + {ID: 1, Value: 10}, + {ID: 2, Value: 11}, + {ID: 3, Value: 12}, + }, rows); diff != "" { + t.Errorf("(-want +got):\n%s", diff) + } +} + +func TestExportToGCS(t *testing.T) { + const ( + projectID = "test" + datasetID = "dataset1" + tableID = "table_a" + publicHost = "127.0.0.1" + bucketName = "test-export-bucket" + ) + + ctx := context.Background() + bqServer, err := server.New(server.TempStorage) + if err != nil { + t.Fatal(err) + } + project := types.NewProject( + projectID, + types.NewDataset( + datasetID, + types.NewTable( + tableID, + []*types.Column{ + types.NewColumn("id", types.INT64), + types.NewColumn("value", types.INT64), + }, + types.Data{ + {"id": int64(1), "value": int64(21)}, + {"id": int64(2), "value": int64(22)}, + {"id": int64(3), "value": int64(23)}, + }, + ), + ), + ) + if err := bqServer.Load(server.StructSource(project)); err != nil { + t.Fatal(err) + } + + testServer := bqServer.TestServer() + storageServer, err := fakestorage.NewServerWithOptions(fakestorage.Options{ + PublicHost: publicHost, + Scheme: "http", + }) + if err != nil { + t.Fatal(err) + } + + storageServerURL := storageServer.URL() + u, err := url.Parse(storageServerURL) + if err != nil { + t.Fatal(err) + } + storageEmulatorHost := fmt.Sprintf("http://%s:%s", publicHost, u.Port()) + t.Setenv("STORAGE_EMULATOR_HOST", storageEmulatorHost) + + defer func() { + testServer.Close() + bqServer.Stop(ctx) + storageServer.Stop() + }() + + client, err := bigquery.NewClient( + ctx, + projectID, + option.WithEndpoint(testServer.URL), + option.WithoutAuthentication(), + ) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + type T struct { + ID int64 + Value int64 + } + + storageClient, err := storage.NewClient( + ctx, + option.WithEndpoint(fmt.Sprintf("%s/storage/v1/", storageEmulatorHost)), + option.WithoutAuthentication(), + ) + if err != nil { + t.Fatal(err) + } + + t.Run("csv", func(t *testing.T) { + sourceName := "path/to/data.csv" + gcsSourceURL := fmt.Sprintf("gs://%s/%s", bucketName, sourceName) + gcsRef := bigquery.NewGCSReference(gcsSourceURL) + gcsRef.DestinationFormat = bigquery.CSV + gcsRef.FieldDelimiter = "," + extractor := client.DatasetInProject(projectID, datasetID).Table(tableID).ExtractorTo(gcsRef) + extractor.DisableHeader = true + job, err := extractor.Run(ctx) + if err != nil { + t.Fatal(err) + } + status, err := job.Wait(ctx) + if err != nil { + t.Fatal(err) + } + if err := status.Err(); err != nil { + t.Fatal(err) + } + reader, err := storageClient.Bucket(bucketName).Object(sourceName).NewReader(ctx) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + records, err := csv.NewReader(reader).ReadAll() + if err != nil { + t.Fatal(err) + } + var rows []*T + for _, record := range records { + id, err := strconv.ParseInt(record[0], 10, 64) + if err != nil { + t.Fatal(err) + } + value, err := strconv.ParseInt(record[1], 10, 64) + if err != nil { + t.Fatal(err) + } + rows = append(rows, &T{ID: id, Value: value}) + } + if diff := cmp.Diff([]*T{ + {ID: 1, Value: 21}, + {ID: 2, Value: 22}, + {ID: 3, Value: 23}, + }, rows); diff != "" { + t.Errorf("(-want +got):\n%s", diff) + } + }) + + t.Run("json", func(t *testing.T) { + sourceName := "path/to/data.json" + gcsSourceURL := fmt.Sprintf("gs://%s/%s", bucketName, sourceName) + gcsRef := bigquery.NewGCSReference(gcsSourceURL) + gcsRef.DestinationFormat = bigquery.JSON + extractor := client.DatasetInProject(projectID, datasetID).Table(tableID).ExtractorTo(gcsRef) + job, err := extractor.Run(ctx) + if err != nil { + t.Fatal(err) + } + status, err := job.Wait(ctx) + if err != nil { + t.Fatal(err) + } + if err := status.Err(); err != nil { + t.Fatal(err) + } + reader, err := storageClient.Bucket(bucketName).Object(sourceName).NewReader(ctx) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + dec := json.NewDecoder(reader) + var rows []*T + for dec.More() { + var v struct { + ID json.Number + Value json.Number + } + if err := dec.Decode(&v); err != nil { + t.Fatal(err) + } + id, err := v.ID.Int64() + if err != nil { + t.Fatal(err) + } + value, err := v.Value.Int64() + if err != nil { + t.Fatal(err) + } + rows = append(rows, &T{ + ID: id, + Value: value, + }) + } + if diff := cmp.Diff([]*T{ + {ID: 1, Value: 21}, + {ID: 2, Value: 22}, + {ID: 3, Value: 23}, + }, rows); diff != "" { + t.Errorf("(-want +got):\n%s", diff) + } + }) +} + func TestQueryWithNamedParams(t *testing.T) { const ( projectID = "test"