From 89bb36ee7c72c852cfa91d53210f52c490800a46 Mon Sep 17 00:00:00 2001 From: Charles Moog Date: Mon, 30 Dec 2019 20:08:24 +1100 Subject: [PATCH 01/24] MySQL support (#1) * initial commit of sql schema validation * adds barebones support for most basic use-case * improves parsing and generation of params * adds decent tests for mysql * improves param implementation - adds tests - adds support for named params - adds support for Limit expressions * adds support for decimal types * resolves some testing issues * generalizes imports * adds null type tests * improves go struct generation * adds command line call to mysql generate * improves support for insert queries and adds tests * improves support for enum types * refactors config to function param * patchs pkg settings arguments * simplifies and improves param parsing * param implementation and test patches * patch placeholder default table * properly handles "select * " expressions * initial commit of sql schema validation * adds barebones support for most basic use-case * improves parsing and generation of params * adds decent tests for mysql * improves param implementation - adds tests - adds support for named params - adds support for Limit expressions * adds support for decimal types * resolves some testing issues * generalizes imports * adds null type tests * improves go struct generation * adds command line call to mysql generate * improves support for insert queries and adds tests * improves support for enum types * refactors config to function param * patchs pkg settings arguments * simplifies and improves param parsing * param implementation and test patches * patch placeholder default table * properly handles "select * " expressions --- go.mod | 6 + go.sum | 289 +++++++++++++++++++++++++ internal/cmd/cmd.go | 41 +++- internal/dinosql/checks_test.go | 1 - internal/dinosql/config.go | 86 +++++++- internal/dinosql/config_test.go | 2 +- internal/dinosql/gen.go | 182 +++++++++------- internal/dinosql/gen_test.go | 37 +++- internal/dinosql/parser.go | 35 +-- internal/dinosql/parser_test.go | 13 +- internal/mysql/gen.go | 263 +++++++++++++++++++++++ internal/mysql/param.go | 128 +++++++++++ internal/mysql/param_test.go | 164 ++++++++++++++ internal/mysql/parse.go | 322 ++++++++++++++++++++++++++++ internal/mysql/parse_test.go | 366 ++++++++++++++++++++++++++++++++ internal/mysql/schema.go | 48 +++++ internal/mysql/test.sql | 16 ++ 17 files changed, 1876 insertions(+), 123 deletions(-) create mode 100644 internal/mysql/gen.go create mode 100644 internal/mysql/param.go create mode 100644 internal/mysql/param_test.go create mode 100644 internal/mysql/parse.go create mode 100644 internal/mysql/parse_test.go create mode 100644 internal/mysql/schema.go create mode 100644 internal/mysql/test.sql diff --git a/go.mod b/go.mod index 1f46d34a1d..0ab1d9fcd6 100644 --- a/go.mod +++ b/go.mod @@ -8,4 +8,10 @@ require ( github.com/jinzhu/inflection v1.0.0 github.com/lfittl/pg_query_go v1.0.0 github.com/spf13/cobra v0.0.5 + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect + golang.org/x/sys v0.0.0-20191220220014-0732a990476f // indirect + google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf // indirect + google.golang.org/grpc v1.26.0 // indirect + vitess.io/vitess v0.0.0-20191113025808-0629f0da20ab ) diff --git a/go.sum b/go.sum index 30821273b3..ea10b1ce2d 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,176 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5/go.mod h1:4/6eNcqZ09BZ9wLK3tZOjBA1nDj+B0728nlX5YRlSmQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/aws/aws-sdk-go v0.0.0-20180223184012-ebef4262e06a/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/etcd v0.0.0-20170626015032-703663d1f6ed/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/safefile v0.0.0-20151022103144-855e8d98f185/go.mod h1:cFRxtTwTOJkz2x3rQUNCYKWC93yP1VKjR8NUhqFxZNU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20161207003320-04f313413ffd/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-ini/ini v1.12.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v0.0.0-20161128002007-199c40a060d1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/consul v1.4.0/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.0.0-20161207011743-d3a67ab21bc8/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v0.0.0-20180801095237-b50017755d44/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v1.2.0/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.2.0/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/krishicks/yaml-patch v0.0.10/go.mod h1:Sm5TchwZS6sm7RJoyg87tzxm2ZcKzdRE4Q7TjNhPrME= github.com/lfittl/pg_query_go v1.0.0 h1:rcHZK5DBEUoxtO6dACP+UVCHKtA1ZsELBW0rSjOXMAE= github.com/lfittl/pg_query_go v1.0.0/go.mod h1:jcikG62RKf+NIWmbLzjjk73m4x6um2pKf3h+TJyINms= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-runewidth v0.0.1/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJuf5HmJAJrKV64MmozrSsk+or0PB5dzdfspg= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v0.0.0-20160824210600-b984ec7fa9ff/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/satori/go.uuid v0.0.0-20160713180306-0aa62d5ddceb/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= @@ -31,12 +178,154 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tchap/go-patricia v0.0.0-20160729071656-dd168db6051b/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw= +github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= +github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b/go.mod h1:JNALoWa+nCXR8SmgLluHcBNVJgyejzpKPZk9pX2yXXE= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190926025831-c00fd9afed17/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190926180325-855e68c8590b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f h1:72l8qCJ1nGxMGH26QVBVIxKd/D34cfGt0OvrPtpemyY= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190830154057-c17b040389b9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190926190326-7ee9db18f195/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf h1:1x8rC5/IgdLMPbPTvlQTN28+rcy8XL9Q19UWUMDyqYs= +google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ldap.v2 v2.5.0/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +vitess.io/vitess v0.0.0-20191113025808-0629f0da20ab h1:Lu6PzNvcd0AVzWPohZE1bCqxvICJGwhpxyiunaes3p4= +vitess.io/vitess v0.0.0-20191113025808-0629f0da20ab/go.mod h1:tLyyCa/hBqubuTQeB34djK7LTtM52DCUcfWYAYf1oZI= +vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= +vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 3c7ffcece3..a5527c59b3 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -11,6 +11,7 @@ import ( "path/filepath" "github.com/kyleconroy/sqlc/internal/dinosql" + "github.com/kyleconroy/sqlc/internal/mysql" "github.com/davecgh/go-spew/spew" pg "github.com/lfittl/pg_query_go" @@ -21,6 +22,7 @@ import ( func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int { rootCmd := &cobra.Command{Use: "sqlc", SilenceUsage: true} rootCmd.AddCommand(checkCmd) + rootCmd.AddCommand(unstable__mysql) rootCmd.AddCommand(genCmd) rootCmd.AddCommand(initCmd) rootCmd.AddCommand(parseCmd) @@ -108,7 +110,7 @@ var genCmd = &cobra.Command{ os.Exit(1) } - settings, err := dinosql.ParseConfig(bytes.NewReader(blob)) + settings, err := dinosql.ParseConfigFile(bytes.NewReader(blob)) if err != nil { switch err { case dinosql.ErrMissingVersion: @@ -153,7 +155,7 @@ var genCmd = &cobra.Command{ continue } - q, err := dinosql.ParseQueries(c, settings, pkg) + q, err := dinosql.ParseQueries(c, pkg) if err != nil { fmt.Fprintf(os.Stderr, "# package %s\n", name) if parserErr, ok := err.(*dinosql.ParserErr); ok { @@ -167,7 +169,7 @@ var genCmd = &cobra.Command{ continue } - files, err := dinosql.Generate(q, settings, pkg) + files, err := dinosql.Generate(q, settings) if err != nil { fmt.Fprintf(os.Stderr, "# package %s\n", name) fmt.Fprintf(os.Stderr, "error generating code: %s\n", err) @@ -214,10 +216,41 @@ var checkCmd = &cobra.Command{ if err != nil { return err } - if _, err := dinosql.ParseQueries(c, settings, pkg); err != nil { + if _, err := dinosql.ParseQueries(c, pkg); err != nil { return err } } return nil }, } + +var unstable__mysql = &cobra.Command{ + Use: "unstable__mysql generate", + Short: "Generate MySQL Queries into typesafe Go code", + RunE: func(cmd *cobra.Command, args []string) error { + blob, err := ioutil.ReadFile("sqlc.json") + if err != nil { + return err + } + + var settings dinosql.GenerateSettings + if err := json.Unmarshal(blob, &settings); err != nil { + return err + } + + for _, pkg := range settings.Packages { + res, err := mysql.GeneratePkg(pkg.Name, pkg.Queries, settings) + if err != nil { + return err + } + for filename, source := range res { + os.MkdirAll(filepath.Dir(filename), 0755) + if err := ioutil.WriteFile(filename, []byte(source), 0644); err != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", filename, err) + os.Exit(1) + } + } + } + return nil + }, +} diff --git a/internal/dinosql/checks_test.go b/internal/dinosql/checks_test.go index 783bfa7b07..96108c15cd 100644 --- a/internal/dinosql/checks_test.go +++ b/internal/dinosql/checks_test.go @@ -12,7 +12,6 @@ import ( func TestFuncs(t *testing.T) { _, err := ParseQueries( pg.NewCatalog(), - GenerateSettings{}, PackageSettings{ Queries: filepath.Join("testdata", "funcs"), }, diff --git a/internal/dinosql/config.go b/internal/dinosql/config.go index 473d776d53..aa272a78c4 100644 --- a/internal/dinosql/config.go +++ b/internal/dinosql/config.go @@ -1,20 +1,78 @@ package dinosql import ( + "bytes" "encoding/json" "errors" "fmt" "io" + "io/ioutil" + "os" + "path/filepath" "strings" "github.com/kyleconroy/sqlc/internal/pg" ) +const errMessageNoVersion = `The configuration file must have a version number. +Set the version to 1 at the top of sqlc.json: + +{ + "version": "1" + ... +} +` + +const errMessageUnknownVersion = `The configuration file has an invalid version number. +The only supported version is "1". +` + +const errMessageNoPackages = `No packages are configured` + +// InitConfig initializes the global config objcet +func InitConfig() (*GenerateSettings, error) { + fmt.Println("Config init func ran") + blob, err := ioutil.ReadFile("sqlc.json") + if err != nil { + fmt.Fprintln(os.Stderr, "error parsing sqlc.json: file does not exist") + os.Exit(1) + } + + settings, err := ParseConfigFile(bytes.NewReader(blob)) + if err != nil { + switch err { + case ErrMissingVersion: + fmt.Fprintf(os.Stderr, errMessageNoVersion) + case ErrUnknownVersion: + fmt.Fprintf(os.Stderr, errMessageUnknownVersion) + case ErrNoPackages: + fmt.Fprintf(os.Stderr, errMessageNoPackages) + } + fmt.Fprintf(os.Stderr, "error parsing sqlc.json: %s\n", err) + os.Exit(1) + } + + for i, pkg := range settings.Packages { + name := pkg.Name + + if pkg.Path == "" { + fmt.Fprintf(os.Stderr, "package[%d]: path must be set\n", i) + continue + } + + if name == "" { + name = filepath.Base(pkg.Path) + } + } + return &settings, nil +} + type GenerateSettings struct { - Version string `json:"version"` - Packages []PackageSettings `json:"packages"` - Overrides []Override `json:"overrides,omitempty"` - Rename map[string]string `json:"rename,omitempty"` + Version string `json:"version"` + Packages []PackageSettings `json:"packages"` + Overrides []Override `json:"overrides,omitempty"` + Rename map[string]string `json:"rename,omitempty"` + PackageMap map[string]PackageSettings } type PackageSettings struct { @@ -97,7 +155,7 @@ var ErrMissingVersion = errors.New("no version number") var ErrUnknownVersion = errors.New("invalid version number") var ErrNoPackages = errors.New("no packages") -func ParseConfig(rd io.Reader) (GenerateSettings, error) { +func ParseConfigFile(rd io.Reader) (GenerateSettings, error) { dec := json.NewDecoder(rd) dec.DisallowUnknownFields() var config GenerateSettings @@ -125,5 +183,21 @@ func ParseConfig(rd io.Reader) (GenerateSettings, error) { } } } - return config, nil + err := config.PopulatePkgMap() + + return config, err +} + +func (s *GenerateSettings) PopulatePkgMap() error { + packageMap := make(map[string]PackageSettings) + + for _, c := range s.Packages { + if c.Name == "" { + return errors.New("Package name must be specified in sqlc.json") + } + packageMap[c.Name] = c + } + s.PackageMap = packageMap + + return nil } diff --git a/internal/dinosql/config_test.go b/internal/dinosql/config_test.go index ef635dae3a..3b3c03fb46 100644 --- a/internal/dinosql/config_test.go +++ b/internal/dinosql/config_test.go @@ -51,7 +51,7 @@ func TestBadConfigs(t *testing.T) { } { tt := test t.Run(tt.name, func(t *testing.T) { - _, err := ParseConfig(strings.NewReader(tt.json)) + _, err := ParseConfigFile(strings.NewReader(tt.json)) if err == nil { t.Fatalf("expected err; got nil") } diff --git a/internal/dinosql/gen.go b/internal/dinosql/gen.go index ea71bc2348..b17a643a95 100644 --- a/internal/dinosql/gen.go +++ b/internal/dinosql/gen.go @@ -51,8 +51,13 @@ func (gf GoField) Tag() string { return strings.Join(tags, ",") } +// TODO: consider making this deep equality from stdlib? +type Comparable interface { + EqualTo(b interface{}) bool +} + type GoStruct struct { - Table *core.FQN + Table Comparable Name string Fields []GoField Comment string @@ -63,7 +68,24 @@ type GoQueryValue struct { Emit bool Name string Struct *GoStruct - typ string + Typ string +} + +type FQNAlias core.FQN + +// Check whether tables are equal +func (a *FQNAlias) EqualTo(other interface{}) bool { + b, ok := other.(*FQNAlias) + if !ok { + return false + } + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + return a.Catalog == b.Catalog && a.Schema == b.Schema && a.Rel == b.Rel } func (v GoQueryValue) EmitStruct() bool { @@ -75,7 +97,7 @@ func (v GoQueryValue) IsStruct() bool { } func (v GoQueryValue) isEmpty() bool { - return v.typ == "" && v.Name == "" && v.Struct == nil + return v.Typ == "" && v.Name == "" && v.Struct == nil } func (v GoQueryValue) Pair() string { @@ -86,8 +108,8 @@ func (v GoQueryValue) Pair() string { } func (v GoQueryValue) Type() string { - if v.typ != "" { - return v.typ + if v.Typ != "" { + return v.Typ } if v.Struct != nil { return v.Struct.Name @@ -101,7 +123,7 @@ func (v GoQueryValue) Params() string { } var out []string if v.Struct == nil { - if strings.HasPrefix(v.typ, "[]") && v.typ != "[]byte" { + if strings.HasPrefix(v.Typ, "[]") && v.Typ != "[]byte" { out = append(out, "pq.Array("+v.Name+")") } else { out = append(out, v.Name) @@ -125,7 +147,7 @@ func (v GoQueryValue) Params() string { func (v GoQueryValue) Scan() string { var out []string if v.Struct == nil { - if strings.HasPrefix(v.typ, "[]") && v.typ != "[]byte" { + if strings.HasPrefix(v.Typ, "[]") && v.Typ != "[]byte" { out = append(out, "pq.Array(&"+v.Name+")") } else { out = append(out, "&"+v.Name) @@ -159,11 +181,18 @@ type GoQuery struct { Arg GoQueryValue } -func (r Result) UsesType(typ string) bool { - for _, strct := range r.Structs() { +type Generateable interface { + Structs(settings GenerateSettings) []GoStruct + PkgName() string + GoQueries(settings GenerateSettings) []GoQuery + Enums(settings GenerateSettings) []GoEnum +} + +func UsesType(r Generateable, Typ string, settings GenerateSettings) bool { + for _, strct := range r.Structs(settings) { for _, f := range strct.Fields { fType := strings.TrimPrefix(f.Type, "[]") - if strings.HasPrefix(fType, typ) { + if strings.HasPrefix(fType, Typ) { return true } } @@ -171,8 +200,8 @@ func (r Result) UsesType(typ string) bool { return false } -func (r Result) UsesArrays() bool { - for _, strct := range r.Structs() { +func UsesArrays(r Generateable, settings GenerateSettings) bool { + for _, strct := range r.Structs(settings) { for _, f := range strct.Fields { if strings.HasPrefix(f.Type, "[]") { return true @@ -182,58 +211,58 @@ func (r Result) UsesArrays() bool { return false } -func (r Result) Imports(settings PackageSettings) func(string) [][]string { +func Imports(r Generateable, settings GenerateSettings) func(string) [][]string { return func(filename string) [][]string { if filename == "db.go" { imps := []string{"context", "database/sql"} - if settings.EmitPreparedQueries { + if settings.PackageMap[r.PkgName()].EmitPreparedQueries { imps = append(imps, "fmt") } return [][]string{imps} } if filename == "models.go" { - return r.ModelImports() + return ModelImports(r, settings) } - return r.QueryImports(filename) + return QueryImports(r, settings, filename) } } -func (r Result) ModelImports() [][]string { +func ModelImports(r Generateable, settings GenerateSettings) [][]string { std := make(map[string]struct{}) - if r.UsesType("sql.Null") { + if UsesType(r, "sql.Null", settings) { std["database/sql"] = struct{}{} } - if r.UsesType("json.RawMessage") { + if UsesType(r, "json.RawMessage", settings) { std["encoding/json"] = struct{}{} } - if r.UsesType("time.Time") { + if UsesType(r, "time.Time", settings) { std["time"] = struct{}{} } - if r.UsesType("net.IP") { + if UsesType(r, "net.IP", settings) { std["net"] = struct{}{} } // Custom imports pkg := make(map[string]struct{}) overrideTypes := map[string]string{} - for _, o := range append(r.Settings.Overrides, r.packageSettings.Overrides...) { + for _, o := range append(settings.Overrides, settings.PackageMap[r.PkgName()].Overrides...) { overrideTypes[o.goTypeName] = o.goPackage } _, overrideNullTime := overrideTypes["pq.NullTime"] - if r.UsesType("pq.NullTime") && !overrideNullTime { + if UsesType(r, "pq.NullTime", settings) && !overrideNullTime { pkg["github.com/lib/pq"] = struct{}{} } _, overrideUUID := overrideTypes["uuid.UUID"] - if r.UsesType("uuid.UUID") && !overrideUUID { + if UsesType(r, "uuid.UUID", settings) && !overrideUUID { pkg["github.com/google/uuid"] = struct{}{} } for goType, importPath := range overrideTypes { - if _, ok := std[importPath]; !ok && r.UsesType(goType) { + if _, ok := std[importPath]; !ok && UsesType(r, goType, settings) { pkg[importPath] = struct{}{} } } @@ -253,7 +282,7 @@ func (r Result) ModelImports() [][]string { return [][]string{stds, pkgs} } -func (r Result) QueryImports(filename string) [][]string { +func QueryImports(r Generateable, settings GenerateSettings, filename string) [][]string { // for _, strct := range r.Structs() { // for _, f := range strct.Fields { // if strings.HasPrefix(f.Type, "[]") { @@ -262,7 +291,7 @@ func (r Result) QueryImports(filename string) [][]string { // } // } var gq []GoQuery - for _, query := range r.GoQueries() { + for _, query := range r.GoQueries(settings) { if query.SourceName == filename { gq = append(gq, query) } @@ -350,7 +379,7 @@ func (r Result) QueryImports(filename string) [][]string { pkg := make(map[string]struct{}) overrideTypes := map[string]string{} - for _, o := range append(r.Settings.Overrides, r.packageSettings.Overrides...) { + for _, o := range append(settings.Overrides, settings.PackageMap[r.PkgName()].Overrides...) { overrideTypes[o.goTypeName] = o.goPackage } @@ -400,7 +429,7 @@ func enumValueName(value string) string { return name } -func (r Result) Enums() []GoEnum { +func (r Result) Enums(settings GenerateSettings) []GoEnum { var enums []GoEnum for name, schema := range r.Catalog.Schemas { if name == "pg_catalog" { @@ -414,7 +443,7 @@ func (r Result) Enums() []GoEnum { enumName = name + "_" + enum.Name } e := GoEnum{ - Name: r.structName(enumName), + Name: StructName(enumName, settings), Comment: enum.Comment, } for _, v := range enum.Vals { @@ -433,8 +462,8 @@ func (r Result) Enums() []GoEnum { return enums } -func (r Result) structName(name string) string { - if rename := r.Settings.Rename[name]; rename != "" { +func StructName(name string, settings GenerateSettings) string { + if rename := settings.Rename[name]; rename != "" { return rename } out := "" @@ -448,7 +477,7 @@ func (r Result) structName(name string) string { return out } -func (r Result) Structs() []GoStruct { +func (r Result) Structs(settings GenerateSettings) []GoStruct { var structs []GoStruct for name, schema := range r.Catalog.Schemas { if name == "pg_catalog" { @@ -462,14 +491,14 @@ func (r Result) Structs() []GoStruct { tableName = name + "_" + table.Name } s := GoStruct{ - Table: &core.FQN{Schema: name, Rel: table.Name}, - Name: inflection.Singular(r.structName(tableName)), + Table: &FQNAlias{Schema: name, Rel: table.Name}, + Name: inflection.Singular(StructName(tableName, settings)), Comment: table.Comment, } for _, column := range table.Columns { s.Fields = append(s.Fields, GoField{ - Name: r.structName(column.Name), - Type: r.goType(column), + Name: StructName(column.Name, settings), + Type: r.goType(column, settings), Tags: map[string]string{"json:": column.Name}, Comment: column.Comment, }) @@ -483,26 +512,26 @@ func (r Result) Structs() []GoStruct { return structs } -func (r Result) goType(col core.Column) string { +func (r Result) goType(col core.Column, settings GenerateSettings) string { // package overrides have a higher precedence - for _, oride := range append(r.Settings.Overrides, r.packageSettings.Overrides...) { + for _, oride := range append(settings.Overrides, settings.PackageMap[r.PkgName()].Overrides...) { if oride.Column != "" && oride.columnName == col.Name && oride.table == col.Table { return oride.goTypeName } } - typ := r.goInnerType(col) + Typ := r.goInnerType(col, settings) if col.IsArray { - return "[]" + typ + return "[]" + Typ } - return typ + return Typ } -func (r Result) goInnerType(col core.Column) string { +func (r Result) goInnerType(col core.Column, settings GenerateSettings) string { columnType := col.DataType notNull := col.NotNull || col.IsArray // package overrides have a higher precedence - for _, oride := range append(r.Settings.Overrides, r.packageSettings.Overrides...) { + for _, oride := range append(settings.Overrides, settings.PackageMap[r.PkgName()].Overrides...) { if oride.PostgresType != "" && oride.PostgresType == columnType && oride.Null != notNull { return oride.goTypeName } @@ -609,10 +638,10 @@ func (r Result) goInnerType(col core.Column) string { for _, enum := range schema.Enums { if columnType == enum.Name { if name == "public" { - return r.structName(enum.Name) + return StructName(enum.Name, settings) } - return r.structName(name + "_" + enum.Name) + return StructName(name+"_"+enum.Name, settings) } } } @@ -628,21 +657,21 @@ func (r Result) goInnerType(col core.Column) string { // JSON tags: count, count_2, count_2 // // This is unlikely to happen, so don't fix it yet -func (r Result) columnsToStruct(name string, columns []core.Column) *GoStruct { +func (r Result) columnsToStruct(name string, columns []core.Column, settings GenerateSettings) *GoStruct { gs := GoStruct{ Name: name, } seen := map[string]int{} for i, c := range columns { tagName := c.Name - fieldName := r.structName(columnName(c, i)) + fieldName := StructName(columnName(c, i), settings) if v := seen[c.Name]; v > 0 { tagName = fmt.Sprintf("%s_%d", tagName, v+1) fieldName = fmt.Sprintf("%s_%d", fieldName, v+1) } gs.Fields = append(gs.Fields, GoField{ Name: fieldName, - Type: r.goType(c), + Type: r.goType(c, settings), Tags: map[string]string{"json:": tagName}, }) seen[c.Name]++ @@ -688,8 +717,8 @@ func compareFQN(a *core.FQN, b *core.FQN) bool { return a.Catalog == b.Catalog && a.Schema == b.Schema && a.Rel == b.Rel } -func (r Result) GoQueries() []GoQuery { - structs := r.Structs() +func (r Result) GoQueries(settings GenerateSettings) []GoQuery { + structs := r.Structs(settings) qs := make([]GoQuery, 0, len(r.Queries)) for _, query := range r.Queries { @@ -702,8 +731,8 @@ func (r Result) GoQueries() []GoQuery { gq := GoQuery{ Cmd: query.Cmd, - ConstantName: lowerTitle(query.Name), - FieldName: lowerTitle(query.Name) + "Stmt", + ConstantName: LowerTitle(query.Name), + FieldName: LowerTitle(query.Name) + "Stmt", MethodName: query.Name, SourceName: query.Filename, SQL: query.SQL, @@ -714,7 +743,7 @@ func (r Result) GoQueries() []GoQuery { p := query.Params[0] gq.Arg = GoQueryValue{ Name: paramName(p), - typ: r.goType(p.Column), + Typ: r.goType(p.Column, settings), } } else if len(query.Params) > 1 { var cols []core.Column @@ -724,7 +753,7 @@ func (r Result) GoQueries() []GoQuery { gq.Arg = GoQueryValue{ Emit: true, Name: "arg", - Struct: r.columnsToStruct(gq.MethodName+"Params", cols), + Struct: r.columnsToStruct(gq.MethodName+"Params", cols, settings), } } @@ -732,7 +761,7 @@ func (r Result) GoQueries() []GoQuery { c := query.Columns[0] gq.Ret = GoQueryValue{ Name: columnName(c, 0), - typ: r.goType(c), + Typ: r.goType(c, settings), } } else if len(query.Columns) > 1 { var gs *GoStruct @@ -745,9 +774,10 @@ func (r Result) GoQueries() []GoQuery { same := true for i, f := range s.Fields { c := query.Columns[i] - sameName := f.Name == r.structName(columnName(c, i)) - sameType := f.Type == r.goType(c) - sameFQN := compareFQN(s.Table, &c.Table) + sameName := f.Name == StructName(columnName(c, i), settings) + sameType := f.Type == r.goType(c, settings) + // TODO: consider making this deep equality from stdlib? + sameFQN := s.Table.EqualTo(&c.Table) if !sameName || !sameType || !sameFQN { same = false } @@ -759,7 +789,7 @@ func (r Result) GoQueries() []GoQuery { } if gs == nil { - gs = r.columnsToStruct(gq.MethodName+"Row", query.Columns) + gs = r.columnsToStruct(gq.MethodName+"Row", query.Columns, settings) emit = true } gq.Ret = GoQueryValue{ @@ -1046,22 +1076,22 @@ type tmplCtx struct { EmitPreparedQueries bool } -func lowerTitle(s string) string { +func LowerTitle(s string) string { a := []rune(s) a[0] = unicode.ToLower(a[0]) return string(a) } -func Generate(r *Result, global GenerateSettings, settings PackageSettings) (map[string]string, error) { - r.packageSettings = settings +func Generate(r Generateable, settings GenerateSettings) (map[string]string, error) { funcMap := template.FuncMap{ - "lowerTitle": lowerTitle, - "imports": r.Imports(settings), + "lowerTitle": LowerTitle, + "imports": Imports(r, settings), } - pkg := settings.Name - if pkg == "" { - pkg = filepath.Base(settings.Path) + pkgName := r.PkgName() + pkgConfig := settings.PackageMap[pkgName] + if pkgName == "" { + pkgName = filepath.Base(pkgConfig.Path) } dbFile := template.Must(template.New("table").Funcs(funcMap).Parse(dbTmpl)) @@ -1069,14 +1099,14 @@ func Generate(r *Result, global GenerateSettings, settings PackageSettings) (map sqlFile := template.Must(template.New("table").Funcs(funcMap).Parse(sqlTmpl)) tctx := tmplCtx{ - Settings: global, - EmitPreparedQueries: settings.EmitPreparedQueries, - EmitJSONTags: settings.EmitJSONTags, + Settings: settings, + EmitPreparedQueries: pkgConfig.EmitPreparedQueries, + EmitJSONTags: pkgConfig.EmitJSONTags, Q: "`", - Package: pkg, - GoQueries: r.GoQueries(), - Enums: r.Enums(), - Structs: r.Structs(), + Package: pkgName, + GoQueries: r.GoQueries(settings), + Enums: r.Enums(settings), + Structs: r.Structs(settings), } output := map[string]string{} @@ -1110,7 +1140,7 @@ func Generate(r *Result, global GenerateSettings, settings PackageSettings) (map } files := map[string]struct{}{} - for _, gq := range r.GoQueries() { + for _, gq := range r.GoQueries(settings) { files[gq.SourceName] = struct{}{} } diff --git a/internal/dinosql/gen_test.go b/internal/dinosql/gen_test.go index 2383fbe590..a1e4869cfc 100644 --- a/internal/dinosql/gen_test.go +++ b/internal/dinosql/gen_test.go @@ -8,6 +8,7 @@ import ( ) func TestColumnsToStruct(t *testing.T) { + pkgName := "db" cols := []pg.Column{ { Name: "other", @@ -52,7 +53,9 @@ func TestColumnsToStruct(t *testing.T) { cols[i].Table = pg.FQN{Schema: "public", Rel: "foo"} } - r := Result{} + r := Result{ + packageName: pkgName, + } // set up column-based override test o := Override{ @@ -67,12 +70,11 @@ func TestColumnsToStruct(t *testing.T) { Column: "foo.languages", } oa.Parse() + // pkgConfig := PackageSettings{ + // Overrides: []Override{o, oa}, + // } - r.packageSettings = PackageSettings{ - Overrides: []Override{o, oa}, - } - - actual := r.columnsToStruct("Foo", cols) + actual := r.columnsToStruct("Foo", cols, mockSettings) expected := &GoStruct{ Name: "Foo", Fields: []GoField{ @@ -90,6 +92,21 @@ func TestColumnsToStruct(t *testing.T) { } } +var mockSettings GenerateSettings + +func init() { + mockSettings = GenerateSettings{ + Version: "1", + Packages: []PackageSettings{ + PackageSettings{ + Name: "db", + }, + }, + Overrides: []Override{}, + } + mockSettings.PopulatePkgMap() +} + func TestInnerType(t *testing.T) { r := Result{} types := map[string]string{ @@ -110,8 +127,8 @@ func TestInnerType(t *testing.T) { goType := v t.Run(k+"-"+v, func(t *testing.T) { col := pg.Column{DataType: dbType, NotNull: true} - if goType != r.goType(col) { - t.Errorf("expected Go type for %s to be %s, not %s", dbType, goType, r.goType(col)) + if goType != r.goType(col, mockSettings) { + t.Errorf("expected Go type for %s to be %s, not %s", dbType, goType, r.goType(col, mockSettings)) } }) } @@ -137,8 +154,8 @@ func TestNullInnerType(t *testing.T) { goType := v t.Run(k+"-"+v, func(t *testing.T) { col := pg.Column{DataType: dbType, NotNull: false} - if goType != r.goType(col) { - t.Errorf("expected Go type for %s to be %s, not %s", dbType, goType, r.goType(col)) + if goType != r.goType(col, mockSettings) { + t.Errorf("expected Go type for %s to be %s, not %s", dbType, goType, r.goType(col, mockSettings)) } }) } diff --git a/internal/dinosql/parser.go b/internal/dinosql/parser.go index 52a8b28d2a..08a9d59304 100644 --- a/internal/dinosql/parser.go +++ b/internal/dinosql/parser.go @@ -171,33 +171,32 @@ type Query struct { } type Result struct { - Settings GenerateSettings - Queries []*Query - Catalog core.Catalog - - // XXX: this is hack so that all of the functions used during Generate can access - // package settings during that process without threading them through every function - // call. we should probably have another type just for generation instead of reusing Result - packageSettings PackageSettings + Queries []*Query + Catalog core.Catalog + packageName string } -func ParseQueries(c core.Catalog, settings GenerateSettings, pkg PackageSettings) (*Result, error) { - f, err := os.Stat(pkg.Queries) +func (r Result) PkgName() string { + return r.packageName +} + +func ParseQueries(c core.Catalog, pkgConfig PackageSettings) (*Result, error) { + f, err := os.Stat(pkgConfig.Queries) if err != nil { - return nil, fmt.Errorf("path %s does not exist", pkg.Queries) + return nil, fmt.Errorf("path %s does not exist", pkgConfig.Queries) } var files []string if f.IsDir() { - listing, err := ioutil.ReadDir(pkg.Queries) + listing, err := ioutil.ReadDir(pkgConfig.Queries) if err != nil { return nil, err } for _, f := range listing { - files = append(files, filepath.Join(pkg.Queries, f.Name())) + files = append(files, filepath.Join(pkgConfig.Queries, f.Name())) } } else { - files = append(files, pkg.Queries) + files = append(files, pkgConfig.Queries) } merr := NewParserErr() @@ -247,9 +246,13 @@ func ParseQueries(c core.Catalog, settings GenerateSettings, pkg PackageSettings return nil, merr } if len(q) == 0 { - return nil, fmt.Errorf("path %s contains no queries", pkg.Queries) + return nil, fmt.Errorf("path %s contains no queries", pkgConfig.Queries) } - return &Result{Catalog: c, Queries: q, Settings: settings}, nil + return &Result{ + Catalog: c, + Queries: q, + packageName: pkgConfig.Name, + }, nil } func location(node nodes.Node) int { diff --git a/internal/dinosql/parser_test.go b/internal/dinosql/parser_test.go index 398a18c1f6..00d45db8f7 100644 --- a/internal/dinosql/parser_test.go +++ b/internal/dinosql/parser_test.go @@ -164,7 +164,8 @@ func TestParseSchema(t *testing.T) { t.Fatal(err) } - q, err := ParseQueries(c, GenerateSettings{}, PackageSettings{ + q, err := ParseQueries(c, PackageSettings{ + Name: "db", Queries: filepath.Join("testdata", "ondeck", "query"), EmitJSONTags: true, }) @@ -173,10 +174,7 @@ func TestParseSchema(t *testing.T) { } t.Run("default", func(t *testing.T) { - output, err := Generate(q, GenerateSettings{}, PackageSettings{ - Name: "ondeck", - EmitJSONTags: true, - }) + output, err := Generate(q, mockSettings) if err != nil { t.Fatal(err) } @@ -185,10 +183,7 @@ func TestParseSchema(t *testing.T) { }) t.Run("prepared", func(t *testing.T) { - output, err := Generate(q, GenerateSettings{}, PackageSettings{ - Name: "prepared", - EmitPreparedQueries: true, - }) + output, err := Generate(q, mockSettings) if err != nil { t.Fatal(err) } diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go new file mode 100644 index 0000000000..9ffe21847c --- /dev/null +++ b/internal/mysql/gen.go @@ -0,0 +1,263 @@ +package mysql + +import ( + "fmt" + "sort" + "strings" + + "github.com/jinzhu/inflection" + "github.com/kyleconroy/sqlc/internal/dinosql" + "vitess.io/vitess/go/vt/sqlparser" +) + +// Result holds the mysql validated queries schema +type Result struct { + Queries []*Query + Schema *Schema + packageName string +} + +func (r *Result) PkgName() string { + return r.packageName +} + +// Enums generates parser-agnostic GoEnum types +func (r *Result) Enums(settings dinosql.GenerateSettings) []dinosql.GoEnum { + var enums []dinosql.GoEnum + for _, table := range r.Schema.tables { + for _, col := range table { + if col.Type.Type == "enum" { + constants := []dinosql.GoConstant{} + enumName := enumNameFromColDef(col, settings) + for _, c := range col.Type.EnumValues { + stripped := stripInnerQuotes(c) + constants = append(constants, dinosql.GoConstant{ + // TODO: maybe add the struct name call to capitalize the name here + Name: stripped, + Value: stripped, + Type: enumName, + }) + } + + goEnum := dinosql.GoEnum{ + Name: enumName, + Comment: "", + Constants: constants, + } + enums = append(enums, goEnum) + } + } + } + return enums +} + +func stripInnerQuotes(identifier string) string { + return strings.Replace(identifier, "'", "", 2) +} + +func enumNameFromColDef(col *sqlparser.ColumnDefinition, settings dinosql.GenerateSettings) string { + return fmt.Sprintf("%sType", + dinosql.StructName(col.Name.String(), settings)) +} + +// Structs marshels each query into a go struct for generation +func (r *Result) Structs(settings dinosql.GenerateSettings) []dinosql.GoStruct { + var structs []dinosql.GoStruct + for tableName, cols := range r.Schema.tables { + s := dinosql.GoStruct{ + Name: inflection.Singular(dinosql.StructName(tableName, settings)), + } + + for _, col := range cols { + s.Fields = append(s.Fields, dinosql.GoField{ + Name: dinosql.StructName(col.Name.String(), settings), + Type: goTypeCol(col, settings), + Tags: map[string]string{"json": col.Name.String()}, + Comment: "", + }) + } + structs = append(structs, s) + } + + return structs +} + +// GoQueries generates parser-agnostic query information for code generation +func (r *Result) GoQueries(settings dinosql.GenerateSettings) []dinosql.GoQuery { + structs := r.Structs(settings) + + qs := make([]dinosql.GoQuery, 0, len(r.Queries)) + for ix, query := range r.Queries { + if query == nil { + panic(fmt.Sprintf("query is nil on index: %v, len: %v", ix, len(r.Queries))) + } + if query.Name == "" { + continue + } + if query.Cmd == "" { + continue + } + + gq := dinosql.GoQuery{ + Cmd: query.Cmd, + ConstantName: dinosql.LowerTitle(query.Name), + FieldName: dinosql.LowerTitle(query.Name) + "Stmt", + MethodName: query.Name, + SourceName: "queries", // query.Filename, + SQL: query.SQL, + // Comments: query.Comments, + } + + if len(query.Params) == 1 { + p := query.Params[0] + gq.Arg = dinosql.GoQueryValue{ + Name: p.name, + Typ: p.typ, + } + } else if len(query.Params) > 1 { + + structInfo := make([]structParams, len(query.Params)) + for i := range query.Params { + structInfo[i] = structParams{ + originalName: query.Params[i].name, + goType: query.Params[i].typ, + } + } + + gq.Arg = dinosql.GoQueryValue{ + Emit: true, + Name: "arg", + Struct: r.columnsToStruct(gq.MethodName+"Params", structInfo, settings), + } + } + + if len(query.Columns) == 1 { + c := query.Columns[0] + gq.Ret = dinosql.GoQueryValue{ + Name: columnName(c, 0), + Typ: goTypeCol(c, settings), + } + } else if len(query.Columns) > 1 { + var gs *dinosql.GoStruct + var emit bool + + for _, s := range structs { + if len(s.Fields) != len(query.Columns) { + continue + } + same := true + for i, f := range s.Fields { + c := query.Columns[i] + sameName := f.Name == dinosql.StructName(columnName(c, i), settings) + sameType := f.Type == goTypeCol(c, settings) + // TODO: consider making this deep equality from stdlib? + // sameFQN := s.Table.EqualTo(&c.Table) + if !sameName || !sameType || true { // !sameFQN + same = false + } + } + if same { + gs = &s + break + } + } + + if gs == nil { + structInfo := make([]structParams, len(query.Columns)) + for i := range query.Columns { + structInfo[i] = structParams{ + originalName: query.Columns[i].Name.String(), + goType: goTypeCol(query.Columns[i], settings), + } + } + gs = r.columnsToStruct(gq.MethodName+"Row", structInfo, settings) + emit = true + } + gq.Ret = dinosql.GoQueryValue{ + Emit: emit, + Name: "i", + Struct: gs, + } + } + + qs = append(qs, gq) + } + sort.Slice(qs, func(i, j int) bool { return qs[i].MethodName < qs[j].MethodName }) + return qs +} + +type structParams struct { + originalName string + goType string +} + +func (r *Result) columnsToStruct(name string, items []structParams, settings dinosql.GenerateSettings) *dinosql.GoStruct { + gs := dinosql.GoStruct{ + Name: name, + } + seen := map[string]int{} + for _, item := range items { + name := item.originalName + typ := item.goType + tagName := name + fieldName := dinosql.StructName(name, settings) + if v := seen[name]; v > 0 { + tagName = fmt.Sprintf("%s_%d", tagName, v+1) + fieldName = fmt.Sprintf("%s_%d", fieldName, v+1) + } + gs.Fields = append(gs.Fields, dinosql.GoField{ + Name: fieldName, + Type: typ, + Tags: map[string]string{"json:": tagName}, + }) + seen[name]++ + } + return &gs +} + +func goTypeCol(col *sqlparser.ColumnDefinition, settings dinosql.GenerateSettings) string { + switch t := col.Type.Type; { + case "varchar" == t: + if col.Type.NotNull { + return "string" + } + return "sql.NullString" + case "int" == t: + if col.Type.NotNull { + return "int" + } + return "sql.NullInt64" + case "float" == t, strings.HasPrefix(strings.ToLower(t), "decimal"): + if col.Type.NotNull { + return "float64" + } + return "sql.NullFloat64" + case "enum" == t: + return enumNameFromColDef(col, settings) + default: + // TODO: remove panic here + panic(fmt.Sprintf("Handle this col type directly: %v\n", col.Type)) + // return col.Type + } +} + +func columnName(c *sqlparser.ColumnDefinition, pos int) string { + if !c.Name.IsEmpty() { + return c.Name.String() + } + return fmt.Sprintf("column_%d", pos+1) +} + +func argName(name string) string { + out := "" + for i, p := range strings.Split(name, "_") { + if i == 0 { + out += strings.ToLower(p) + } else if p == "id" { + out += "ID" + } else { + out += strings.Title(p) + } + } + return out +} diff --git a/internal/mysql/param.go b/internal/mysql/param.go new file mode 100644 index 0000000000..f950a2c118 --- /dev/null +++ b/internal/mysql/param.go @@ -0,0 +1,128 @@ +package mysql + +import ( + "fmt" + "strings" + + "github.com/kyleconroy/sqlc/internal/dinosql" + "vitess.io/vitess/go/vt/sqlparser" +) + +type Param struct { + originalName string + name string + typ string +} + +func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, settings dinosql.GenerateSettings) ([]*Param, error) { + params := []*Param{} + if limit == nil { + return params, nil + } + + parseLimitSubExp := func(node sqlparser.Expr) { + switch v := node.(type) { + case *sqlparser.SQLVal: + if v.Type == sqlparser.ValArg { + params = append(params, &Param{ + originalName: string(v.Val), + name: "limit", + typ: "uint32", + }) + } + } + } + + parseLimitSubExp(limit.Offset) + parseLimitSubExp(limit.Rowcount) + + return params, nil +} + +func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, defaultTable string, settings dinosql.GenerateSettings) ([]*Param, error) { + params := []*Param{} + switch v := e.(type) { + case *sqlparser.Where: + if v == nil { + return params, nil + } + return paramsInWhereExpr(v.Expr, s, defaultTable, settings) + case *sqlparser.ComparisonExpr: + p, found, err := paramInComparison(v, s, defaultTable, settings) + if err != nil { + return nil, err + } + if found { + params = append(params, p) + } + case *sqlparser.AndExpr: + left, err := paramsInWhereExpr(v.Left, s, defaultTable, settings) + if err != nil { + return nil, err + } + params = append(params, left...) + right, err := paramsInWhereExpr(v.Right, s, defaultTable, settings) + if err != nil { + return nil, err + } + params = append(params, right...) + case *sqlparser.OrExpr: + left, err := paramsInWhereExpr(v.Left, s, defaultTable, settings) + if err != nil { + return nil, err + } + params = append(params, left...) + right, err := paramsInWhereExpr(v.Right, s, defaultTable, settings) + if err != nil { + return nil, err + } + params = append(params, right...) + default: + panic(fmt.Sprintf("Failed to handle %T in where", v)) + } + + return params, nil +} + +func paramInComparison(cond *sqlparser.ComparisonExpr, s *Schema, defaultTable string, settings dinosql.GenerateSettings) (*Param, bool, error) { + p := &Param{} + var colIdent sqlparser.ColIdent + walker := func(node sqlparser.SQLNode) (bool, error) { + switch v := node.(type) { + case *sqlparser.ColName: + colDfn, err := s.getColType(v, defaultTable) + if err != nil { + return false, err + } + p.typ = goTypeCol(colDfn, settings) + colIdent = colDfn.Name + + case *sqlparser.SQLVal: + if v.Type == sqlparser.ValArg { + p.originalName = string(v.Val) + } + } + return true, nil + } + err := sqlparser.Walk(walker, cond) + if err != nil { + return nil, false, err + } + if p.originalName != "" && p.typ != "" { + p.name = paramName(colIdent, p.originalName) + return p, true, nil + } + return nil, false, nil +} + +func paramName(col sqlparser.ColIdent, originalName string) string { + str := col.String() + if !strings.HasPrefix(originalName, ":v") { + return originalName[1:] + } + if str != "" { + return str + } + num := originalName[2] + return fmt.Sprintf("param%v", num) +} diff --git a/internal/mysql/param_test.go b/internal/mysql/param_test.go new file mode 100644 index 0000000000..feb66fb866 --- /dev/null +++ b/internal/mysql/param_test.go @@ -0,0 +1,164 @@ +package mysql + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "vitess.io/vitess/go/vt/sqlparser" +) + +func TestSelectParamSearcher(t *testing.T) { + type testCase struct { + input string + output []*Param + } + + tests := []testCase{ + testCase{ + input: "SELECT first_name, id, last_name FROM users WHERE id < ?", + output: []*Param{&Param{ + originalName: ":v1", + name: "id", + typ: "int", + }, + }, + }, + testCase{ + input: `SELECT + users.id, + users.first_name, + orders.price + FROM + orders + LEFT JOIN users ON orders.user_id = users.id + WHERE orders.price > :minPrice`, + output: []*Param{ + &Param{ + originalName: ":minPrice", + name: "minPrice", + typ: "float64", + }, + }, + }, + testCase{ + input: "SELECT first_name, id, last_name FROM users WHERE id = :targetID", + output: []*Param{&Param{ + originalName: ":targetID", + name: "targetID", + typ: "int", + }, + }, + }, + testCase{ + input: "SELECT first_name, last_name FROM users WHERE age < :maxAge AND last_name = :inFamily", + output: []*Param{ + &Param{ + originalName: ":maxAge", + name: "maxAge", + typ: "int", + }, + &Param{ + originalName: ":inFamily", + name: "inFamily", + typ: "sql.NullString", + }, + }, + }, + testCase{ + input: "SELECT first_name, last_name FROM users LIMIT ?", + output: []*Param{ + &Param{ + originalName: ":v1", + name: "limit", + typ: "uint32", + }, + }, + }, + } + for _, tCase := range tests { + tree, err := sqlparser.Parse(tCase.input) + if err != nil { + t.Errorf("Failed to parse input query") + } + selectStm, ok := tree.(*sqlparser.Select) + + limitParams, err := paramsInLimitExpr(selectStm.Limit, mockSchema, mockSettings) + if err != nil { + t.Errorf("Failed to parse limit expression params: %v", err) + } + whereParams, err := paramsInWhereExpr(selectStm.Where, mockSchema, "users", mockSettings) + if err != nil { + t.Errorf("Failed to parse where expression params: %v", err) + } + + params := append(limitParams, whereParams...) + if !ok { + t.Errorf("Test case is not SELECT statement as expected") + } + + // TODO: get this out of the unit test and/or deprecate defaultTable + defaultTable := getDefaultTable(&selectStm.From) + keep(defaultTable) + + if !reflect.DeepEqual(params, tCase.output) { + t.Errorf("Param searcher returned unexpected result\nResult: %v\nExpected: %v", + spew.Sdump(params), spew.Sdump(tCase.output)) + } + } +} + +func TestInsertParamSearcher(t *testing.T) { + type testCase struct { + input string + output []*Param + expectedNames []string + } + + tests := []testCase{ + testCase{ + input: "INSERT INTO users (first_name, last_name) VALUES (?, ?)", + output: []*Param{ + &Param{ + originalName: ":v1", + name: "first_name", + typ: "string", + }, + &Param{ + originalName: ":v2", + name: "last_name", + typ: "sql.NullString", + }, + }, + expectedNames: []string{"first_name", "last_name"}, + }, + } + for _, tCase := range tests { + tree, err := sqlparser.Parse(tCase.input) + if err != nil { + t.Errorf("Failed to parse input query") + } + insertStm, ok := tree.(*sqlparser.Insert) + if !ok { + t.Errorf("Test case is not SELECT statement as expected") + } + result, err := parseInsert(insertStm, tCase.input, mockSchema, mockSettings) + if err != nil { + t.Errorf("Failed to parse insert statement.") + } + + if !reflect.DeepEqual(result.Params, tCase.output) { + t.Errorf("Param searcher returned unexpected result\nResult: %v\nExpected: %v\nQuery: %s", + spew.Sdump(result.Params), spew.Sdump(tCase.output), tCase.input) + } + if len(result.Params) != len(tCase.expectedNames) { + t.Errorf("Insufficient test cases. Mismatch in length of expected param names and parsed params") + } + for ix, p := range result.Params { + if p.name != tCase.expectedNames[ix] { + t.Errorf("Derived param does not match expected output.\nResult: %v\nExpected: %v", + p.name, tCase.expectedNames[ix]) + } + } + } +} diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go new file mode 100644 index 0000000000..357251fdab --- /dev/null +++ b/internal/mysql/parse.go @@ -0,0 +1,322 @@ +package mysql + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/kyleconroy/sqlc/internal/dinosql" + "vitess.io/vitess/go/vt/sqlparser" +) + +// Query holds the data for walking and validating mysql querys +type Query struct { + SQL string + Columns []*sqlparser.ColumnDefinition + Params []*Param + Name string + Cmd string // TODO: Pick a better name. One of: one, many, exec, execrows + defaultTableName string // for columns that are not qualified + schemaLookup *Schema +} + +func parseFile(filepath string, inPkg string, s *Schema, settings dinosql.GenerateSettings) (*Result, error) { + file, err := os.Open(filepath) + if err != nil { + return nil, fmt.Errorf("Failed to open file [%v]: %v", filepath, err) + } + contents, err := ioutil.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("Failed to read contents of file [%v]: %v", filepath, err) + } + rawQueries := strings.Split(string(contents), "\n\n") + + parsedQueries := []*Query{} + + for _, query := range rawQueries { + result, err := parseQueryString(query, s, settings) + if err != nil { + return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filepath, err) + } + if result == nil { + continue + } + parsedQueries = append(parsedQueries, result) + } + + r := Result{ + Queries: parsedQueries, + Schema: s, + packageName: inPkg, + } + return &r, nil +} + +func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { + tree, err := sqlparser.Parse(query) + + if err != nil { + return nil, err + } + + switch tree := tree.(type) { + case *sqlparser.Select: + defaultTableName := getDefaultTable(&tree.From) + res, err := parseSelect(tree, query, s, defaultTableName, settings) + if err != nil { + return nil, fmt.Errorf("Failed to parse SELECT query: %v", err) + } + return res, nil + case *sqlparser.Insert: + insert, err := parseInsert(tree, query, s, settings) + if err != nil { + return nil, fmt.Errorf("Failed to parse INSERT query: %v", err) + } + return insert, nil + case *sqlparser.Update: + update, err := parseUpdate(tree, query, s, settings) + if err != nil { + return nil, fmt.Errorf("Failed to parse UPDATE query: %v", err) + } + return update, nil + case *sqlparser.DDL: + s.Add(tree) + return nil, nil + default: + panic("Unsupported SQL statement type") + // return &Query{}, nil + } + return nil, fmt.Errorf("Failed to parse query statement: %v", query) +} + +func (q *Query) parseNameAndCmd() error { + if q == nil { + return errors.New("Cannot parse name and cmd from null query") + } + _, comments := sqlparser.SplitMarginComments(q.SQL) + err := q.parseLeadingComment(comments.Leading) + if err != nil { + return fmt.Errorf("Failed to parse leading comment %v", err) + } + return nil +} + +func parseSelect(tree *sqlparser.Select, query string, s *Schema, defaultTableName string, settings dinosql.GenerateSettings) (*Query, error) { + // handle * expressions first by expanding all columns of the default table + _, ok := tree.SelectExprs[0].(*sqlparser.StarExpr) + if ok { + colNames := []sqlparser.SelectExpr{} + colDfns := s.tables[defaultTableName] + for _, col := range colDfns { + colNames = append(colNames, &sqlparser.AliasedExpr{ + Expr: &sqlparser.ColName{ + Name: col.Name, + }}, + ) + } + tree.SelectExprs = colNames + } + parsedQuery := Query{ + SQL: query, + defaultTableName: defaultTableName, + schemaLookup: s, + } + err := sqlparser.Walk(parsedQuery.visit, tree) + if err != nil { + return nil, err + } + + whereParams, err := paramsInWhereExpr(tree.Where, s, defaultTableName, settings) + if err != nil { + return nil, err + } + + limitParams, err := paramsInLimitExpr(tree.Limit, s, settings) + if err != nil { + return nil, err + } + parsedQuery.Params = append(whereParams, limitParams...) + + err = parsedQuery.parseNameAndCmd() + if err != nil { + return nil, err + } + parsedQuery.SQL = sqlparser.String(tree) + + return &parsedQuery, nil +} + +func getDefaultTable(node *sqlparser.TableExprs) string { + // TODO: improve this + var tableName string + visit := func(node sqlparser.SQLNode) (bool, error) { + switch v := node.(type) { + case sqlparser.TableName: + if name := v.Name.String(); name != "" { + tableName = name + return false, nil + } + } + return true, nil + } + sqlparser.Walk(visit, node) + return tableName +} + +func parseUpdate(node *sqlparser.Update, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { + defaultTable := getDefaultTable(&node.TableExprs) + + params := []*Param{} + for _, updateExpr := range node.Exprs { + col := updateExpr.Name + newValue, isParam := updateExpr.Expr.(*sqlparser.SQLVal) + if !isParam { + continue + } + colDfn, err := s.getColType(col, defaultTable) + if err != nil { + return nil, fmt.Errorf("Failed to determine type of a parameter's column: %v", err) + } + originalParamName := string(newValue.Val) + param := Param{ + originalName: originalParamName, + name: paramName(colDfn.Name, originalParamName), + typ: goTypeCol(colDfn, settings), + } + params = append(params, ¶m) + } + + whereParams, err := paramsInWhereExpr(node.Where.Expr, s, defaultTable, settings) + if err != nil { + return nil, fmt.Errorf("Failed to parse params from WHERE expression: %v", err) + } + + parsedQuery := Query{ + SQL: query, + Columns: nil, + Params: append(params, whereParams...), + defaultTableName: defaultTable, + schemaLookup: s, + } + parsedQuery.parseNameAndCmd() + + return &parsedQuery, nil +} + +func parseInsert(node *sqlparser.Insert, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { + cols := node.Columns + tableName := node.Table.Name.String() + rows, ok := node.Rows.(sqlparser.Values) + if !ok { + return nil, fmt.Errorf("Unknown insert row type of %T", node.Rows) + } + + params := []*Param{} + + for _, row := range rows { + for colIx, item := range row { + switch v := item.(type) { + case *sqlparser.SQLVal: + if v.Type == sqlparser.ValArg { + colName := cols[colIx].String() + colDfn, _ := s.schemaLookup(tableName, colName) + varName := string(v.Val) + p := &Param{ + originalName: varName, + name: paramName(colDfn.Name, varName), + typ: goTypeCol(colDfn, settings), + } + params = append(params, p) + } + default: + panic("Error occurred in parsing INSERT statement") + } + } + } + parsedQuery := &Query{ + SQL: query, + Params: params, + Columns: nil, + defaultTableName: tableName, + schemaLookup: s, + } + parsedQuery.parseNameAndCmd() + return parsedQuery, nil +} + +func (q *Query) parseLeadingComment(comment string) error { + for _, line := range strings.Split(comment, "\n") { + if !strings.HasPrefix(line, "/* name:") { + continue + } + part := strings.Split(strings.TrimSpace(line), " ") + if len(part) == 3 { + return fmt.Errorf("missing query type [':one', ':many', ':exec', ':execrows']: %s", line) + } + if len(part) != 5 { + return fmt.Errorf("invalid query comment: %s", line) + } + queryName := part[2] + queryType := strings.TrimSpace(part[3]) + switch queryType { + case ":one", ":many", ":exec", ":execrows": + default: + return fmt.Errorf("invalid query type: %s", queryType) + } + // if err := validateQueryName(queryName); err != nil { + // return err + // } + q.Name = queryName + q.Cmd = queryType + } + return nil +} +func isExpr(exp sqlparser.Expr) {} + +func (q *Query) visit(node sqlparser.SQLNode) (bool, error) { + switch v := node.(type) { + case *sqlparser.AliasedExpr: + err := sqlparser.Walk(q.visitColNames, v) + if err != nil { + return false, err + } + case *sqlparser.StarExpr: + cols, ok := q.schemaLookup.tables[q.defaultTableName] + if !ok { + return false, fmt.Errorf("Failed to expand * expression into columns from schema, tableName: %v", q.defaultTableName) + } + q.Columns = cols + return false, nil + default: + // fmt.Printf("Did not handle %T\n", v) + } + return true, nil +} + +func (q *Query) visitColNames(node sqlparser.SQLNode) (bool, error) { + switch v := node.(type) { + case *sqlparser.ColName: + colTyp, err := q.schemaLookup.getColType(v, q.defaultTableName) + if err != nil { + return false, fmt.Errorf("Failed to get column type for [%v]: %v", v.Name.String(), err) + } + q.Columns = append(q.Columns, colTyp) + } + return true, nil +} + +func GeneratePkg(pkgName string, querysPath string, settings dinosql.GenerateSettings) (map[string]string, error) { + s := NewSchema() + result, err := parseFile(querysPath, pkgName, s, settings) + if err != nil { + return nil, err + } + output, err := dinosql.Generate(result, settings) + if err != nil { + return nil, fmt.Errorf("Failed to generate output: %v", err) + } + + return output, nil +} diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go new file mode 100644 index 0000000000..e9dca22324 --- /dev/null +++ b/internal/mysql/parse_test.go @@ -0,0 +1,366 @@ +package mysql + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/kyleconroy/sqlc/internal/dinosql" + "vitess.io/vitess/go/vt/sqlparser" +) + +func init() { + initMockSchema() +} + +const query = ` +/* name: GetAllStudents :many */ +SELECT school_id, id FROM students WHERE id = :id + ? +` + +const create = ` + CREATE TABLE students ( + id int, + school_id VARCHAR(255), + school_lat VARCHAR(255), + PRIMARY KEY (ID) + );` + +const filename = "test.sql" + +func TestParseFile(t *testing.T) { + // s := NewSchema() + // _, err := parseFile(filename, s) + // keep(err) + tree, _ := sqlparser.Parse("SELECT id, first_name FROM users WHERE age < ?") + p := sqlparser.NewParsedQuery(tree) + // spew.Dump(p) + // for k, _ := + result := sqlparser.GetBindvars(tree) + newVars := make(map[string]string) + for k := range result { + newVars[k] = "?" + } + // spew.Dump(newVars) + keep(p) + // p.GenerateQuery(newVars) + // r, _ := p.MarshalJSON() + // spew.Dump(string(r)) + // spew.Dump(p.GenerateQuery()) +} + +var mockSettings = dinosql.GenerateSettings{ + Version: "1", + Packages: []dinosql.PackageSettings{ + dinosql.PackageSettings{ + Name: "db", + }, + }, + Overrides: []dinosql.Override{}, +} + +func TestGenerate(t *testing.T) { + // t.Skip() + s := NewSchema() + result, _ := parseFile(filename, "db", s, mockSettings) + output, err := dinosql.Generate(result, mockSettings) + if err != nil { + t.Errorf("Failed to generate output: %v", err) + } + keep(output) + // for k, v := range output { + // fmt.Println(k) + // fmt.Println(v) + // fmt.Println("") + // } +} + +func keep(interface{}) {} + +var mockSchema *Schema + +func initMockSchema() { + var schemaMap = make(map[string][]*sqlparser.ColumnDefinition) + mockSchema = &Schema{ + tables: schemaMap, + } + schemaMap["users"] = []*sqlparser.ColumnDefinition{ + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("first_name"), + Type: sqlparser.ColumnType{ + Type: "varchar", + NotNull: true, + // could add more here later if needed + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("last_name"), + Type: sqlparser.ColumnType{ + Type: "varchar", + NotNull: false, + // could add more here later if needed + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("id"), + Type: sqlparser.ColumnType{ + Type: "int", + NotNull: true, + Autoincrement: true, + // could add more here later if needed + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("age"), + Type: sqlparser.ColumnType{ + Type: "int", + NotNull: true, + // could add more here later if needed + }, + }, + } + schemaMap["orders"] = []*sqlparser.ColumnDefinition{ + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("id"), + Type: sqlparser.ColumnType{ + Type: "int", + NotNull: true, + Autoincrement: true, + // could add more here later if needed + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("price"), + Type: sqlparser.ColumnType{ + Type: "DECIMAL(13, 4)", + NotNull: true, + Autoincrement: true, + // could add more here later if needed + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("user_id"), + Type: sqlparser.ColumnType{ + Type: "int", + NotNull: true, + // could add more here later if needed + }, + }, + } +} + +func filterCols(allCols []*sqlparser.ColumnDefinition, tableNames map[string]struct{}) []*sqlparser.ColumnDefinition { + filteredCols := []*sqlparser.ColumnDefinition{} + for _, col := range allCols { + if _, ok := tableNames[col.Name.String()]; ok { + filteredCols = append(filteredCols, col) + } + } + return filteredCols +} + +func TestParseSelect(t *testing.T) { + type expected struct { + query string + schema *Schema + } + type testCase struct { + input expected + output *Query + } + query2 := `/* name: GetAll :many */ + SELECT * FROM users;` + tests := []testCase{ + testCase{ + input: expected{ + query: `/* name: GetNameByID :one */ + SELECT first_name, last_name FROM users WHERE id = ?`, + schema: mockSchema, + }, + output: &Query{ + SQL: `select first_name, last_name from users where id = :v1`, + Columns: filterCols(mockSchema.tables["users"], map[string]struct{}{"first_name": struct{}{}, "last_name": struct{}{}}), + Params: []*Param{ + &Param{ + originalName: ":v1", + name: "id", + typ: "int", + }}, + Name: "GetNameByID", + Cmd: ":one", + defaultTableName: "users", + schemaLookup: mockSchema, + }, + }, + testCase{ + input: expected{ + query: query2, + schema: mockSchema, + }, + output: &Query{ + SQL: "select first_name, last_name, id, age from users", + Columns: mockSchema.tables["users"], + Params: []*Param{}, + Name: "GetAll", + Cmd: ":many", + defaultTableName: "users", + schemaLookup: mockSchema, + }, + }, + } + + for _, testCase := range tests { + q, err := parseQueryString(testCase.input.query, testCase.input.schema, mockSettings) + if err != nil { + t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", query, spew.Sdump(testCase.input.schema)) + } + + err = q.parseNameAndCmd() + if err != nil { + t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", query, spew.Sdump(testCase.input.schema)) + } + if !reflect.DeepEqual(testCase.output, q) { + t.Errorf("Parsing query returned differently than expected.") + t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) + } + } +} + +func TestParseLeadingComment(t *testing.T) { + type expected struct { + name string + cmd string + } + type testCase struct { + input string + output expected + } + + tests := []testCase{ + testCase{ + input: "/* name: GetPeopleByID :many */", + output: expected{name: "GetPeopleByID", cmd: ":many"}, + }, + } + + for _, tCase := range tests { + qu := &Query{} + err := qu.parseLeadingComment(tCase.input) + if err != nil { + t.Errorf("Failed to parse leading comment %v", err) + } + if qu.Name != tCase.output.name || qu.Cmd != tCase.output.cmd { + t.Errorf("Leading comment parser returned unexpcted result: %v\n:\n Expected: [%v]\nRecieved:[%v]\n", + err, spew.Sdump(tCase.output), spew.Sdump(qu)) + } + + } +} + +func TestSchemaLookup(t *testing.T) { + firstNameColDfn, err := mockSchema.schemaLookup("users", "first_name") + if err != nil { + t.Errorf("Failed to get column schema from mock schema: %v", err) + } + + expected := filterCols(mockSchema.tables["users"], map[string]struct{}{"first_name": struct{}{}}) + if !reflect.DeepEqual(firstNameColDfn, expected[0]) { + t.Errorf("Table schema lookup returned unexpected result") + } +} + +func TestParseInsert(t *testing.T) { + type expected struct { + query string + schema *Schema + } + type testCase struct { + input expected + output *Query + } + query1 := `/* name: InsertNewUser :exec */ + INSERT INTO users (first_name, last_name) VALUES (?, ?)` + query2 := `/* name: UpdateUserAt :exec */ + UPDATE users SET first_name = ?, last_name = ? WHERE id > ? AND first_name = ? LIMIT 3` + tests := []testCase{ + testCase{ + input: expected{ + query: query1, + schema: mockSchema, + }, + output: &Query{ + SQL: query1, + Columns: nil, + Params: []*Param{ + &Param{ + originalName: ":v1", + name: "first_name", + typ: "string", + }, + &Param{ + originalName: ":v2", + name: "last_name", + typ: "sql.NullString", + }, + }, + Name: "InsertNewUser", + Cmd: ":exec", + defaultTableName: "users", + schemaLookup: mockSchema, + }, + }, + testCase{ + input: expected{ + query: query2, + schema: mockSchema, + }, + output: &Query{ + SQL: query2, + Columns: nil, + Params: []*Param{ + &Param{ + originalName: ":v1", + name: "first_name", + typ: "string", + }, + &Param{ + originalName: ":v2", + name: "last_name", + typ: "sql.NullString", + }, + &Param{ + originalName: ":v3", + name: "id", + typ: "int", + }, + &Param{ + originalName: ":v4", + name: "first_name", + typ: "string", + }, + }, + Name: "UpdateUserAt", + Cmd: ":exec", + defaultTableName: "users", + schemaLookup: mockSchema, + }, + }, + } + + for ix, testCase := range tests { + q, err := parseQueryString(testCase.input.query, testCase.input.schema, mockSettings) + if err != nil { + t.Errorf("Parsing failed with query: [%v]\n", err) + continue + } + + err = q.parseNameAndCmd() + if err != nil { + t.Errorf("Parsing failed with query index: %d: [%v]\n", ix, query) + } + if !reflect.DeepEqual(testCase.output, q) { + t.Errorf("Parsing query returned differently than expected.") + t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) + } + } +} diff --git a/internal/mysql/schema.go b/internal/mysql/schema.go new file mode 100644 index 0000000000..111b9e2b7b --- /dev/null +++ b/internal/mysql/schema.go @@ -0,0 +1,48 @@ +package mysql + +import ( + "fmt" + + "vitess.io/vitess/go/vt/sqlparser" +) + +// NewSchema gives a newly instantiated MySQL schema map +func NewSchema() *Schema { + return &Schema{ + tables: make(map[string]([]*sqlparser.ColumnDefinition)), + } +} + +// Schema proves that information for mapping columns in queries to their respective table definitions +// and validating that they are correct so as to map to the correct Go type +type Schema struct { + tables map[string]([]*sqlparser.ColumnDefinition) +} + +func (s *Schema) getColType(col *sqlparser.ColName, defaultTableName string) (*sqlparser.ColumnDefinition, error) { + if !col.Qualifier.IsEmpty() { + return s.schemaLookup(col.Qualifier.Name.String(), col.Name.String()) + } + return s.schemaLookup(defaultTableName, col.Name.String()) +} + +// Add add a MySQL table definition to the schema map +func (s *Schema) Add(table *sqlparser.DDL) { + name := table.Table.Name.String() + s.tables[name] = table.TableSpec.Columns +} + +func (s *Schema) schemaLookup(table string, col string) (*sqlparser.ColumnDefinition, error) { + cols, ok := s.tables[table] + if !ok { + return nil, fmt.Errorf("Table [%v] not found in Schema", table) + } + + for _, colDef := range cols { + if colDef.Name.EqualString(col) { + return colDef, nil + } + } + + return nil, fmt.Errorf("Column [%v] not found in table [%v]", col, table) +} diff --git a/internal/mysql/test.sql b/internal/mysql/test.sql new file mode 100644 index 0000000000..3ec9c0898b --- /dev/null +++ b/internal/mysql/test.sql @@ -0,0 +1,16 @@ +CREATE TABLE students ( + id int NOT NULL, + school_id VARCHAR(255) NOT NULL, + school_lat FLOAT, + department ENUM("English", "Math"), + PRIMARY KEY (ID) +); + +/* name: GetAllStudents :many */ +SELECT school_id, id FROM students WHERE id = ? AND school_id = ? + +/* name: GetSomeStudents :one */ +SELECT school_id, id FROM students WHERE school_id = ? + +/* name: StudentByID :one */ +SELECT id, school_lat FROM students WHERE id = ? LIMIT 10 \ No newline at end of file From 129a851ea295020abf2383d4c6dcdb6c70c38a2e Mon Sep 17 00:00:00 2001 From: Charles Moog Date: Tue, 31 Dec 2019 13:35:38 +1300 Subject: [PATCH 02/24] MySQL (#2) (#2) * adds support for mysql functions and select alias exprs * adds example usage * cleans tests * patches generate test --- internal/dinosql/config.go | 42 ---------- internal/mysql/README.md | 8 ++ internal/mysql/example/.gitignore | 1 + internal/mysql/example/queries.sql | 17 ++++ internal/mysql/example/sqlc.json | 13 +++ internal/mysql/functions.go | 12 +++ internal/mysql/parse.go | 75 ++++++++++-------- internal/mysql/parse_test.go | 113 +++++++++++++-------------- internal/mysql/schema.go | 17 +++- internal/mysql/test.sql | 16 ---- internal/mysql/test_data/queries.sql | 17 ++++ internal/mysql/test_data/sqlc.json | 13 +++ 12 files changed, 194 insertions(+), 150 deletions(-) create mode 100644 internal/mysql/README.md create mode 100644 internal/mysql/example/.gitignore create mode 100644 internal/mysql/example/queries.sql create mode 100644 internal/mysql/example/sqlc.json create mode 100644 internal/mysql/functions.go delete mode 100644 internal/mysql/test.sql create mode 100644 internal/mysql/test_data/queries.sql create mode 100644 internal/mysql/test_data/sqlc.json diff --git a/internal/dinosql/config.go b/internal/dinosql/config.go index aa272a78c4..e3a08ce1cf 100644 --- a/internal/dinosql/config.go +++ b/internal/dinosql/config.go @@ -1,14 +1,10 @@ package dinosql import ( - "bytes" "encoding/json" "errors" "fmt" "io" - "io/ioutil" - "os" - "path/filepath" "strings" "github.com/kyleconroy/sqlc/internal/pg" @@ -29,44 +25,6 @@ The only supported version is "1". const errMessageNoPackages = `No packages are configured` -// InitConfig initializes the global config objcet -func InitConfig() (*GenerateSettings, error) { - fmt.Println("Config init func ran") - blob, err := ioutil.ReadFile("sqlc.json") - if err != nil { - fmt.Fprintln(os.Stderr, "error parsing sqlc.json: file does not exist") - os.Exit(1) - } - - settings, err := ParseConfigFile(bytes.NewReader(blob)) - if err != nil { - switch err { - case ErrMissingVersion: - fmt.Fprintf(os.Stderr, errMessageNoVersion) - case ErrUnknownVersion: - fmt.Fprintf(os.Stderr, errMessageUnknownVersion) - case ErrNoPackages: - fmt.Fprintf(os.Stderr, errMessageNoPackages) - } - fmt.Fprintf(os.Stderr, "error parsing sqlc.json: %s\n", err) - os.Exit(1) - } - - for i, pkg := range settings.Packages { - name := pkg.Name - - if pkg.Path == "" { - fmt.Fprintf(os.Stderr, "package[%d]: path must be set\n", i) - continue - } - - if name == "" { - name = filepath.Base(pkg.Path) - } - } - return &settings, nil -} - type GenerateSettings struct { Version string `json:"version"` Packages []PackageSettings `json:"packages"` diff --git a/internal/mysql/README.md b/internal/mysql/README.md new file mode 100644 index 0000000000..80f839ee03 --- /dev/null +++ b/internal/mysql/README.md @@ -0,0 +1,8 @@ +# MySQL Support + +## Example Usage + +``` +$ cd ./example +$ unstable__mysql generate +``` diff --git a/internal/mysql/example/.gitignore b/internal/mysql/example/.gitignore new file mode 100644 index 0000000000..472fecd969 --- /dev/null +++ b/internal/mysql/example/.gitignore @@ -0,0 +1 @@ +*.go \ No newline at end of file diff --git a/internal/mysql/example/queries.sql b/internal/mysql/example/queries.sql new file mode 100644 index 0000000000..fecba7823e --- /dev/null +++ b/internal/mysql/example/queries.sql @@ -0,0 +1,17 @@ +CREATE TABLE teachers ( + id int NOT NULL, + school_id VARCHAR(255) NOT NULL, + school_lat FLOAT, + school_lng FLOAT, + department ENUM("English", "Math"), + PRIMARY KEY (ID) +); + +/* name: GetTeachersByID :one */ +SELECT * FROM teachers WHERE id = ? + +/* name: GetSomeTeachers :one */ +SELECT school_id, id FROM teachers WHERE school_lng > ? AND school_lat < ?; + +/* name: TeachersByID :one */ +SELECT id, school_lat FROM teachers WHERE id = ? LIMIT 10 \ No newline at end of file diff --git a/internal/mysql/example/sqlc.json b/internal/mysql/example/sqlc.json new file mode 100644 index 0000000000..b09d8467af --- /dev/null +++ b/internal/mysql/example/sqlc.json @@ -0,0 +1,13 @@ +{ + "version": "1", + "packages": [ + { + "name": "teachersDB", + "emit_json_tags": true, + "emit_prepared_queries": false, + "path": "./", + "queries": "./queries.sql", + "schema": "./queries.sql" + } + ] +} diff --git a/internal/mysql/functions.go b/internal/mysql/functions.go new file mode 100644 index 0000000000..70f21e57bd --- /dev/null +++ b/internal/mysql/functions.go @@ -0,0 +1,12 @@ +package mysql + +func functionReturnType(f string) string { + switch f { + case "avg", "count", "instr", "sum", "min", "max", "length", "char_length": + return "int" + case "concat", "left", "replace", "substring", "trim", "find_in_set", "format": + return "varchar" + default: + panic("unknown mysql function type") + } +} diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 357251fdab..7bc3d4be70 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -118,15 +118,14 @@ func parseSelect(tree *sqlparser.Select, query string, s *Schema, defaultTableNa } tree.SelectExprs = colNames } + parsedQuery := Query{ SQL: query, defaultTableName: defaultTableName, schemaLookup: s, } - err := sqlparser.Walk(parsedQuery.visit, tree) - if err != nil { - return nil, err - } + + parsedQuery.Columns = parseSelectAliasExpr(tree.SelectExprs, s, defaultTableName) whereParams, err := paramsInWhereExpr(tree.Where, s, defaultTableName, settings) if err != nil { @@ -273,40 +272,52 @@ func (q *Query) parseLeadingComment(comment string) error { } return nil } -func isExpr(exp sqlparser.Expr) {} -func (q *Query) visit(node sqlparser.SQLNode) (bool, error) { - switch v := node.(type) { - case *sqlparser.AliasedExpr: - err := sqlparser.Walk(q.visitColNames, v) - if err != nil { - return false, err - } - case *sqlparser.StarExpr: - cols, ok := q.schemaLookup.tables[q.defaultTableName] - if !ok { - return false, fmt.Errorf("Failed to expand * expression into columns from schema, tableName: %v", q.defaultTableName) - } - q.Columns = cols - return false, nil - default: - // fmt.Printf("Did not handle %T\n", v) - } - return true, nil -} +func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, defaultTable string) []*sqlparser.ColumnDefinition { + colDfns := []*sqlparser.ColumnDefinition{} + for _, col := range exprs { + switch expr := col.(type) { + case *sqlparser.AliasedExpr: + hasAlias := !expr.As.IsEmpty() -func (q *Query) visitColNames(node sqlparser.SQLNode) (bool, error) { - switch v := node.(type) { - case *sqlparser.ColName: - colTyp, err := q.schemaLookup.getColType(v, q.defaultTableName) - if err != nil { - return false, fmt.Errorf("Failed to get column type for [%v]: %v", v.Name.String(), err) + switch v := expr.Expr.(type) { + case *sqlparser.ColName: + res, err := s.getColType(v, defaultTable) + if err != nil { + panic(fmt.Sprintf("Column not found in schema: %v", err)) + } + if hasAlias { + res.Name = expr.As // applys the alias + } + colDfns = append(colDfns, res) + case *sqlparser.FuncExpr: + funcName := v.Name.Lowered() + funcType := functionReturnType(funcName) + + var returnVal sqlparser.ColIdent + if hasAlias { + returnVal = expr.As + } else { + returnVal = sqlparser.NewColIdent(funcName) + } + + colDfn := &sqlparser.ColumnDefinition{ + Name: returnVal, + Type: sqlparser.ColumnType{ + Type: funcType, + NotNull: true, + }, + } + colDfns = append(colDfns, colDfn) + } + default: + panic(fmt.Sprintf("Failed to handle select expr of type : %T\n", expr)) } - q.Columns = append(q.Columns, colTyp) } - return true, nil + return colDfns } +// GeneratePkg is the main entry to mysql generator package func GeneratePkg(pkgName string, querysPath string, settings dinosql.GenerateSettings) (map[string]string, error) { s := NewSchema() result, err := parseFile(querysPath, pkgName, s, settings) diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go index e9dca22324..d259b2b3d0 100644 --- a/internal/mysql/parse_test.go +++ b/internal/mysql/parse_test.go @@ -1,6 +1,8 @@ package mysql import ( + "encoding/json" + "io/ioutil" "reflect" "testing" @@ -13,41 +15,8 @@ func init() { initMockSchema() } -const query = ` -/* name: GetAllStudents :many */ -SELECT school_id, id FROM students WHERE id = :id + ? -` - -const create = ` - CREATE TABLE students ( - id int, - school_id VARCHAR(255), - school_lat VARCHAR(255), - PRIMARY KEY (ID) - );` - -const filename = "test.sql" - -func TestParseFile(t *testing.T) { - // s := NewSchema() - // _, err := parseFile(filename, s) - // keep(err) - tree, _ := sqlparser.Parse("SELECT id, first_name FROM users WHERE age < ?") - p := sqlparser.NewParsedQuery(tree) - // spew.Dump(p) - // for k, _ := - result := sqlparser.GetBindvars(tree) - newVars := make(map[string]string) - for k := range result { - newVars[k] = "?" - } - // spew.Dump(newVars) - keep(p) - // p.GenerateQuery(newVars) - // r, _ := p.MarshalJSON() - // spew.Dump(string(r)) - // spew.Dump(p.GenerateQuery()) -} +const filename = "test_data/queries.sql" +const configPath = "test_data/sqlc.json" var mockSettings = dinosql.GenerateSettings{ Version: "1", @@ -59,20 +28,23 @@ var mockSettings = dinosql.GenerateSettings{ Overrides: []dinosql.Override{}, } -func TestGenerate(t *testing.T) { - // t.Skip() - s := NewSchema() - result, _ := parseFile(filename, "db", s, mockSettings) - output, err := dinosql.Generate(result, mockSettings) +func TestParseConfig(t *testing.T) { + blob, err := ioutil.ReadFile(configPath) if err != nil { - t.Errorf("Failed to generate output: %v", err) + t.Fatal(err) + } + + var settings dinosql.GenerateSettings + if err := json.Unmarshal(blob, &settings); err != nil { + t.Fatal(err) + } +} + +func TestGeneratePkg(t *testing.T) { + _, err := GeneratePkg(mockSettings.Packages[0].Name, filename, mockSettings) + if err != nil { + t.Fatal(err) } - keep(output) - // for k, v := range output { - // fmt.Println(k) - // fmt.Println(v) - // fmt.Println("") - // } } func keep(interface{}) {} @@ -90,7 +62,6 @@ func initMockSchema() { Type: sqlparser.ColumnType{ Type: "varchar", NotNull: true, - // could add more here later if needed }, }, &sqlparser.ColumnDefinition{ @@ -98,7 +69,6 @@ func initMockSchema() { Type: sqlparser.ColumnType{ Type: "varchar", NotNull: false, - // could add more here later if needed }, }, &sqlparser.ColumnDefinition{ @@ -107,7 +77,6 @@ func initMockSchema() { Type: "int", NotNull: true, Autoincrement: true, - // could add more here later if needed }, }, &sqlparser.ColumnDefinition{ @@ -115,7 +84,6 @@ func initMockSchema() { Type: sqlparser.ColumnType{ Type: "int", NotNull: true, - // could add more here later if needed }, }, } @@ -126,7 +94,6 @@ func initMockSchema() { Type: "int", NotNull: true, Autoincrement: true, - // could add more here later if needed }, }, &sqlparser.ColumnDefinition{ @@ -135,7 +102,6 @@ func initMockSchema() { Type: "DECIMAL(13, 4)", NotNull: true, Autoincrement: true, - // could add more here later if needed }, }, &sqlparser.ColumnDefinition{ @@ -143,7 +109,6 @@ func initMockSchema() { Type: sqlparser.ColumnType{ Type: "int", NotNull: true, - // could add more here later if needed }, }, } @@ -171,6 +136,38 @@ func TestParseSelect(t *testing.T) { query2 := `/* name: GetAll :many */ SELECT * FROM users;` tests := []testCase{ + testCase{ + input: expected{ + query: `/* name: GetCount :one */ + SELECT id my_id, COUNT(id) id_count FROM users WHERE id > 4`, + schema: mockSchema, + }, + output: &Query{ + SQL: "select id as my_id, COUNT(id) as id_count from users where id > 4", + Columns: []*sqlparser.ColumnDefinition{ + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("my_id"), + Type: sqlparser.ColumnType{ + Type: "int", + NotNull: true, + Autoincrement: true, + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("id_count"), + Type: sqlparser.ColumnType{ + Type: "int", + NotNull: true, + }, + }, + }, + Params: []*Param{}, + Name: "GetCount", + Cmd: ":one", + defaultTableName: "users", + schemaLookup: mockSchema, + }, + }, testCase{ input: expected{ query: `/* name: GetNameByID :one */ @@ -212,16 +209,16 @@ func TestParseSelect(t *testing.T) { for _, testCase := range tests { q, err := parseQueryString(testCase.input.query, testCase.input.schema, mockSettings) if err != nil { - t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", query, spew.Sdump(testCase.input.schema)) + t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", testCase.input.query, spew.Sdump(testCase.input.schema)) } err = q.parseNameAndCmd() if err != nil { - t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", query, spew.Sdump(testCase.input.schema)) + t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", testCase.input.query, spew.Sdump(testCase.input.schema)) } if !reflect.DeepEqual(testCase.output, q) { t.Errorf("Parsing query returned differently than expected.") - t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) + // t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) } } } @@ -356,7 +353,7 @@ func TestParseInsert(t *testing.T) { err = q.parseNameAndCmd() if err != nil { - t.Errorf("Parsing failed with query index: %d: [%v]\n", ix, query) + t.Errorf("Parsing failed with query index: %d: [%v]\n", ix, testCase.input.query) } if !reflect.DeepEqual(testCase.output, q) { t.Errorf("Parsing query returned differently than expected.") diff --git a/internal/mysql/schema.go b/internal/mysql/schema.go index 111b9e2b7b..2306b0652e 100644 --- a/internal/mysql/schema.go +++ b/internal/mysql/schema.go @@ -19,11 +19,24 @@ type Schema struct { tables map[string]([]*sqlparser.ColumnDefinition) } +// returns a deep copy of the column definition for using as a query return type or param type func (s *Schema) getColType(col *sqlparser.ColName, defaultTableName string) (*sqlparser.ColumnDefinition, error) { if !col.Qualifier.IsEmpty() { - return s.schemaLookup(col.Qualifier.Name.String(), col.Name.String()) + colDfn, err := s.schemaLookup(col.Qualifier.Name.String(), col.Name.String()) + if err != nil { + return nil, err + } + return &sqlparser.ColumnDefinition{ + Name: colDfn.Name, Type: colDfn.Type, + }, nil + } + colDfn, err := s.schemaLookup(defaultTableName, col.Name.String()) + if err != nil { + return nil, err } - return s.schemaLookup(defaultTableName, col.Name.String()) + return &sqlparser.ColumnDefinition{ + Name: colDfn.Name, Type: colDfn.Type, + }, nil } // Add add a MySQL table definition to the schema map diff --git a/internal/mysql/test.sql b/internal/mysql/test.sql deleted file mode 100644 index 3ec9c0898b..0000000000 --- a/internal/mysql/test.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE TABLE students ( - id int NOT NULL, - school_id VARCHAR(255) NOT NULL, - school_lat FLOAT, - department ENUM("English", "Math"), - PRIMARY KEY (ID) -); - -/* name: GetAllStudents :many */ -SELECT school_id, id FROM students WHERE id = ? AND school_id = ? - -/* name: GetSomeStudents :one */ -SELECT school_id, id FROM students WHERE school_id = ? - -/* name: StudentByID :one */ -SELECT id, school_lat FROM students WHERE id = ? LIMIT 10 \ No newline at end of file diff --git a/internal/mysql/test_data/queries.sql b/internal/mysql/test_data/queries.sql new file mode 100644 index 0000000000..fecba7823e --- /dev/null +++ b/internal/mysql/test_data/queries.sql @@ -0,0 +1,17 @@ +CREATE TABLE teachers ( + id int NOT NULL, + school_id VARCHAR(255) NOT NULL, + school_lat FLOAT, + school_lng FLOAT, + department ENUM("English", "Math"), + PRIMARY KEY (ID) +); + +/* name: GetTeachersByID :one */ +SELECT * FROM teachers WHERE id = ? + +/* name: GetSomeTeachers :one */ +SELECT school_id, id FROM teachers WHERE school_lng > ? AND school_lat < ?; + +/* name: TeachersByID :one */ +SELECT id, school_lat FROM teachers WHERE id = ? LIMIT 10 \ No newline at end of file diff --git a/internal/mysql/test_data/sqlc.json b/internal/mysql/test_data/sqlc.json new file mode 100644 index 0000000000..b09d8467af --- /dev/null +++ b/internal/mysql/test_data/sqlc.json @@ -0,0 +1,13 @@ +{ + "version": "1", + "packages": [ + { + "name": "teachersDB", + "emit_json_tags": true, + "emit_prepared_queries": false, + "path": "./", + "queries": "./queries.sql", + "schema": "./queries.sql" + } + ] +} From cdd5f1bd7add177c47062512464ab4fce26f812e Mon Sep 17 00:00:00 2001 From: Charles Moog Date: Tue, 31 Dec 2019 13:36:46 +1300 Subject: [PATCH 03/24] Update README.md --- internal/mysql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mysql/README.md b/internal/mysql/README.md index 80f839ee03..d739a8f572 100644 --- a/internal/mysql/README.md +++ b/internal/mysql/README.md @@ -4,5 +4,5 @@ ``` $ cd ./example -$ unstable__mysql generate +$ sqlc unstable__mysql generate ``` From 778e9b6ca0488a466a4c03c0ce6222686602000d Mon Sep 17 00:00:00 2001 From: cmoog Date: Tue, 31 Dec 2019 17:14:42 +1300 Subject: [PATCH 04/24] minimizes diff and fixes bug in Comparable EqualTo method --- internal/cmd/cmd.go | 2 +- internal/dinosql/config.go | 4 ++-- internal/dinosql/config_test.go | 2 +- internal/dinosql/gen.go | 32 ++++++++++++++++---------------- internal/dinosql/gen_test.go | 32 ++++++++++++++++++++++---------- internal/dinosql/parser.go | 19 +++++++++++-------- internal/dinosql/parser_test.go | 17 ++++++++--------- 7 files changed, 61 insertions(+), 47 deletions(-) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index a5527c59b3..60dfbeee0d 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -110,7 +110,7 @@ var genCmd = &cobra.Command{ os.Exit(1) } - settings, err := dinosql.ParseConfigFile(bytes.NewReader(blob)) + settings, err := dinosql.ParseConfig(bytes.NewReader(blob)) if err != nil { switch err { case dinosql.ErrMissingVersion: diff --git a/internal/dinosql/config.go b/internal/dinosql/config.go index e3a08ce1cf..6b692665ab 100644 --- a/internal/dinosql/config.go +++ b/internal/dinosql/config.go @@ -113,7 +113,7 @@ var ErrMissingVersion = errors.New("no version number") var ErrUnknownVersion = errors.New("invalid version number") var ErrNoPackages = errors.New("no packages") -func ParseConfigFile(rd io.Reader) (GenerateSettings, error) { +func ParseConfig(rd io.Reader) (GenerateSettings, error) { dec := json.NewDecoder(rd) dec.DisallowUnknownFields() var config GenerateSettings @@ -151,7 +151,7 @@ func (s *GenerateSettings) PopulatePkgMap() error { for _, c := range s.Packages { if c.Name == "" { - return errors.New("Package name must be specified in sqlc.json") + panic("Package name must be specified in sqlc.json") } packageMap[c.Name] = c } diff --git a/internal/dinosql/config_test.go b/internal/dinosql/config_test.go index 3b3c03fb46..ef635dae3a 100644 --- a/internal/dinosql/config_test.go +++ b/internal/dinosql/config_test.go @@ -51,7 +51,7 @@ func TestBadConfigs(t *testing.T) { } { tt := test t.Run(tt.name, func(t *testing.T) { - _, err := ParseConfigFile(strings.NewReader(tt.json)) + _, err := ParseConfig(strings.NewReader(tt.json)) if err == nil { t.Fatalf("expected err; got nil") } diff --git a/internal/dinosql/gen.go b/internal/dinosql/gen.go index 0602457126..ae129bf981 100644 --- a/internal/dinosql/gen.go +++ b/internal/dinosql/gen.go @@ -51,11 +51,6 @@ func (gf GoField) Tag() string { return strings.Join(tags, ",") } -// TODO: consider making this deep equality from stdlib? -type Comparable interface { - EqualTo(b interface{}) bool -} - type GoStruct struct { Table Comparable Name string @@ -63,7 +58,6 @@ type GoStruct struct { Comment string } -// TODO: Terrible name type GoQueryValue struct { Emit bool Name string @@ -71,13 +65,18 @@ type GoQueryValue struct { Typ string } +// TODO: consider making this deep equality from stdlib? +type Comparable interface { + EqualTo(b interface{}) bool +} + type FQNAlias core.FQN // Check whether tables are equal func (a *FQNAlias) EqualTo(other interface{}) bool { - b, ok := other.(*FQNAlias) + b, ok := other.(*core.FQN) if !ok { - return false + panic("Unknown ") } if a == nil && b == nil { return true @@ -188,11 +187,11 @@ type Generateable interface { Enums(settings GenerateSettings) []GoEnum } -func UsesType(r Generateable, Typ string, settings GenerateSettings) bool { +func UsesType(r Generateable, typ string, settings GenerateSettings) bool { for _, strct := range r.Structs(settings) { for _, f := range strct.Fields { fType := strings.TrimPrefix(f.Type, "[]") - if strings.HasPrefix(fType, Typ) { + if strings.HasPrefix(fType, typ) { return true } } @@ -519,11 +518,11 @@ func (r Result) goType(col core.Column, settings GenerateSettings) string { return oride.goTypeName } } - Typ := r.goInnerType(col, settings) + typ := r.goInnerType(col, settings) if col.IsArray { - return "[]" + Typ + return "[]" + typ } - return Typ + return typ } func (r Result) goInnerType(col core.Column, settings GenerateSettings) string { @@ -776,9 +775,10 @@ func (r Result) GoQueries(settings GenerateSettings) []GoQuery { c := query.Columns[i] sameName := f.Name == StructName(columnName(c, i), settings) sameType := f.Type == r.goType(c, settings) - // TODO: consider making this deep equality from stdlib? - sameFQN := s.Table.EqualTo(&c.Table) - if !sameName || !sameType || !sameFQN { + + // TODO: clean this up!! + sameTable := s.Table.EqualTo(&c.Table) + if !sameName || !sameType || !sameTable { same = false } } diff --git a/internal/dinosql/gen_test.go b/internal/dinosql/gen_test.go index a1e4869cfc..b61ed74993 100644 --- a/internal/dinosql/gen_test.go +++ b/internal/dinosql/gen_test.go @@ -1,6 +1,7 @@ package dinosql import ( + "path/filepath" "testing" "github.com/google/go-cmp/cmp" @@ -8,7 +9,6 @@ import ( ) func TestColumnsToStruct(t *testing.T) { - pkgName := "db" cols := []pg.Column{ { Name: "other", @@ -53,10 +53,6 @@ func TestColumnsToStruct(t *testing.T) { cols[i].Table = pg.FQN{Schema: "public", Rel: "foo"} } - r := Result{ - packageName: pkgName, - } - // set up column-based override test o := Override{ GoType: "example.com/pkg.CustomType", @@ -70,9 +66,15 @@ func TestColumnsToStruct(t *testing.T) { Column: "foo.languages", } oa.Parse() - // pkgConfig := PackageSettings{ - // Overrides: []Override{o, oa}, - // } + + pkgName := "test_override" + + r := Result{ + packageName: pkgName, + } + mockSettings.PackageMap[pkgName] = PackageSettings{ + Overrides: []Override{o, oa}, + } actual := r.columnsToStruct("Foo", cols, mockSettings) expected := &GoStruct{ @@ -101,6 +103,16 @@ func init() { PackageSettings{ Name: "db", }, + PackageSettings{ + Name: "prepared", + Queries: filepath.Join("testdata", "ondeck", "query"), + EmitPreparedQueries: true, + }, + PackageSettings{ + Name: "ondeck", + Queries: filepath.Join("testdata", "ondeck", "query"), + EmitJSONTags: true, + }, }, Overrides: []Override{}, } @@ -108,7 +120,7 @@ func init() { } func TestInnerType(t *testing.T) { - r := Result{} + r := Result{packageName: "db"} types := map[string]string{ "integer": "int32", "int": "int32", @@ -135,7 +147,7 @@ func TestInnerType(t *testing.T) { } func TestNullInnerType(t *testing.T) { - r := Result{} + r := Result{packageName: "db"} types := map[string]string{ "integer": "sql.NullInt32", "int": "sql.NullInt32", diff --git a/internal/dinosql/parser.go b/internal/dinosql/parser.go index 08a9d59304..fc3098f0c5 100644 --- a/internal/dinosql/parser.go +++ b/internal/dinosql/parser.go @@ -177,26 +177,29 @@ type Result struct { } func (r Result) PkgName() string { + if r.packageName == "" { + panic("Package name is empty") + } return r.packageName } -func ParseQueries(c core.Catalog, pkgConfig PackageSettings) (*Result, error) { - f, err := os.Stat(pkgConfig.Queries) +func ParseQueries(c core.Catalog, pkg PackageSettings) (*Result, error) { + f, err := os.Stat(pkg.Queries) if err != nil { - return nil, fmt.Errorf("path %s does not exist", pkgConfig.Queries) + return nil, fmt.Errorf("path %s does not exist", pkg.Queries) } var files []string if f.IsDir() { - listing, err := ioutil.ReadDir(pkgConfig.Queries) + listing, err := ioutil.ReadDir(pkg.Queries) if err != nil { return nil, err } for _, f := range listing { - files = append(files, filepath.Join(pkgConfig.Queries, f.Name())) + files = append(files, filepath.Join(pkg.Queries, f.Name())) } } else { - files = append(files, pkgConfig.Queries) + files = append(files, pkg.Queries) } merr := NewParserErr() @@ -246,12 +249,12 @@ func ParseQueries(c core.Catalog, pkgConfig PackageSettings) (*Result, error) { return nil, merr } if len(q) == 0 { - return nil, fmt.Errorf("path %s contains no queries", pkgConfig.Queries) + return nil, fmt.Errorf("path %s contains no queries", pkg.Queries) } return &Result{ Catalog: c, Queries: q, - packageName: pkgConfig.Name, + packageName: pkg.Name, }, nil } diff --git a/internal/dinosql/parser_test.go b/internal/dinosql/parser_test.go index 00d45db8f7..757165cf52 100644 --- a/internal/dinosql/parser_test.go +++ b/internal/dinosql/parser_test.go @@ -164,16 +164,11 @@ func TestParseSchema(t *testing.T) { t.Fatal(err) } - q, err := ParseQueries(c, PackageSettings{ - Name: "db", - Queries: filepath.Join("testdata", "ondeck", "query"), - EmitJSONTags: true, - }) - if err != nil { - t.Fatal(err) - } - t.Run("default", func(t *testing.T) { + q, err := ParseQueries(c, mockSettings.PackageMap["ondeck"]) + if err != nil { + t.Fatal(err) + } output, err := Generate(q, mockSettings) if err != nil { t.Fatal(err) @@ -183,6 +178,10 @@ func TestParseSchema(t *testing.T) { }) t.Run("prepared", func(t *testing.T) { + q, err := ParseQueries(c, mockSettings.PackageMap["prepared"]) + if err != nil { + t.Fatal(err) + } output, err := Generate(q, mockSettings) if err != nil { t.Fatal(err) From 947d824d4a101fe29ee40177cb972a8d598a580c Mon Sep 17 00:00:00 2001 From: cmoog Date: Tue, 31 Dec 2019 22:09:56 +1300 Subject: [PATCH 05/24] properly replaces "?" in query string --- internal/mysql/param.go | 12 ++++++++++++ internal/mysql/parse.go | 26 +++++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/internal/mysql/param.go b/internal/mysql/param.go index f950a2c118..847a9a63e0 100644 --- a/internal/mysql/param.go +++ b/internal/mysql/param.go @@ -2,6 +2,7 @@ package mysql import ( "fmt" + "regexp" "strings" "github.com/kyleconroy/sqlc/internal/dinosql" @@ -126,3 +127,14 @@ func paramName(col sqlparser.ColIdent, originalName string) string { num := originalName[2] return fmt.Sprintf("param%v", num) } + +func replaceParamStrs(query string, params []*Param) (string, error) { + for ix := range params { + re, err := regexp.Compile(fmt.Sprintf("(:v%d)", ix+1)) + if err != nil { + return "", err + } + query = re.ReplaceAllString(query, "?") + } + return query, nil +} diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 7bc3d4be70..a57c184b9c 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -15,11 +15,11 @@ import ( type Query struct { SQL string Columns []*sqlparser.ColumnDefinition - Params []*Param - Name string - Cmd string // TODO: Pick a better name. One of: one, many, exec, execrows - defaultTableName string // for columns that are not qualified - schemaLookup *Schema + Params []*Param // "?" params in the query string + Name string // the Go function name + Cmd string // TODO: Pick a better name. One of: one, many, exec, execrows + defaultTableName string // for columns that are not qualified + schemaLookup *Schema // for validating and conversion to Go types } func parseFile(filepath string, inPkg string, s *Schema, settings dinosql.GenerateSettings) (*Result, error) { @@ -60,7 +60,7 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings if err != nil { return nil, err } - + var parsedQuery *Query switch tree := tree.(type) { case *sqlparser.Select: defaultTableName := getDefaultTable(&tree.From) @@ -68,27 +68,31 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings if err != nil { return nil, fmt.Errorf("Failed to parse SELECT query: %v", err) } - return res, nil + parsedQuery = res case *sqlparser.Insert: insert, err := parseInsert(tree, query, s, settings) if err != nil { return nil, fmt.Errorf("Failed to parse INSERT query: %v", err) } - return insert, nil + parsedQuery = insert case *sqlparser.Update: update, err := parseUpdate(tree, query, s, settings) if err != nil { return nil, fmt.Errorf("Failed to parse UPDATE query: %v", err) } - return update, nil + parsedQuery = update case *sqlparser.DDL: s.Add(tree) return nil, nil default: panic("Unsupported SQL statement type") - // return &Query{}, nil } - return nil, fmt.Errorf("Failed to parse query statement: %v", query) + paramsReplacedQuery, err := replaceParamStrs(parsedQuery.SQL, parsedQuery.Params) + if err != nil { + return nil, fmt.Errorf("Failed to replace param variables in query string: %v", err) + } + parsedQuery.SQL = paramsReplacedQuery + return parsedQuery, nil } func (q *Query) parseNameAndCmd() error { From add6ea7b51f5850281b5e6bba664cb3fbe4c8f02 Mon Sep 17 00:00:00 2001 From: cmoog Date: Tue, 31 Dec 2019 22:13:10 +1300 Subject: [PATCH 06/24] patch test --- internal/mysql/parse_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go index d259b2b3d0..cdd172e997 100644 --- a/internal/mysql/parse_test.go +++ b/internal/mysql/parse_test.go @@ -175,7 +175,7 @@ func TestParseSelect(t *testing.T) { schema: mockSchema, }, output: &Query{ - SQL: `select first_name, last_name from users where id = :v1`, + SQL: `select first_name, last_name from users where id = ?`, Columns: filterCols(mockSchema.tables["users"], map[string]struct{}{"first_name": struct{}{}, "last_name": struct{}{}}), Params: []*Param{ &Param{ From 12491c85465c21e898230f7a15129ebc96335697 Mon Sep 17 00:00:00 2001 From: cmoog Date: Wed, 1 Jan 2020 01:13:28 +1300 Subject: [PATCH 07/24] adds additional example queries --- internal/mysql/example/queries.sql | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/mysql/example/queries.sql b/internal/mysql/example/queries.sql index fecba7823e..266c604f03 100644 --- a/internal/mysql/example/queries.sql +++ b/internal/mysql/example/queries.sql @@ -1,12 +1,22 @@ CREATE TABLE teachers ( id int NOT NULL, + first_name varchar(255), + last_name varchar(255), school_id VARCHAR(255) NOT NULL, school_lat FLOAT, school_lng FLOAT, department ENUM("English", "Math"), - PRIMARY KEY (ID) + PRIMARY KEY (id) ); +CREATE TABLE students ( + id int NOT NULL, + student_id varchar(10), + first_name varchar(255), + last_name varchar(255), + PRIMARY KEY (id) +) + /* name: GetTeachersByID :one */ SELECT * FROM teachers WHERE id = ? @@ -14,4 +24,9 @@ SELECT * FROM teachers WHERE id = ? SELECT school_id, id FROM teachers WHERE school_lng > ? AND school_lat < ?; /* name: TeachersByID :one */ -SELECT id, school_lat FROM teachers WHERE id = ? LIMIT 10 \ No newline at end of file +SELECT id, school_lat FROM teachers WHERE id = ? LIMIT 10 + +/* name: GetStudents :many */ +SELECT students.first_name, students.last_name, teachers.first_name teacherFirstName + FROM students + LEFT JOIN teachers on teachers.school_id = students.school_id \ No newline at end of file From 903f87cb978f816cc30a175e005e83490881f499 Mon Sep 17 00:00:00 2001 From: cmoog Date: Wed, 1 Jan 2020 22:38:45 +1300 Subject: [PATCH 08/24] adds support for table alias' --- internal/mysql/README.md | 5 ++ internal/mysql/functions.go | 5 +- internal/mysql/param.go | 20 ++++---- internal/mysql/param_test.go | 13 ++--- internal/mysql/parse.go | 98 ++++++++++++++++++++++++------------ internal/mysql/parse_test.go | 61 +++++++++++++++++++--- internal/mysql/schema.go | 17 +++++-- 7 files changed, 159 insertions(+), 60 deletions(-) diff --git a/internal/mysql/README.md b/internal/mysql/README.md index d739a8f572..f831ec269b 100644 --- a/internal/mysql/README.md +++ b/internal/mysql/README.md @@ -6,3 +6,8 @@ $ cd ./example $ sqlc unstable__mysql generate ``` + +## Missing Features + +- support for the `queries` field being specified as a directory of files +- missing many MySQL types and function returns types diff --git a/internal/mysql/functions.go b/internal/mysql/functions.go index 70f21e57bd..9616a2b66f 100644 --- a/internal/mysql/functions.go +++ b/internal/mysql/functions.go @@ -1,5 +1,8 @@ package mysql +import "fmt" + +// converts MySQL function name to MySQL return type func functionReturnType(f string) string { switch f { case "avg", "count", "instr", "sum", "min", "max", "length", "char_length": @@ -7,6 +10,6 @@ func functionReturnType(f string) string { case "concat", "left", "replace", "substring", "trim", "find_in_set", "format": return "varchar" default: - panic("unknown mysql function type") + panic(fmt.Sprintf("unknown mysql function type [%v]", f)) } } diff --git a/internal/mysql/param.go b/internal/mysql/param.go index 847a9a63e0..83a4ac5d33 100644 --- a/internal/mysql/param.go +++ b/internal/mysql/param.go @@ -15,7 +15,7 @@ type Param struct { typ string } -func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, settings dinosql.GenerateSettings) ([]*Param, error) { +func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, tableAliasMap map[string]string, settings dinosql.GenerateSettings) ([]*Param, error) { params := []*Param{} if limit == nil { return params, nil @@ -40,16 +40,16 @@ func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, settings dinosql.Gener return params, nil } -func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, defaultTable string, settings dinosql.GenerateSettings) ([]*Param, error) { +func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, tableAliasMap map[string]string, defaultTable string, settings dinosql.GenerateSettings) ([]*Param, error) { params := []*Param{} switch v := e.(type) { case *sqlparser.Where: if v == nil { return params, nil } - return paramsInWhereExpr(v.Expr, s, defaultTable, settings) + return paramsInWhereExpr(v.Expr, s, tableAliasMap, defaultTable, settings) case *sqlparser.ComparisonExpr: - p, found, err := paramInComparison(v, s, defaultTable, settings) + p, found, err := paramInComparison(v, s, tableAliasMap, defaultTable, settings) if err != nil { return nil, err } @@ -57,23 +57,23 @@ func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, defaultTable string, sett params = append(params, p) } case *sqlparser.AndExpr: - left, err := paramsInWhereExpr(v.Left, s, defaultTable, settings) + left, err := paramsInWhereExpr(v.Left, s, tableAliasMap, defaultTable, settings) if err != nil { return nil, err } params = append(params, left...) - right, err := paramsInWhereExpr(v.Right, s, defaultTable, settings) + right, err := paramsInWhereExpr(v.Right, s, tableAliasMap, defaultTable, settings) if err != nil { return nil, err } params = append(params, right...) case *sqlparser.OrExpr: - left, err := paramsInWhereExpr(v.Left, s, defaultTable, settings) + left, err := paramsInWhereExpr(v.Left, s, tableAliasMap, defaultTable, settings) if err != nil { return nil, err } params = append(params, left...) - right, err := paramsInWhereExpr(v.Right, s, defaultTable, settings) + right, err := paramsInWhereExpr(v.Right, s, tableAliasMap, defaultTable, settings) if err != nil { return nil, err } @@ -85,13 +85,13 @@ func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, defaultTable string, sett return params, nil } -func paramInComparison(cond *sqlparser.ComparisonExpr, s *Schema, defaultTable string, settings dinosql.GenerateSettings) (*Param, bool, error) { +func paramInComparison(cond *sqlparser.ComparisonExpr, s *Schema, tableAliasMap map[string]string, defaultTable string, settings dinosql.GenerateSettings) (*Param, bool, error) { p := &Param{} var colIdent sqlparser.ColIdent walker := func(node sqlparser.SQLNode) (bool, error) { switch v := node.(type) { case *sqlparser.ColName: - colDfn, err := s.getColType(v, defaultTable) + colDfn, err := s.getColType(v, tableAliasMap, defaultTable) if err != nil { return false, err } diff --git a/internal/mysql/param_test.go b/internal/mysql/param_test.go index feb66fb866..b323b1157c 100644 --- a/internal/mysql/param_test.go +++ b/internal/mysql/param_test.go @@ -83,11 +83,16 @@ func TestSelectParamSearcher(t *testing.T) { } selectStm, ok := tree.(*sqlparser.Select) - limitParams, err := paramsInLimitExpr(selectStm.Limit, mockSchema, mockSettings) + tableAliasMap, err := parseFrom(selectStm.From) + if err != nil { + t.Errorf("Failed to parse table name alias's: %v", err) + } + + limitParams, err := paramsInLimitExpr(selectStm.Limit, mockSchema, tableAliasMap, mockSettings) if err != nil { t.Errorf("Failed to parse limit expression params: %v", err) } - whereParams, err := paramsInWhereExpr(selectStm.Where, mockSchema, "users", mockSettings) + whereParams, err := paramsInWhereExpr(selectStm.Where, mockSchema, tableAliasMap, "users", mockSettings) if err != nil { t.Errorf("Failed to parse where expression params: %v", err) } @@ -97,10 +102,6 @@ func TestSelectParamSearcher(t *testing.T) { t.Errorf("Test case is not SELECT statement as expected") } - // TODO: get this out of the unit test and/or deprecate defaultTable - defaultTable := getDefaultTable(&selectStm.From) - keep(defaultTable) - if !reflect.DeepEqual(params, tCase.output) { t.Errorf("Param searcher returned unexpected result\nResult: %v\nExpected: %v", spew.Sdump(params), spew.Sdump(tCase.output)) diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index a57c184b9c..a7d96928a4 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -7,19 +7,20 @@ import ( "os" "strings" + "github.com/davecgh/go-spew/spew" "github.com/kyleconroy/sqlc/internal/dinosql" "vitess.io/vitess/go/vt/sqlparser" ) // Query holds the data for walking and validating mysql querys type Query struct { - SQL string - Columns []*sqlparser.ColumnDefinition - Params []*Param // "?" params in the query string - Name string // the Go function name - Cmd string // TODO: Pick a better name. One of: one, many, exec, execrows - defaultTableName string // for columns that are not qualified - schemaLookup *Schema // for validating and conversion to Go types + SQL string // the string representation of the parsed query + Columns []*sqlparser.ColumnDefinition // definitions for all columns returned by this query + Params []*Param // "?" params in the query string + Name string // the Go function name + Cmd string // TODO: Pick a better name. One of: one, many, exec, execrows + defaultTableName string // for columns that are not qualified + schemaLookup *Schema // for validation and conversion to Go types } func parseFile(filepath string, inPkg string, s *Schema, settings dinosql.GenerateSettings) (*Result, error) { @@ -63,8 +64,7 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings var parsedQuery *Query switch tree := tree.(type) { case *sqlparser.Select: - defaultTableName := getDefaultTable(&tree.From) - res, err := parseSelect(tree, query, s, defaultTableName, settings) + res, err := parseSelect(tree, query, s, settings) if err != nil { return nil, fmt.Errorf("Failed to parse SELECT query: %v", err) } @@ -107,7 +107,13 @@ func (q *Query) parseNameAndCmd() error { return nil } -func parseSelect(tree *sqlparser.Select, query string, s *Schema, defaultTableName string, settings dinosql.GenerateSettings) (*Query, error) { +func parseSelect(tree *sqlparser.Select, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { + tableAliasMap, err := parseFrom(tree.From) + if err != nil { + return nil, fmt.Errorf("Failed to parse table name alias's: %v", err) + } + defaultTableName := getDefaultTable(tableAliasMap) + // handle * expressions first by expanding all columns of the default table _, ok := tree.SelectExprs[0].(*sqlparser.StarExpr) if ok { @@ -128,15 +134,18 @@ func parseSelect(tree *sqlparser.Select, query string, s *Schema, defaultTableNa defaultTableName: defaultTableName, schemaLookup: s, } + cols, err := parseSelectAliasExpr(tree.SelectExprs, s, tableAliasMap, defaultTableName) + if err != nil { + return nil, err + } + parsedQuery.Columns = cols - parsedQuery.Columns = parseSelectAliasExpr(tree.SelectExprs, s, defaultTableName) - - whereParams, err := paramsInWhereExpr(tree.Where, s, defaultTableName, settings) + whereParams, err := paramsInWhereExpr(tree.Where, s, tableAliasMap, defaultTableName, settings) if err != nil { return nil, err } - limitParams, err := paramsInLimitExpr(tree.Limit, s, settings) + limitParams, err := paramsInLimitExpr(tree.Limit, s, tableAliasMap, settings) if err != nil { return nil, err } @@ -151,25 +160,48 @@ func parseSelect(tree *sqlparser.Select, query string, s *Schema, defaultTableNa return &parsedQuery, nil } -func getDefaultTable(node *sqlparser.TableExprs) string { - // TODO: improve this - var tableName string - visit := func(node sqlparser.SQLNode) (bool, error) { - switch v := node.(type) { - case sqlparser.TableName: - if name := v.Name.String(); name != "" { - tableName = name - return false, nil +func parseFrom(from sqlparser.TableExprs) (map[string]string, error) { + tables := make(map[string]string) + for _, expr := range from { + switch v := expr.(type) { + case *sqlparser.AliasedTableExpr: + name, ok := v.Expr.(sqlparser.TableName) + if !ok { + return nil, fmt.Errorf("Failed to parse AliasedTableExpr name: %v", spew.Sdump(v)) } + if v.As.String() != "" { + tables[v.As.String()] = name.Name.String() + } else { + tables[name.Name.String()] = name.Name.String() + } + case *sqlparser.JoinTableExpr: + return parseFrom([]sqlparser.TableExpr{v.LeftExpr, v.RightExpr}) + default: + return nil, fmt.Errorf("Failed to parse table expr: %v", spew.Sdump(v)) } - return true, nil } - sqlparser.Walk(visit, node) - return tableName + return tables, nil +} + +func getDefaultTable(tableAliasMap map[string]string) string { + if len(tableAliasMap) != 1 { + return "" + } + for _, val := range tableAliasMap { + return val + } + panic("Should never be reached.") } func parseUpdate(node *sqlparser.Update, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { - defaultTable := getDefaultTable(&node.TableExprs) + tableAliasMap, err := parseFrom(node.TableExprs) + if err != nil { + return nil, fmt.Errorf("Failed to parse table name alias's: %v", err) + } + defaultTable := getDefaultTable(tableAliasMap) + if err != nil { + return nil, err + } params := []*Param{} for _, updateExpr := range node.Exprs { @@ -178,7 +210,7 @@ func parseUpdate(node *sqlparser.Update, query string, s *Schema, settings dinos if !isParam { continue } - colDfn, err := s.getColType(col, defaultTable) + colDfn, err := s.getColType(col, tableAliasMap, defaultTable) if err != nil { return nil, fmt.Errorf("Failed to determine type of a parameter's column: %v", err) } @@ -191,7 +223,7 @@ func parseUpdate(node *sqlparser.Update, query string, s *Schema, settings dinos params = append(params, ¶m) } - whereParams, err := paramsInWhereExpr(node.Where.Expr, s, defaultTable, settings) + whereParams, err := paramsInWhereExpr(node.Where.Expr, s, tableAliasMap, defaultTable, settings) if err != nil { return nil, fmt.Errorf("Failed to parse params from WHERE expression: %v", err) } @@ -277,7 +309,7 @@ func (q *Query) parseLeadingComment(comment string) error { return nil } -func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, defaultTable string) []*sqlparser.ColumnDefinition { +func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, tableAliasMap map[string]string, defaultTable string) ([]*sqlparser.ColumnDefinition, error) { colDfns := []*sqlparser.ColumnDefinition{} for _, col := range exprs { switch expr := col.(type) { @@ -286,7 +318,7 @@ func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, defaultTable s switch v := expr.Expr.(type) { case *sqlparser.ColName: - res, err := s.getColType(v, defaultTable) + res, err := s.getColType(v, tableAliasMap, defaultTable) if err != nil { panic(fmt.Sprintf("Column not found in schema: %v", err)) } @@ -315,10 +347,10 @@ func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, defaultTable s colDfns = append(colDfns, colDfn) } default: - panic(fmt.Sprintf("Failed to handle select expr of type : %T\n", expr)) + return nil, fmt.Errorf("Failed to handle select expr of type : %T", expr) } } - return colDfns + return colDfns, nil } // GeneratePkg is the main entry to mysql generator package diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go index cdd172e997..e55585fd8e 100644 --- a/internal/mysql/parse_test.go +++ b/internal/mysql/parse_test.go @@ -204,21 +204,70 @@ func TestParseSelect(t *testing.T) { schemaLookup: mockSchema, }, }, + testCase{ + input: expected{ + query: `/* name: GetAllUsersOrders :many */ + SELECT u.id, u.first_name, o.price, o.id order_id + FROM orders o LEFT JOIN users u ON u.id = o.user_id`, + schema: mockSchema, + }, + output: &Query{ + SQL: "select u.id, u.first_name, o.price, o.id as order_id from orders as o left join users as u on u.id = o.user_id", + Columns: []*sqlparser.ColumnDefinition{ + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("id"), + Type: sqlparser.ColumnType{ + Type: "int", + Autoincrement: true, + NotNull: true, + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("first_name"), + Type: sqlparser.ColumnType{ + Type: "varchar", + NotNull: true, + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("price"), + Type: sqlparser.ColumnType{ + Type: "DECIMAL(13, 4)", + Autoincrement: true, + NotNull: true, + }, + }, + &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent("order_id"), + Type: sqlparser.ColumnType{ + Type: "int", + Autoincrement: true, + NotNull: true, + }, + }, + }, + Params: []*Param{}, + Name: "GetAllUsersOrders", + Cmd: ":many", + defaultTableName: "", // TODO: verify that this is desired behaviour + schemaLookup: mockSchema, + }, + }, } - for _, testCase := range tests { + for ix, testCase := range tests { q, err := parseQueryString(testCase.input.query, testCase.input.schema, mockSettings) if err != nil { - t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", testCase.input.query, spew.Sdump(testCase.input.schema)) + t.Errorf("Parsing failed with query: [%v]\n%v", testCase.input.query, err) } err = q.parseNameAndCmd() if err != nil { - t.Errorf("Parsing failed withe query: [%v]\n:schema: %v", testCase.input.query, spew.Sdump(testCase.input.schema)) + t.Errorf("Parsing failed with query: [%v]\n%v", testCase.input.query, err) } if !reflect.DeepEqual(testCase.output, q) { - t.Errorf("Parsing query returned differently than expected.") - // t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) + t.Errorf("Parsing query returned differently than expected. Index: %v", ix) + t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) } } } @@ -266,7 +315,7 @@ func TestSchemaLookup(t *testing.T) { } } -func TestParseInsert(t *testing.T) { +func TestParseInsertUpdate(t *testing.T) { type expected struct { query string schema *Schema diff --git a/internal/mysql/schema.go b/internal/mysql/schema.go index 2306b0652e..5b00259f97 100644 --- a/internal/mysql/schema.go +++ b/internal/mysql/schema.go @@ -20,16 +20,25 @@ type Schema struct { } // returns a deep copy of the column definition for using as a query return type or param type -func (s *Schema) getColType(col *sqlparser.ColName, defaultTableName string) (*sqlparser.ColumnDefinition, error) { +func (s *Schema) getColType(col *sqlparser.ColName, tableAliasMap map[string]string, defaultTableName string) (*sqlparser.ColumnDefinition, error) { if !col.Qualifier.IsEmpty() { - colDfn, err := s.schemaLookup(col.Qualifier.Name.String(), col.Name.String()) - if err != nil { - return nil, err + var colDfn *sqlparser.ColumnDefinition + var err error + if realTableName, ok := tableAliasMap[col.Qualifier.Name.String()]; ok { + colDfn, err = s.schemaLookup(realTableName, col.Name.String()) + } else { + colDfn, err = s.schemaLookup(col.Qualifier.Name.String(), col.Name.String()) + if err != nil { + return nil, err + } } return &sqlparser.ColumnDefinition{ Name: colDfn.Name, Type: colDfn.Type, }, nil } + if defaultTableName == "" { + return nil, fmt.Errorf("Column reference [%v] is ambiguous -- Add a qualifier", col.Name.String()) + } colDfn, err := s.schemaLookup(defaultTableName, col.Name.String()) if err != nil { return nil, err From ea0ac86e9d5f443ad65f0ba98a60d8904873f3a1 Mon Sep 17 00:00:00 2001 From: cmoog Date: Thu, 2 Jan 2020 10:38:57 +1300 Subject: [PATCH 09/24] allows null returns for left join colums --- internal/mysql/param.go | 6 +++--- internal/mysql/param_test.go | 2 +- internal/mysql/parse.go | 42 +++++++++++++++++++++++++++--------- internal/mysql/parse_test.go | 14 ++++++------ internal/mysql/schema.go | 26 +++++++++++----------- 5 files changed, 56 insertions(+), 34 deletions(-) diff --git a/internal/mysql/param.go b/internal/mysql/param.go index 83a4ac5d33..b26b420883 100644 --- a/internal/mysql/param.go +++ b/internal/mysql/param.go @@ -15,7 +15,7 @@ type Param struct { typ string } -func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, tableAliasMap map[string]string, settings dinosql.GenerateSettings) ([]*Param, error) { +func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, tableAliasMap FromTables, settings dinosql.GenerateSettings) ([]*Param, error) { params := []*Param{} if limit == nil { return params, nil @@ -40,7 +40,7 @@ func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, tableAliasMap map[stri return params, nil } -func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, tableAliasMap map[string]string, defaultTable string, settings dinosql.GenerateSettings) ([]*Param, error) { +func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, tableAliasMap FromTables, defaultTable string, settings dinosql.GenerateSettings) ([]*Param, error) { params := []*Param{} switch v := e.(type) { case *sqlparser.Where: @@ -85,7 +85,7 @@ func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, tableAliasMap map[string] return params, nil } -func paramInComparison(cond *sqlparser.ComparisonExpr, s *Schema, tableAliasMap map[string]string, defaultTable string, settings dinosql.GenerateSettings) (*Param, bool, error) { +func paramInComparison(cond *sqlparser.ComparisonExpr, s *Schema, tableAliasMap FromTables, defaultTable string, settings dinosql.GenerateSettings) (*Param, bool, error) { p := &Param{} var colIdent sqlparser.ColIdent walker := func(node sqlparser.SQLNode) (bool, error) { diff --git a/internal/mysql/param_test.go b/internal/mysql/param_test.go index b323b1157c..a7dccaf6d3 100644 --- a/internal/mysql/param_test.go +++ b/internal/mysql/param_test.go @@ -83,7 +83,7 @@ func TestSelectParamSearcher(t *testing.T) { } selectStm, ok := tree.(*sqlparser.Select) - tableAliasMap, err := parseFrom(selectStm.From) + tableAliasMap, err := parseFrom(selectStm.From, false) if err != nil { t.Errorf("Failed to parse table name alias's: %v", err) } diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index a7d96928a4..7b87a381d6 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -108,7 +108,7 @@ func (q *Query) parseNameAndCmd() error { } func parseSelect(tree *sqlparser.Select, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { - tableAliasMap, err := parseFrom(tree.From) + tableAliasMap, err := parseFrom(tree.From, false) if err != nil { return nil, fmt.Errorf("Failed to parse table name alias's: %v", err) } @@ -160,8 +160,14 @@ func parseSelect(tree *sqlparser.Select, query string, s *Schema, settings dinos return &parsedQuery, nil } -func parseFrom(from sqlparser.TableExprs) (map[string]string, error) { - tables := make(map[string]string) +type FromTable struct { + TrueName string // the true table name as described in the schema + IsLeftJoined bool // which could result in null columns +} +type FromTables map[string]FromTable + +func parseFrom(from sqlparser.TableExprs, isLeftJoined bool) (FromTables, error) { + tables := make(map[string]FromTable) for _, expr := range from { switch v := expr.(type) { case *sqlparser.AliasedTableExpr: @@ -169,13 +175,29 @@ func parseFrom(from sqlparser.TableExprs) (map[string]string, error) { if !ok { return nil, fmt.Errorf("Failed to parse AliasedTableExpr name: %v", spew.Sdump(v)) } + t := FromTable{ + TrueName: name.Name.String(), + IsLeftJoined: isLeftJoined, + } if v.As.String() != "" { - tables[v.As.String()] = name.Name.String() + tables[v.As.String()] = t } else { - tables[name.Name.String()] = name.Name.String() + tables[name.Name.String()] = t } case *sqlparser.JoinTableExpr: - return parseFrom([]sqlparser.TableExpr{v.LeftExpr, v.RightExpr}) + left, err := parseFrom([]sqlparser.TableExpr{v.LeftExpr}, false) + if err != nil { + return nil, err + } + right, err := parseFrom([]sqlparser.TableExpr{v.RightExpr}, true) + if err != nil { + return nil, err + } + // merge the left and right maps + for k, v := range left { + right[k] = v + } + return right, nil default: return nil, fmt.Errorf("Failed to parse table expr: %v", spew.Sdump(v)) } @@ -183,18 +205,18 @@ func parseFrom(from sqlparser.TableExprs) (map[string]string, error) { return tables, nil } -func getDefaultTable(tableAliasMap map[string]string) string { +func getDefaultTable(tableAliasMap FromTables) string { if len(tableAliasMap) != 1 { return "" } for _, val := range tableAliasMap { - return val + return val.TrueName } panic("Should never be reached.") } func parseUpdate(node *sqlparser.Update, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { - tableAliasMap, err := parseFrom(node.TableExprs) + tableAliasMap, err := parseFrom(node.TableExprs, false) if err != nil { return nil, fmt.Errorf("Failed to parse table name alias's: %v", err) } @@ -309,7 +331,7 @@ func (q *Query) parseLeadingComment(comment string) error { return nil } -func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, tableAliasMap map[string]string, defaultTable string) ([]*sqlparser.ColumnDefinition, error) { +func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, tableAliasMap FromTables, defaultTable string) ([]*sqlparser.ColumnDefinition, error) { colDfns := []*sqlparser.ColumnDefinition{} for _, col := range exprs { switch expr := col.(type) { diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go index e55585fd8e..07ec28eac4 100644 --- a/internal/mysql/parse_test.go +++ b/internal/mysql/parse_test.go @@ -139,7 +139,7 @@ func TestParseSelect(t *testing.T) { testCase{ input: expected{ query: `/* name: GetCount :one */ - SELECT id my_id, COUNT(id) id_count FROM users WHERE id > 4`, + SELECT id my_id, COUNT(id) id_count FROM users WHERE id > 4`, schema: mockSchema, }, output: &Query{ @@ -171,7 +171,7 @@ func TestParseSelect(t *testing.T) { testCase{ input: expected{ query: `/* name: GetNameByID :one */ - SELECT first_name, last_name FROM users WHERE id = ?`, + SELECT first_name, last_name FROM users WHERE id = ?`, schema: mockSchema, }, output: &Query{ @@ -207,26 +207,26 @@ func TestParseSelect(t *testing.T) { testCase{ input: expected{ query: `/* name: GetAllUsersOrders :many */ - SELECT u.id, u.first_name, o.price, o.id order_id + SELECT u.id user_id, u.first_name, o.price, o.id order_id FROM orders o LEFT JOIN users u ON u.id = o.user_id`, schema: mockSchema, }, output: &Query{ - SQL: "select u.id, u.first_name, o.price, o.id as order_id from orders as o left join users as u on u.id = o.user_id", + SQL: "select u.id as user_id, u.first_name, o.price, o.id as order_id from orders as o left join users as u on u.id = o.user_id", Columns: []*sqlparser.ColumnDefinition{ &sqlparser.ColumnDefinition{ - Name: sqlparser.NewColIdent("id"), + Name: sqlparser.NewColIdent("user_id"), Type: sqlparser.ColumnType{ Type: "int", Autoincrement: true, - NotNull: true, + NotNull: false, // beause of the left join }, }, &sqlparser.ColumnDefinition{ Name: sqlparser.NewColIdent("first_name"), Type: sqlparser.ColumnType{ Type: "varchar", - NotNull: true, + NotNull: false, // because of left join }, }, &sqlparser.ColumnDefinition{ diff --git a/internal/mysql/schema.go b/internal/mysql/schema.go index 5b00259f97..06f47835e8 100644 --- a/internal/mysql/schema.go +++ b/internal/mysql/schema.go @@ -20,21 +20,21 @@ type Schema struct { } // returns a deep copy of the column definition for using as a query return type or param type -func (s *Schema) getColType(col *sqlparser.ColName, tableAliasMap map[string]string, defaultTableName string) (*sqlparser.ColumnDefinition, error) { +func (s *Schema) getColType(col *sqlparser.ColName, tableAliasMap FromTables, defaultTableName string) (*sqlparser.ColumnDefinition, error) { if !col.Qualifier.IsEmpty() { - var colDfn *sqlparser.ColumnDefinition - var err error - if realTableName, ok := tableAliasMap[col.Qualifier.Name.String()]; ok { - colDfn, err = s.schemaLookup(realTableName, col.Name.String()) - } else { - colDfn, err = s.schemaLookup(col.Qualifier.Name.String(), col.Name.String()) - if err != nil { - return nil, err - } + realTable, ok := tableAliasMap[col.Qualifier.Name.String()] + if !ok { + return nil, fmt.Errorf("Column qualifier [%v] not found in table alias map", col.Qualifier.Name.String()) } - return &sqlparser.ColumnDefinition{ - Name: colDfn.Name, Type: colDfn.Type, - }, nil + colDfn, err := s.schemaLookup(realTable.TrueName, col.Name.String()) + colDfnCopy := *colDfn + if err != nil { + return nil, err + } + if realTable.IsLeftJoined { + colDfnCopy.Type.NotNull = false + } + return &colDfnCopy, nil } if defaultTableName == "" { return nil, fmt.Errorf("Column reference [%v] is ambiguous -- Add a qualifier", col.Name.String()) From 9f31cb97f679c1f43a6d67985b66ec0922661f87 Mon Sep 17 00:00:00 2001 From: cmoog Date: Thu, 2 Jan 2020 11:42:20 +1300 Subject: [PATCH 10/24] patches left join nulls and gives example --- internal/mysql/example/queries.sql | 13 ++++++++----- internal/mysql/parse.go | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/mysql/example/queries.sql b/internal/mysql/example/queries.sql index 266c604f03..0331492dc3 100644 --- a/internal/mysql/example/queries.sql +++ b/internal/mysql/example/queries.sql @@ -2,7 +2,8 @@ CREATE TABLE teachers ( id int NOT NULL, first_name varchar(255), last_name varchar(255), - school_id VARCHAR(255) NOT NULL, + school_id int NOT NULL, + class_id int NOT NULL, school_lat FLOAT, school_lng FLOAT, department ENUM("English", "Math"), @@ -11,7 +12,7 @@ CREATE TABLE teachers ( CREATE TABLE students ( id int NOT NULL, - student_id varchar(10), + class_id int NOT NULL, first_name varchar(255), last_name varchar(255), PRIMARY KEY (id) @@ -26,7 +27,9 @@ SELECT school_id, id FROM teachers WHERE school_lng > ? AND school_lat < ?; /* name: TeachersByID :one */ SELECT id, school_lat FROM teachers WHERE id = ? LIMIT 10 -/* name: GetStudents :many */ -SELECT students.first_name, students.last_name, teachers.first_name teacherFirstName +/* name: GetStudentsTeacher :one */ +SELECT students.first_name, students.last_name, teachers.first_name teacherFirstName, +teachers.id teacher_id FROM students - LEFT JOIN teachers on teachers.school_id = students.school_id \ No newline at end of file + Left JOIN teachers on teachers.class_id = students.class_id + WHERE students.id = ? diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 7b87a381d6..49472a89d2 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -185,11 +185,12 @@ func parseFrom(from sqlparser.TableExprs, isLeftJoined bool) (FromTables, error) tables[name.Name.String()] = t } case *sqlparser.JoinTableExpr: + isLeftJoin := v.Join == "left join" left, err := parseFrom([]sqlparser.TableExpr{v.LeftExpr}, false) if err != nil { return nil, err } - right, err := parseFrom([]sqlparser.TableExpr{v.RightExpr}, true) + right, err := parseFrom([]sqlparser.TableExpr{v.RightExpr}, isLeftJoin) if err != nil { return nil, err } From b2c24e70f0880754e8cf2b3241ff514791f3cda0 Mon Sep 17 00:00:00 2001 From: cmoog Date: Thu, 2 Jan 2020 11:45:20 +1300 Subject: [PATCH 11/24] patches named param replacement --- internal/mysql/example/queries.sql | 2 +- internal/mysql/param.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/mysql/example/queries.sql b/internal/mysql/example/queries.sql index 0331492dc3..ff927fba39 100644 --- a/internal/mysql/example/queries.sql +++ b/internal/mysql/example/queries.sql @@ -32,4 +32,4 @@ SELECT students.first_name, students.last_name, teachers.first_name teacherFirst teachers.id teacher_id FROM students Left JOIN teachers on teachers.class_id = students.class_id - WHERE students.id = ? + WHERE students.id = :studentID diff --git a/internal/mysql/param.go b/internal/mysql/param.go index b26b420883..ce3b800ad2 100644 --- a/internal/mysql/param.go +++ b/internal/mysql/param.go @@ -129,8 +129,8 @@ func paramName(col sqlparser.ColIdent, originalName string) string { } func replaceParamStrs(query string, params []*Param) (string, error) { - for ix := range params { - re, err := regexp.Compile(fmt.Sprintf("(:v%d)", ix+1)) + for _, p := range params { + re, err := regexp.Compile(fmt.Sprintf("(%v)", p.originalName)) if err != nil { return "", err } From e95074eafbc18c8661e92a332661817d4e17d459 Mon Sep 17 00:00:00 2001 From: cmoog Date: Thu, 2 Jan 2020 21:11:09 +1300 Subject: [PATCH 12/24] comments for exported items --- internal/mysql/gen.go | 1 + internal/mysql/param.go | 2 ++ internal/mysql/parse.go | 9 ++++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 9ffe21847c..7e3ed37646 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -17,6 +17,7 @@ type Result struct { packageName string } +// PkgName exposes the result set's associated go package identifier as specified in the sqlc.json config. func (r *Result) PkgName() string { return r.packageName } diff --git a/internal/mysql/param.go b/internal/mysql/param.go index ce3b800ad2..b0481af68d 100644 --- a/internal/mysql/param.go +++ b/internal/mysql/param.go @@ -9,6 +9,8 @@ import ( "vitess.io/vitess/go/vt/sqlparser" ) +// Param describes a runtime query parameter with its +// associated type. Example: "SELECT name FROM users id = ?" type Param struct { originalName string name string diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 49472a89d2..63091e2416 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -47,12 +47,11 @@ func parseFile(filepath string, inPkg string, s *Schema, settings dinosql.Genera parsedQueries = append(parsedQueries, result) } - r := Result{ + return &Result{ Queries: parsedQueries, Schema: s, packageName: inPkg, - } - return &r, nil + }, nil } func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { @@ -160,10 +159,14 @@ func parseSelect(tree *sqlparser.Select, query string, s *Schema, settings dinos return &parsedQuery, nil } +// FromTable describes a table reference in the "FROM" clause of a query. type FromTable struct { TrueName string // the true table name as described in the schema IsLeftJoined bool // which could result in null columns } + +// FromTables describes a map between table alias expressions and the +// proper table name type FromTables map[string]FromTable func parseFrom(from sqlparser.TableExprs, isLeftJoined bool) (FromTables, error) { From 644f77b07796e63bdf9cb1dab2c0ae8164b1c55a Mon Sep 17 00:00:00 2001 From: cmoog Date: Thu, 2 Jan 2020 22:00:24 +1300 Subject: [PATCH 13/24] adds a few common col types --- internal/mysql/gen.go | 7 ++++++- internal/mysql/param.go | 3 +++ internal/mysql/schema.go | 5 ++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 7e3ed37646..6aaca2c7e3 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -5,6 +5,7 @@ import ( "sort" "strings" + "github.com/davecgh/go-spew/spew" "github.com/jinzhu/inflection" "github.com/kyleconroy/sqlc/internal/dinosql" "vitess.io/vitess/go/vt/sqlparser" @@ -235,9 +236,13 @@ func goTypeCol(col *sqlparser.ColumnDefinition, settings dinosql.GenerateSetting return "sql.NullFloat64" case "enum" == t: return enumNameFromColDef(col, settings) + case "date" == t, "timestamp" == t: + return "time.Time" + case "boolean" == t: + return "bool" default: // TODO: remove panic here - panic(fmt.Sprintf("Handle this col type directly: %v\n", col.Type)) + panic(fmt.Sprintf("Handle this col type directly: %v\n", spew.Sdump(col.Type))) // return col.Type } } diff --git a/internal/mysql/param.go b/internal/mysql/param.go index b0481af68d..283221b627 100644 --- a/internal/mysql/param.go +++ b/internal/mysql/param.go @@ -80,6 +80,9 @@ func paramsInWhereExpr(e sqlparser.SQLNode, s *Schema, tableAliasMap FromTables, return nil, err } params = append(params, right...) + case *sqlparser.IsExpr: + // TODO: see if there is a use case for params in IS expressions + return []*Param{}, nil default: panic(fmt.Sprintf("Failed to handle %T in where", v)) } diff --git a/internal/mysql/schema.go b/internal/mysql/schema.go index 06f47835e8..12900d4cb0 100644 --- a/internal/mysql/schema.go +++ b/internal/mysql/schema.go @@ -27,10 +27,10 @@ func (s *Schema) getColType(col *sqlparser.ColName, tableAliasMap FromTables, de return nil, fmt.Errorf("Column qualifier [%v] not found in table alias map", col.Qualifier.Name.String()) } colDfn, err := s.schemaLookup(realTable.TrueName, col.Name.String()) - colDfnCopy := *colDfn if err != nil { return nil, err } + colDfnCopy := *colDfn if realTable.IsLeftJoined { colDfnCopy.Type.NotNull = false } @@ -51,6 +51,9 @@ func (s *Schema) getColType(col *sqlparser.ColName, tableAliasMap FromTables, de // Add add a MySQL table definition to the schema map func (s *Schema) Add(table *sqlparser.DDL) { name := table.Table.Name.String() + if table.TableSpec == nil { + panic(fmt.Sprintf("Failed to parse table [%v] schema.", name)) + } s.tables[name] = table.TableSpec.Columns } From a65460e178ae3b43e960ba30dfd39dd995d029b1 Mon Sep 17 00:00:00 2001 From: cmoog Date: Thu, 2 Jan 2020 22:30:11 +1300 Subject: [PATCH 14/24] adds support for group _concat --- internal/mysql/functions.go | 6 ++++-- internal/mysql/gen.go | 2 +- internal/mysql/parse.go | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/mysql/functions.go b/internal/mysql/functions.go index 9616a2b66f..01d6a53aaf 100644 --- a/internal/mysql/functions.go +++ b/internal/mysql/functions.go @@ -1,13 +1,15 @@ package mysql -import "fmt" +import ( + "fmt" +) // converts MySQL function name to MySQL return type func functionReturnType(f string) string { switch f { case "avg", "count", "instr", "sum", "min", "max", "length", "char_length": return "int" - case "concat", "left", "replace", "substring", "trim", "find_in_set", "format": + case "concat", "left", "replace", "substring", "trim", "find_in_set", "format", "group_concat": return "varchar" default: panic(fmt.Sprintf("unknown mysql function type [%v]", f)) diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 6aaca2c7e3..0f1fbc0a7a 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -219,7 +219,7 @@ func (r *Result) columnsToStruct(name string, items []structParams, settings din func goTypeCol(col *sqlparser.ColumnDefinition, settings dinosql.GenerateSettings) string { switch t := col.Type.Type; { - case "varchar" == t: + case "varchar" == t, "text" == t: if col.Type.NotNull { return "string" } diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 63091e2416..387ab92981 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -352,6 +352,14 @@ func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, tableAliasMap res.Name = expr.As // applys the alias } colDfns = append(colDfns, res) + case *sqlparser.GroupConcatExpr: + colDfns = append(colDfns, &sqlparser.ColumnDefinition{ + Name: sqlparser.NewColIdent(expr.As.String()), + Type: sqlparser.ColumnType{ + Type: "varchar", + NotNull: true, + }, + }) case *sqlparser.FuncExpr: funcName := v.Name.Lowered() funcType := functionReturnType(funcName) From 5498437e37043a2aed584b61c4ef594eac6f46fb Mon Sep 17 00:00:00 2001 From: cmoog Date: Thu, 2 Jan 2020 23:18:47 +1300 Subject: [PATCH 15/24] adds support for delete statement --- internal/mysql/parse.go | 55 ++++++++++++++++++++++++++++++++---- internal/mysql/parse_test.go | 15 ++++------ 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 387ab92981..c9cd829cdc 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -1,7 +1,6 @@ package mysql import ( - "errors" "fmt" "io/ioutil" "os" @@ -37,6 +36,9 @@ func parseFile(filepath string, inPkg string, s *Schema, settings dinosql.Genera parsedQueries := []*Query{} for _, query := range rawQueries { + if query == "" { + continue + } result, err := parseQueryString(query, s, settings) if err != nil { return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filepath, err) @@ -63,11 +65,11 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings var parsedQuery *Query switch tree := tree.(type) { case *sqlparser.Select: - res, err := parseSelect(tree, query, s, settings) + selectQuery, err := parseSelect(tree, query, s, settings) if err != nil { return nil, fmt.Errorf("Failed to parse SELECT query: %v", err) } - parsedQuery = res + parsedQuery = selectQuery case *sqlparser.Insert: insert, err := parseInsert(tree, query, s, settings) if err != nil { @@ -80,13 +82,21 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings return nil, fmt.Errorf("Failed to parse UPDATE query: %v", err) } parsedQuery = update + case *sqlparser.Delete: + delete, err := parseDelete(tree, query, s, settings) + delete.schemaLookup = nil + spew.Dump(delete) + if err != nil { + return nil, fmt.Errorf("Failed to parse DELETE query: %v", err) + } + parsedQuery = delete case *sqlparser.DDL: s.Add(tree) return nil, nil default: panic("Unsupported SQL statement type") } - paramsReplacedQuery, err := replaceParamStrs(parsedQuery.SQL, parsedQuery.Params) + paramsReplacedQuery, err := replaceParamStrs(sqlparser.String(tree), parsedQuery.Params) if err != nil { return nil, fmt.Errorf("Failed to replace param variables in query string: %v", err) } @@ -96,7 +106,7 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings func (q *Query) parseNameAndCmd() error { if q == nil { - return errors.New("Cannot parse name and cmd from null query") + return fmt.Errorf("Cannot parse name and cmd from null query") } _, comments := sqlparser.SplitMarginComments(q.SQL) err := q.parseLeadingComment(comments.Leading) @@ -154,7 +164,6 @@ func parseSelect(tree *sqlparser.Select, query string, s *Schema, settings dinos if err != nil { return nil, err } - parsedQuery.SQL = sqlparser.String(tree) return &parsedQuery, nil } @@ -307,6 +316,40 @@ func parseInsert(node *sqlparser.Insert, query string, s *Schema, settings dinos return parsedQuery, nil } +func parseDelete(node *sqlparser.Delete, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { + tableAliasMap, err := parseFrom(node.TableExprs, false) + if err != nil { + return nil, fmt.Errorf("Failed to parse table name alias's: %v", err) + } + defaultTableName := getDefaultTable(tableAliasMap) + if err != nil { + return nil, err + } + + whereParams, err := paramsInWhereExpr(node.Where, s, tableAliasMap, defaultTableName, settings) + if err != nil { + return nil, err + } + + limitParams, err := paramsInLimitExpr(node.Limit, s, tableAliasMap, settings) + if err != nil { + return nil, err + } + parsedQuery := &Query{ + SQL: query, + Params: append(whereParams, limitParams...), + Columns: nil, + defaultTableName: defaultTableName, + schemaLookup: s, + } + err = parsedQuery.parseNameAndCmd() + if err != nil { + return nil, err + } + + return parsedQuery, nil +} + func (q *Query) parseLeadingComment(comment string) error { for _, line := range strings.Split(comment, "\n") { if !strings.HasPrefix(line, "/* name:") { diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go index 07ec28eac4..555c275e9b 100644 --- a/internal/mysql/parse_test.go +++ b/internal/mysql/parse_test.go @@ -133,8 +133,6 @@ func TestParseSelect(t *testing.T) { input expected output *Query } - query2 := `/* name: GetAll :many */ - SELECT * FROM users;` tests := []testCase{ testCase{ input: expected{ @@ -191,7 +189,8 @@ func TestParseSelect(t *testing.T) { }, testCase{ input: expected{ - query: query2, + query: `/* name: GetAll :many */ + SELECT * FROM users;`, schema: mockSchema, }, output: &Query{ @@ -324,10 +323,8 @@ func TestParseInsertUpdate(t *testing.T) { input expected output *Query } - query1 := `/* name: InsertNewUser :exec */ - INSERT INTO users (first_name, last_name) VALUES (?, ?)` - query2 := `/* name: UpdateUserAt :exec */ - UPDATE users SET first_name = ?, last_name = ? WHERE id > ? AND first_name = ? LIMIT 3` + query1 := "/* name: InsertNewUser :exec */\nINSERT INTO users (first_name, last_name) VALUES (?, ?)" + query2 := "/* name: UpdateUserAt :exec */\nUPDATE users SET first_name = ?, last_name = ? WHERE id > ? AND first_name = ? LIMIT 3" tests := []testCase{ testCase{ input: expected{ @@ -335,7 +332,7 @@ func TestParseInsertUpdate(t *testing.T) { schema: mockSchema, }, output: &Query{ - SQL: query1, + SQL: "insert into users(first_name, last_name) values (?, ?)", Columns: nil, Params: []*Param{ &Param{ @@ -361,7 +358,7 @@ func TestParseInsertUpdate(t *testing.T) { schema: mockSchema, }, output: &Query{ - SQL: query2, + SQL: "update users set first_name = ?, last_name = ? where id > ? and first_name = ? limit 3", Columns: nil, Params: []*Param{ &Param{ From d016d470eaf0f080b667e2d182d70b1e9a2f0d56 Mon Sep 17 00:00:00 2001 From: cmoog Date: Fri, 3 Jan 2020 13:39:43 +1300 Subject: [PATCH 16/24] removes debug logging --- internal/mysql/parse.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index c9cd829cdc..cdcbf4aca2 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -85,7 +85,6 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings case *sqlparser.Delete: delete, err := parseDelete(tree, query, s, settings) delete.schemaLookup = nil - spew.Dump(delete) if err != nil { return nil, fmt.Errorf("Failed to parse DELETE query: %v", err) } From 54c5c49aa2d53ceca3856893bcd8aef9e0489f04 Mon Sep 17 00:00:00 2001 From: cmoog Date: Fri, 3 Jan 2020 14:01:44 +1300 Subject: [PATCH 17/24] patches config parsing --- internal/cmd/cmd.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 60dfbeee0d..bc080f501e 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -201,13 +201,13 @@ var checkCmd = &cobra.Command{ Use: "compile", Short: "Statically check SQL for syntax and type errors", RunE: func(cmd *cobra.Command, args []string) error { - blob, err := ioutil.ReadFile("sqlc.json") + file, err := os.Open("sqlc.json") if err != nil { return err } - var settings dinosql.GenerateSettings - if err := json.Unmarshal(blob, &settings); err != nil { + settings, err := dinosql.ParseConfig(file) + if err != nil { return err } @@ -228,13 +228,13 @@ var unstable__mysql = &cobra.Command{ Use: "unstable__mysql generate", Short: "Generate MySQL Queries into typesafe Go code", RunE: func(cmd *cobra.Command, args []string) error { - blob, err := ioutil.ReadFile("sqlc.json") + file, err := os.Open("sqlc.json") if err != nil { return err } - var settings dinosql.GenerateSettings - if err := json.Unmarshal(blob, &settings); err != nil { + settings, err := dinosql.ParseConfig(file) + if err != nil { return err } From 32de207071317f67a705b74b499b7db1bd17e20b Mon Sep 17 00:00:00 2001 From: cmoog Date: Fri, 3 Jan 2020 14:07:42 +1300 Subject: [PATCH 18/24] patches json tags --- internal/mysql/gen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 0f1fbc0a7a..9a78efaf0a 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -74,7 +74,7 @@ func (r *Result) Structs(settings dinosql.GenerateSettings) []dinosql.GoStruct { s.Fields = append(s.Fields, dinosql.GoField{ Name: dinosql.StructName(col.Name.String(), settings), Type: goTypeCol(col, settings), - Tags: map[string]string{"json": col.Name.String()}, + Tags: map[string]string{"json:": col.Name.String()}, Comment: "", }) } From 865e3b008fdbba0cec4f055345500b3a16d81867 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 6 Jan 2020 14:29:59 -0800 Subject: [PATCH 19/24] internal/cmd: Add database field to packages Each package now has a `database` field, which can be set to either `mysql` or `postgresql`. --- internal/cmd/cmd.go | 95 ++++++++++++++------------------ internal/dinosql/config.go | 11 ++++ internal/mysql/example/sqlc.json | 1 + internal/mysql/gen.go | 2 +- internal/mysql/parse.go | 23 ++++---- 5 files changed, 65 insertions(+), 67 deletions(-) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index bc080f501e..49373b6f07 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -22,7 +22,6 @@ import ( func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int { rootCmd := &cobra.Command{Use: "sqlc", SilenceUsage: true} rootCmd.AddCommand(checkCmd) - rootCmd.AddCommand(unstable__mysql) rootCmd.AddCommand(genCmd) rootCmd.AddCommand(initCmd) rootCmd.AddCommand(parseCmd) @@ -141,35 +140,54 @@ var genCmd = &cobra.Command{ name = filepath.Base(pkg.Path) } - c, err := dinosql.ParseCatalog(pkg.Schema) - if err != nil { - fmt.Fprintf(os.Stderr, "# package %s\n", name) - if parserErr, ok := err.(*dinosql.ParserErr); ok { - for _, fileErr := range parserErr.Errs { - fmt.Fprintf(os.Stderr, "%s:%d:%d: %s\n", fileErr.Filename, fileErr.Line, fileErr.Column, fileErr.Err) + var result dinosql.Generateable + + switch pkg.Database { + + case dinosql.DatabaseMySQL: + // Experimental MySQL support + q, err := mysql.GeneratePkg(pkg.Name, pkg.Queries, settings) + if err != nil { + fmt.Fprintf(os.Stderr, "# package %s\n", name) + fmt.Fprintf(os.Stderr, "error parsing file: %s\n", err) + errored = true + continue + } + result = q + + case dinosql.DatabasePostgreSQL: + c, err := dinosql.ParseCatalog(pkg.Schema) + if err != nil { + fmt.Fprintf(os.Stderr, "# package %s\n", name) + if parserErr, ok := err.(*dinosql.ParserErr); ok { + for _, fileErr := range parserErr.Errs { + fmt.Fprintf(os.Stderr, "%s:%d:%d: %s\n", fileErr.Filename, fileErr.Line, fileErr.Column, fileErr.Err) + } + } else { + fmt.Fprintf(os.Stderr, "error parsing schema: %s\n", err) } - } else { - fmt.Fprintf(os.Stderr, "error parsing schema: %s\n", err) + errored = true + continue } - errored = true - continue - } - q, err := dinosql.ParseQueries(c, pkg) - if err != nil { - fmt.Fprintf(os.Stderr, "# package %s\n", name) - if parserErr, ok := err.(*dinosql.ParserErr); ok { - for _, fileErr := range parserErr.Errs { - fmt.Fprintf(os.Stderr, "%s:%d:%d: %s\n", fileErr.Filename, fileErr.Line, fileErr.Column, fileErr.Err) + q, err := dinosql.ParseQueries(c, pkg) + if err != nil { + fmt.Fprintf(os.Stderr, "# package %s\n", name) + if parserErr, ok := err.(*dinosql.ParserErr); ok { + for _, fileErr := range parserErr.Errs { + fmt.Fprintf(os.Stderr, "%s:%d:%d: %s\n", fileErr.Filename, fileErr.Line, fileErr.Column, fileErr.Err) + } + } else { + fmt.Fprintf(os.Stderr, "error parsing queries: %s\n", err) } - } else { - fmt.Fprintf(os.Stderr, "error parsing queries: %s\n", err) + errored = true + continue } - errored = true - continue + result = q + } - files, err := dinosql.Generate(q, settings) + files, err := dinosql.Generate(result, settings) if err != nil { fmt.Fprintf(os.Stderr, "# package %s\n", name) fmt.Fprintf(os.Stderr, "error generating code: %s\n", err) @@ -223,34 +241,3 @@ var checkCmd = &cobra.Command{ return nil }, } - -var unstable__mysql = &cobra.Command{ - Use: "unstable__mysql generate", - Short: "Generate MySQL Queries into typesafe Go code", - RunE: func(cmd *cobra.Command, args []string) error { - file, err := os.Open("sqlc.json") - if err != nil { - return err - } - - settings, err := dinosql.ParseConfig(file) - if err != nil { - return err - } - - for _, pkg := range settings.Packages { - res, err := mysql.GeneratePkg(pkg.Name, pkg.Queries, settings) - if err != nil { - return err - } - for filename, source := range res { - os.MkdirAll(filepath.Dir(filename), 0755) - if err := ioutil.WriteFile(filename, []byte(source), 0644); err != nil { - fmt.Fprintf(os.Stderr, "%s: %s\n", filename, err) - os.Exit(1) - } - } - } - return nil - }, -} diff --git a/internal/dinosql/config.go b/internal/dinosql/config.go index 6b692665ab..3d388777cc 100644 --- a/internal/dinosql/config.go +++ b/internal/dinosql/config.go @@ -33,8 +33,16 @@ type GenerateSettings struct { PackageMap map[string]PackageSettings } +type Database string + +const ( + DatabaseMySQL Database = "mysql" + DatabasePostgreSQL Database = "postgresql" +) + type PackageSettings struct { Name string `json:"name"` + Database Database `json:"database,omitempty"` Path string `json:"path"` Schema string `json:"schema"` Queries string `json:"queries"` @@ -140,6 +148,9 @@ func ParseConfig(rd io.Reader) (GenerateSettings, error) { return config, err } } + if config.Packages[j].Database == "" { + config.Packages[j].Database = DatabasePostgreSQL + } } err := config.PopulatePkgMap() diff --git a/internal/mysql/example/sqlc.json b/internal/mysql/example/sqlc.json index b09d8467af..223bd64de1 100644 --- a/internal/mysql/example/sqlc.json +++ b/internal/mysql/example/sqlc.json @@ -3,6 +3,7 @@ "packages": [ { "name": "teachersDB", + "database": "mysql", "emit_json_tags": true, "emit_prepared_queries": false, "path": "./", diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 9a78efaf0a..794e86aeea 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -105,7 +105,7 @@ func (r *Result) GoQueries(settings dinosql.GenerateSettings) []dinosql.GoQuery ConstantName: dinosql.LowerTitle(query.Name), FieldName: dinosql.LowerTitle(query.Name) + "Stmt", MethodName: query.Name, - SourceName: "queries", // query.Filename, + SourceName: query.Filename, SQL: query.SQL, // Comments: query.Comments, } diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index cdcbf4aca2..31d334abd5 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strings" "github.com/davecgh/go-spew/spew" @@ -20,16 +21,18 @@ type Query struct { Cmd string // TODO: Pick a better name. One of: one, many, exec, execrows defaultTableName string // for columns that are not qualified schemaLookup *Schema // for validation and conversion to Go types + + Filename string } -func parseFile(filepath string, inPkg string, s *Schema, settings dinosql.GenerateSettings) (*Result, error) { - file, err := os.Open(filepath) +func parseFile(filename string, inPkg string, s *Schema, settings dinosql.GenerateSettings) (*Result, error) { + file, err := os.Open(filename) if err != nil { - return nil, fmt.Errorf("Failed to open file [%v]: %v", filepath, err) + return nil, fmt.Errorf("Failed to open file [%v]: %v", filename, err) } contents, err := ioutil.ReadAll(file) if err != nil { - return nil, fmt.Errorf("Failed to read contents of file [%v]: %v", filepath, err) + return nil, fmt.Errorf("Failed to read contents of file [%v]: %v", filename, err) } rawQueries := strings.Split(string(contents), "\n\n") @@ -41,11 +44,12 @@ func parseFile(filepath string, inPkg string, s *Schema, settings dinosql.Genera } result, err := parseQueryString(query, s, settings) if err != nil { - return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filepath, err) + return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filename, err) } if result == nil { continue } + result.Filename = filepath.Base(filename) parsedQueries = append(parsedQueries, result) } @@ -430,16 +434,11 @@ func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, tableAliasMap } // GeneratePkg is the main entry to mysql generator package -func GeneratePkg(pkgName string, querysPath string, settings dinosql.GenerateSettings) (map[string]string, error) { +func GeneratePkg(pkgName string, querysPath string, settings dinosql.GenerateSettings) (*Result, error) { s := NewSchema() result, err := parseFile(querysPath, pkgName, s, settings) if err != nil { return nil, err } - output, err := dinosql.Generate(result, settings) - if err != nil { - return nil, fmt.Errorf("Failed to generate output: %v", err) - } - - return output, nil + return result, nil } From a0f799c8b22fa6e88701f20dd469bb5c28afcf5d Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 6 Jan 2020 15:06:52 -0800 Subject: [PATCH 20/24] internal/mysql: Parse a directory of SQL files --- internal/cmd/cmd.go | 2 +- internal/dinosql/parser.go | 31 +++++++++++++++------- internal/mysql/parse.go | 50 +++++++++++++++++++++--------------- internal/mysql/parse_test.go | 2 +- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 49373b6f07..2f2205499f 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -146,7 +146,7 @@ var genCmd = &cobra.Command{ case dinosql.DatabaseMySQL: // Experimental MySQL support - q, err := mysql.GeneratePkg(pkg.Name, pkg.Queries, settings) + q, err := mysql.GeneratePkg(pkg.Name, pkg.Schema, pkg.Queries, settings) if err != nil { fmt.Fprintf(os.Stderr, "# package %s\n", name) fmt.Fprintf(os.Stderr, "error parsing file: %s\n", err) diff --git a/internal/dinosql/parser.go b/internal/dinosql/parser.go index fc3098f0c5..03c29d6546 100644 --- a/internal/dinosql/parser.go +++ b/internal/dinosql/parser.go @@ -58,27 +58,26 @@ func (e *ParserErr) Error() string { return fmt.Sprintf("multiple errors: %d errors", len(e.Errs)) } -func ParseCatalog(schema string) (core.Catalog, error) { - f, err := os.Stat(schema) +func ReadSQLFiles(path string) ([]string, error) { + f, err := os.Stat(path) if err != nil { - return core.Catalog{}, fmt.Errorf("path %s does not exist", schema) + return nil, fmt.Errorf("path %s does not exist", path) } var files []string if f.IsDir() { - listing, err := ioutil.ReadDir(schema) + listing, err := ioutil.ReadDir(path) if err != nil { - return core.Catalog{}, err + return nil, err } for _, f := range listing { - files = append(files, filepath.Join(schema, f.Name())) + files = append(files, filepath.Join(path, f.Name())) } } else { - files = append(files, schema) + files = append(files, path) } - merr := NewParserErr() - c := core.NewCatalog() + var sql []string for _, filename := range files { if !strings.HasSuffix(filename, ".sql") { continue @@ -86,6 +85,20 @@ func ParseCatalog(schema string) (core.Catalog, error) { if strings.HasPrefix(filepath.Base(filename), ".") { continue } + sql = append(sql, filename) + } + return sql, nil +} + +func ParseCatalog(schema string) (core.Catalog, error) { + files, err := ReadSQLFiles(schema) + if err != nil { + return core.Catalog{}, err + } + + merr := NewParserErr() + c := core.NewCatalog() + for _, filename := range files { blob, err := ioutil.ReadFile(filename) if err != nil { merr.Add(filename, "", 0, err) diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 31d334abd5..3b3c577fb0 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -25,32 +25,38 @@ type Query struct { Filename string } -func parseFile(filename string, inPkg string, s *Schema, settings dinosql.GenerateSettings) (*Result, error) { - file, err := os.Open(filename) +func parsePath(sqlPath string, inPkg string, s *Schema, settings dinosql.GenerateSettings) (*Result, error) { + files, err := dinosql.ReadSQLFiles(sqlPath) if err != nil { - return nil, fmt.Errorf("Failed to open file [%v]: %v", filename, err) - } - contents, err := ioutil.ReadAll(file) - if err != nil { - return nil, fmt.Errorf("Failed to read contents of file [%v]: %v", filename, err) + return nil, err } - rawQueries := strings.Split(string(contents), "\n\n") parsedQueries := []*Query{} - - for _, query := range rawQueries { - if query == "" { - continue + for _, filename := range files { + file, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("Failed to open file [%v]: %v", filename, err) } - result, err := parseQueryString(query, s, settings) + contents, err := ioutil.ReadAll(file) if err != nil { - return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filename, err) + return nil, fmt.Errorf("Failed to read contents of file [%v]: %v", filename, err) } - if result == nil { - continue + rawQueries := strings.Split(string(contents), "\n\n") + for _, query := range rawQueries { + fmt.Println(query) + if query == "" { + continue + } + result, err := parseQueryString(query, s, settings) + if err != nil { + return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filename, err) + } + if result == nil { + continue + } + result.Filename = filepath.Base(filename) + parsedQueries = append(parsedQueries, result) } - result.Filename = filepath.Base(filename) - parsedQueries = append(parsedQueries, result) } return &Result{ @@ -434,9 +440,13 @@ func parseSelectAliasExpr(exprs sqlparser.SelectExprs, s *Schema, tableAliasMap } // GeneratePkg is the main entry to mysql generator package -func GeneratePkg(pkgName string, querysPath string, settings dinosql.GenerateSettings) (*Result, error) { +func GeneratePkg(pkgName, schemaPath, querysPath string, settings dinosql.GenerateSettings) (*Result, error) { s := NewSchema() - result, err := parseFile(querysPath, pkgName, s, settings) + _, err := parsePath(schemaPath, pkgName, s, settings) + if err != nil { + return nil, err + } + result, err := parsePath(querysPath, pkgName, s, settings) if err != nil { return nil, err } diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go index 555c275e9b..efa187ea0e 100644 --- a/internal/mysql/parse_test.go +++ b/internal/mysql/parse_test.go @@ -41,7 +41,7 @@ func TestParseConfig(t *testing.T) { } func TestGeneratePkg(t *testing.T) { - _, err := GeneratePkg(mockSettings.Packages[0].Name, filename, mockSettings) + _, err := GeneratePkg(mockSettings.Packages[0].Name, filename, filename, mockSettings) if err != nil { t.Fatal(err) } From 1381ab3b4839bee19afb3040b09a05a4bc1d86bd Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 6 Jan 2020 16:18:51 -0800 Subject: [PATCH 21/24] internal/mysql: Parse SQL via sqlparser.Tokenizer --- internal/mysql/gen.go | 8 +- internal/mysql/param.go | 22 ++-- internal/mysql/param_test.go | 52 +++++----- internal/mysql/parse.go | 92 +++++++++-------- internal/mysql/parse_test.go | 147 +++++++++++++++------------ internal/mysql/test_data/queries.sql | 8 +- 6 files changed, 180 insertions(+), 149 deletions(-) diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 794e86aeea..55fe9ef79f 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -113,16 +113,16 @@ func (r *Result) GoQueries(settings dinosql.GenerateSettings) []dinosql.GoQuery if len(query.Params) == 1 { p := query.Params[0] gq.Arg = dinosql.GoQueryValue{ - Name: p.name, - Typ: p.typ, + Name: p.Name, + Typ: p.Typ, } } else if len(query.Params) > 1 { structInfo := make([]structParams, len(query.Params)) for i := range query.Params { structInfo[i] = structParams{ - originalName: query.Params[i].name, - goType: query.Params[i].typ, + originalName: query.Params[i].Name, + goType: query.Params[i].Typ, } } diff --git a/internal/mysql/param.go b/internal/mysql/param.go index 283221b627..021f074c48 100644 --- a/internal/mysql/param.go +++ b/internal/mysql/param.go @@ -12,9 +12,9 @@ import ( // Param describes a runtime query parameter with its // associated type. Example: "SELECT name FROM users id = ?" type Param struct { - originalName string - name string - typ string + OriginalName string + Name string + Typ string } func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, tableAliasMap FromTables, settings dinosql.GenerateSettings) ([]*Param, error) { @@ -28,9 +28,9 @@ func paramsInLimitExpr(limit *sqlparser.Limit, s *Schema, tableAliasMap FromTabl case *sqlparser.SQLVal: if v.Type == sqlparser.ValArg { params = append(params, &Param{ - originalName: string(v.Val), - name: "limit", - typ: "uint32", + OriginalName: string(v.Val), + Name: "limit", + Typ: "uint32", }) } } @@ -100,12 +100,12 @@ func paramInComparison(cond *sqlparser.ComparisonExpr, s *Schema, tableAliasMap if err != nil { return false, err } - p.typ = goTypeCol(colDfn, settings) + p.Typ = goTypeCol(colDfn, settings) colIdent = colDfn.Name case *sqlparser.SQLVal: if v.Type == sqlparser.ValArg { - p.originalName = string(v.Val) + p.OriginalName = string(v.Val) } } return true, nil @@ -114,8 +114,8 @@ func paramInComparison(cond *sqlparser.ComparisonExpr, s *Schema, tableAliasMap if err != nil { return nil, false, err } - if p.originalName != "" && p.typ != "" { - p.name = paramName(colIdent, p.originalName) + if p.OriginalName != "" && p.Typ != "" { + p.Name = paramName(colIdent, p.OriginalName) return p, true, nil } return nil, false, nil @@ -135,7 +135,7 @@ func paramName(col sqlparser.ColIdent, originalName string) string { func replaceParamStrs(query string, params []*Param) (string, error) { for _, p := range params { - re, err := regexp.Compile(fmt.Sprintf("(%v)", p.originalName)) + re, err := regexp.Compile(fmt.Sprintf("(%v)", p.OriginalName)) if err != nil { return "", err } diff --git a/internal/mysql/param_test.go b/internal/mysql/param_test.go index a7dccaf6d3..40c7268c18 100644 --- a/internal/mysql/param_test.go +++ b/internal/mysql/param_test.go @@ -18,9 +18,9 @@ func TestSelectParamSearcher(t *testing.T) { testCase{ input: "SELECT first_name, id, last_name FROM users WHERE id < ?", output: []*Param{&Param{ - originalName: ":v1", - name: "id", - typ: "int", + OriginalName: ":v1", + Name: "id", + Typ: "int", }, }, }, @@ -35,18 +35,18 @@ func TestSelectParamSearcher(t *testing.T) { WHERE orders.price > :minPrice`, output: []*Param{ &Param{ - originalName: ":minPrice", - name: "minPrice", - typ: "float64", + OriginalName: ":minPrice", + Name: "minPrice", + Typ: "float64", }, }, }, testCase{ input: "SELECT first_name, id, last_name FROM users WHERE id = :targetID", output: []*Param{&Param{ - originalName: ":targetID", - name: "targetID", - typ: "int", + OriginalName: ":targetID", + Name: "targetID", + Typ: "int", }, }, }, @@ -54,14 +54,14 @@ func TestSelectParamSearcher(t *testing.T) { input: "SELECT first_name, last_name FROM users WHERE age < :maxAge AND last_name = :inFamily", output: []*Param{ &Param{ - originalName: ":maxAge", - name: "maxAge", - typ: "int", + OriginalName: ":maxAge", + Name: "maxAge", + Typ: "int", }, &Param{ - originalName: ":inFamily", - name: "inFamily", - typ: "sql.NullString", + OriginalName: ":inFamily", + Name: "inFamily", + Typ: "sql.NullString", }, }, }, @@ -69,9 +69,9 @@ func TestSelectParamSearcher(t *testing.T) { input: "SELECT first_name, last_name FROM users LIMIT ?", output: []*Param{ &Param{ - originalName: ":v1", - name: "limit", - typ: "uint32", + OriginalName: ":v1", + Name: "limit", + Typ: "uint32", }, }, }, @@ -121,14 +121,14 @@ func TestInsertParamSearcher(t *testing.T) { input: "INSERT INTO users (first_name, last_name) VALUES (?, ?)", output: []*Param{ &Param{ - originalName: ":v1", - name: "first_name", - typ: "string", + OriginalName: ":v1", + Name: "first_name", + Typ: "string", }, &Param{ - originalName: ":v2", - name: "last_name", - typ: "sql.NullString", + OriginalName: ":v2", + Name: "last_name", + Typ: "sql.NullString", }, }, expectedNames: []string{"first_name", "last_name"}, @@ -156,9 +156,9 @@ func TestInsertParamSearcher(t *testing.T) { t.Errorf("Insufficient test cases. Mismatch in length of expected param names and parsed params") } for ix, p := range result.Params { - if p.name != tCase.expectedNames[ix] { + if p.Name != tCase.expectedNames[ix] { t.Errorf("Derived param does not match expected output.\nResult: %v\nExpected: %v", - p.name, tCase.expectedNames[ix]) + p.Name, tCase.expectedNames[ix]) } } } diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 3b3c577fb0..d7b713995d 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -2,6 +2,7 @@ package mysql import ( "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -19,8 +20,8 @@ type Query struct { Params []*Param // "?" params in the query string Name string // the Go function name Cmd string // TODO: Pick a better name. One of: one, many, exec, execrows - defaultTableName string // for columns that are not qualified - schemaLookup *Schema // for validation and conversion to Go types + DefaultTableName string // for columns that are not qualified + SchemaLookup *Schema // for validation and conversion to Go types Filename string } @@ -41,22 +42,11 @@ func parsePath(sqlPath string, inPkg string, s *Schema, settings dinosql.Generat if err != nil { return nil, fmt.Errorf("Failed to read contents of file [%v]: %v", filename, err) } - rawQueries := strings.Split(string(contents), "\n\n") - for _, query := range rawQueries { - fmt.Println(query) - if query == "" { - continue - } - result, err := parseQueryString(query, s, settings) - if err != nil { - return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filename, err) - } - if result == nil { - continue - } - result.Filename = filepath.Base(filename) - parsedQueries = append(parsedQueries, result) + queries, err := parseContents(filename, string(contents), s, settings) + if err != nil { + return nil, err } + parsedQueries = append(parsedQueries, queries...) } return &Result{ @@ -66,12 +56,33 @@ func parsePath(sqlPath string, inPkg string, s *Schema, settings dinosql.Generat }, nil } -func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { - tree, err := sqlparser.Parse(query) - - if err != nil { - return nil, err +func parseContents(filename, contents string, s *Schema, settings dinosql.GenerateSettings) ([]*Query, error) { + t := sqlparser.NewStringTokenizer(contents) + var queries []*Query + var start int + for { + q, err := sqlparser.ParseNext(t) + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + query := contents[start : t.Position-1] + result, err := parseQueryString(q, query, s, settings) + if err != nil { + return nil, fmt.Errorf("Failed to parse query in filepath [%v]: %v", filename, err) + } + start = t.Position + if result == nil { + continue + } + result.Filename = filepath.Base(filename) + queries = append(queries, result) } + return queries, nil +} + +func parseQueryString(tree sqlparser.Statement, query string, s *Schema, settings dinosql.GenerateSettings) (*Query, error) { var parsedQuery *Query switch tree := tree.(type) { case *sqlparser.Select: @@ -94,7 +105,7 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings parsedQuery = update case *sqlparser.Delete: delete, err := parseDelete(tree, query, s, settings) - delete.schemaLookup = nil + delete.SchemaLookup = nil if err != nil { return nil, fmt.Errorf("Failed to parse DELETE query: %v", err) } @@ -103,7 +114,8 @@ func parseQueryString(query string, s *Schema, settings dinosql.GenerateSettings s.Add(tree) return nil, nil default: - panic("Unsupported SQL statement type") + // panic("Unsupported SQL statement type") + return nil, nil } paramsReplacedQuery, err := replaceParamStrs(sqlparser.String(tree), parsedQuery.Params) if err != nil { @@ -149,8 +161,8 @@ func parseSelect(tree *sqlparser.Select, query string, s *Schema, settings dinos parsedQuery := Query{ SQL: query, - defaultTableName: defaultTableName, - schemaLookup: s, + DefaultTableName: defaultTableName, + SchemaLookup: s, } cols, err := parseSelectAliasExpr(tree.SelectExprs, s, tableAliasMap, defaultTableName) if err != nil { @@ -260,9 +272,9 @@ func parseUpdate(node *sqlparser.Update, query string, s *Schema, settings dinos } originalParamName := string(newValue.Val) param := Param{ - originalName: originalParamName, - name: paramName(colDfn.Name, originalParamName), - typ: goTypeCol(colDfn, settings), + OriginalName: originalParamName, + Name: paramName(colDfn.Name, originalParamName), + Typ: goTypeCol(colDfn, settings), } params = append(params, ¶m) } @@ -276,8 +288,8 @@ func parseUpdate(node *sqlparser.Update, query string, s *Schema, settings dinos SQL: query, Columns: nil, Params: append(params, whereParams...), - defaultTableName: defaultTable, - schemaLookup: s, + DefaultTableName: defaultTable, + SchemaLookup: s, } parsedQuery.parseNameAndCmd() @@ -303,9 +315,9 @@ func parseInsert(node *sqlparser.Insert, query string, s *Schema, settings dinos colDfn, _ := s.schemaLookup(tableName, colName) varName := string(v.Val) p := &Param{ - originalName: varName, - name: paramName(colDfn.Name, varName), - typ: goTypeCol(colDfn, settings), + OriginalName: varName, + Name: paramName(colDfn.Name, varName), + Typ: goTypeCol(colDfn, settings), } params = append(params, p) } @@ -318,8 +330,8 @@ func parseInsert(node *sqlparser.Insert, query string, s *Schema, settings dinos SQL: query, Params: params, Columns: nil, - defaultTableName: tableName, - schemaLookup: s, + DefaultTableName: tableName, + SchemaLookup: s, } parsedQuery.parseNameAndCmd() return parsedQuery, nil @@ -348,8 +360,8 @@ func parseDelete(node *sqlparser.Delete, query string, s *Schema, settings dinos SQL: query, Params: append(whereParams, limitParams...), Columns: nil, - defaultTableName: defaultTableName, - schemaLookup: s, + DefaultTableName: defaultTableName, + SchemaLookup: s, } err = parsedQuery.parseNameAndCmd() if err != nil { @@ -444,11 +456,11 @@ func GeneratePkg(pkgName, schemaPath, querysPath string, settings dinosql.Genera s := NewSchema() _, err := parsePath(schemaPath, pkgName, s, settings) if err != nil { - return nil, err + return nil, fmt.Errorf("schema failure: %w", err) } result, err := parsePath(querysPath, pkgName, s, settings) if err != nil { - return nil, err + return nil, fmt.Errorf("query failure: %w", err) } return result, nil } diff --git a/internal/mysql/parse_test.go b/internal/mysql/parse_test.go index efa187ea0e..0a1b39ca7c 100644 --- a/internal/mysql/parse_test.go +++ b/internal/mysql/parse_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" "github.com/kyleconroy/sqlc/internal/dinosql" "vitess.io/vitess/go/vt/sqlparser" ) @@ -130,11 +131,13 @@ func TestParseSelect(t *testing.T) { schema *Schema } type testCase struct { + name string input expected output *Query } tests := []testCase{ testCase{ + name: "get_count", input: expected{ query: `/* name: GetCount :one */ SELECT id my_id, COUNT(id) id_count FROM users WHERE id > 4`, @@ -162,11 +165,12 @@ func TestParseSelect(t *testing.T) { Params: []*Param{}, Name: "GetCount", Cmd: ":one", - defaultTableName: "users", - schemaLookup: mockSchema, + DefaultTableName: "users", + SchemaLookup: mockSchema, }, }, testCase{ + name: "get_name_by_id", input: expected{ query: `/* name: GetNameByID :one */ SELECT first_name, last_name FROM users WHERE id = ?`, @@ -177,17 +181,18 @@ func TestParseSelect(t *testing.T) { Columns: filterCols(mockSchema.tables["users"], map[string]struct{}{"first_name": struct{}{}, "last_name": struct{}{}}), Params: []*Param{ &Param{ - originalName: ":v1", - name: "id", - typ: "int", + OriginalName: ":v1", + Name: "id", + Typ: "int", }}, Name: "GetNameByID", Cmd: ":one", - defaultTableName: "users", - schemaLookup: mockSchema, + DefaultTableName: "users", + SchemaLookup: mockSchema, }, }, testCase{ + name: "get_all", input: expected{ query: `/* name: GetAll :many */ SELECT * FROM users;`, @@ -199,11 +204,12 @@ func TestParseSelect(t *testing.T) { Params: []*Param{}, Name: "GetAll", Cmd: ":many", - defaultTableName: "users", - schemaLookup: mockSchema, + DefaultTableName: "users", + SchemaLookup: mockSchema, }, }, testCase{ + name: "get_all_users_orders", input: expected{ query: `/* name: GetAllUsersOrders :many */ SELECT u.id user_id, u.first_name, o.price, o.id order_id @@ -248,26 +254,30 @@ func TestParseSelect(t *testing.T) { Params: []*Param{}, Name: "GetAllUsersOrders", Cmd: ":many", - defaultTableName: "", // TODO: verify that this is desired behaviour - schemaLookup: mockSchema, + DefaultTableName: "", // TODO: verify that this is desired behaviour + SchemaLookup: mockSchema, }, }, } - for ix, testCase := range tests { - q, err := parseQueryString(testCase.input.query, testCase.input.schema, mockSettings) - if err != nil { - t.Errorf("Parsing failed with query: [%v]\n%v", testCase.input.query, err) - } - - err = q.parseNameAndCmd() - if err != nil { - t.Errorf("Parsing failed with query: [%v]\n%v", testCase.input.query, err) - } - if !reflect.DeepEqual(testCase.output, q) { - t.Errorf("Parsing query returned differently than expected. Index: %v", ix) - t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) - } + for _, tt := range tests { + testCase := tt + t.Run(tt.name, func(t *testing.T) { + qs, err := parseContents("example.sql", testCase.input.query, testCase.input.schema, mockSettings) + if err != nil { + t.Fatalf("Parsing failed with query: [%v]\n", err) + } + if len(qs) != 1 { + t.Fatalf("Expected one query, not %d", len(qs)) + } + q := qs[0] + q.SchemaLookup = nil + q.Filename = "" + testCase.output.SchemaLookup = nil + if diff := cmp.Diff(testCase.output, q); diff != "" { + t.Errorf("parsed query differs: \n%s", diff) + } + }) } } @@ -320,13 +330,18 @@ func TestParseInsertUpdate(t *testing.T) { schema *Schema } type testCase struct { + name string input expected output *Query } - query1 := "/* name: InsertNewUser :exec */\nINSERT INTO users (first_name, last_name) VALUES (?, ?)" - query2 := "/* name: UpdateUserAt :exec */\nUPDATE users SET first_name = ?, last_name = ? WHERE id > ? AND first_name = ? LIMIT 3" + query1 := `/* name: InsertNewUser :exec */ +INSERT INTO users (first_name, last_name) VALUES (?, ?)` + query2 := `/* name: UpdateUserAt :exec */ +UPDATE users SET first_name = ?, last_name = ? WHERE id > ? AND first_name = ? LIMIT 3` + tests := []testCase{ testCase{ + name: "insert_users", input: expected{ query: query1, schema: mockSchema, @@ -336,23 +351,24 @@ func TestParseInsertUpdate(t *testing.T) { Columns: nil, Params: []*Param{ &Param{ - originalName: ":v1", - name: "first_name", - typ: "string", + OriginalName: ":v1", + Name: "first_name", + Typ: "string", }, &Param{ - originalName: ":v2", - name: "last_name", - typ: "sql.NullString", + OriginalName: ":v2", + Name: "last_name", + Typ: "sql.NullString", }, }, Name: "InsertNewUser", Cmd: ":exec", - defaultTableName: "users", - schemaLookup: mockSchema, + DefaultTableName: "users", + SchemaLookup: mockSchema, }, }, testCase{ + name: "update_users", input: expected{ query: query2, schema: mockSchema, @@ -362,48 +378,51 @@ func TestParseInsertUpdate(t *testing.T) { Columns: nil, Params: []*Param{ &Param{ - originalName: ":v1", - name: "first_name", - typ: "string", + OriginalName: ":v1", + Name: "first_name", + Typ: "string", }, &Param{ - originalName: ":v2", - name: "last_name", - typ: "sql.NullString", + OriginalName: ":v2", + Name: "last_name", + Typ: "sql.NullString", }, &Param{ - originalName: ":v3", - name: "id", - typ: "int", + OriginalName: ":v3", + Name: "id", + Typ: "int", }, &Param{ - originalName: ":v4", - name: "first_name", - typ: "string", + OriginalName: ":v4", + Name: "first_name", + Typ: "string", }, }, Name: "UpdateUserAt", Cmd: ":exec", - defaultTableName: "users", - schemaLookup: mockSchema, + DefaultTableName: "users", + SchemaLookup: mockSchema, }, }, } - for ix, testCase := range tests { - q, err := parseQueryString(testCase.input.query, testCase.input.schema, mockSettings) - if err != nil { - t.Errorf("Parsing failed with query: [%v]\n", err) - continue - } - - err = q.parseNameAndCmd() - if err != nil { - t.Errorf("Parsing failed with query index: %d: [%v]\n", ix, testCase.input.query) - } - if !reflect.DeepEqual(testCase.output, q) { - t.Errorf("Parsing query returned differently than expected.") - t.Logf("Expected: %v\nResult: %v\n", spew.Sdump(testCase.output), spew.Sdump(q)) - } + for _, tt := range tests { + testCase := tt + t.Run(tt.name, func(t *testing.T) { + qs, err := parseContents("example.sql", testCase.input.query, testCase.input.schema, mockSettings) + if err != nil { + t.Fatalf("Parsing failed with query: [%v]\n", err) + } + if len(qs) != 1 { + t.Fatalf("Expected one query, not %d", len(qs)) + } + q := qs[0] + testCase.output.SchemaLookup = nil + q.SchemaLookup = nil + q.Filename = "" + if diff := cmp.Diff(testCase.output, q); diff != "" { + t.Errorf("parsed query differs: \n%s", diff) + } + }) } } diff --git a/internal/mysql/test_data/queries.sql b/internal/mysql/test_data/queries.sql index fecba7823e..9342dec835 100644 --- a/internal/mysql/test_data/queries.sql +++ b/internal/mysql/test_data/queries.sql @@ -8,10 +8,10 @@ CREATE TABLE teachers ( ); /* name: GetTeachersByID :one */ -SELECT * FROM teachers WHERE id = ? +SELECT * FROM teachers WHERE id = ?; -/* name: GetSomeTeachers :one */ +-- name: GetSomeTeachers :one SELECT school_id, id FROM teachers WHERE school_lng > ? AND school_lat < ?; -/* name: TeachersByID :one */ -SELECT id, school_lat FROM teachers WHERE id = ? LIMIT 10 \ No newline at end of file +-- name: TeachersByID :one +SELECT id, school_lat FROM teachers WHERE id = ? LIMIT 10; From e6d12d5d2d5abbd08ebb3d79310615a597861e67 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 6 Jan 2020 16:38:32 -0800 Subject: [PATCH 22/24] internal/mysql: Handle more DDL types --- internal/dinosql/gen.go | 2 +- internal/mysql/gen.go | 9 ++++----- internal/mysql/parse.go | 2 +- internal/mysql/schema.go | 13 ++++++++----- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/dinosql/gen.go b/internal/dinosql/gen.go index 6d27de1338..71f2f40b1e 100644 --- a/internal/dinosql/gen.go +++ b/internal/dinosql/gen.go @@ -654,7 +654,7 @@ func (r Result) goInnerType(col core.Column, settings GenerateSettings) string { } } } - log.Printf("unknown Postgres type: %s\n", columnType) + log.Printf("unknown PostgreSQL type: %s\n", columnType) return "interface{}" } } diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 55fe9ef79f..9533287c09 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -2,10 +2,10 @@ package mysql import ( "fmt" + "log" "sort" "strings" - "github.com/davecgh/go-spew/spew" "github.com/jinzhu/inflection" "github.com/kyleconroy/sqlc/internal/dinosql" "vitess.io/vitess/go/vt/sqlparser" @@ -224,7 +224,7 @@ func goTypeCol(col *sqlparser.ColumnDefinition, settings dinosql.GenerateSetting return "string" } return "sql.NullString" - case "int" == t: + case "int" == t, "integer" == t: if col.Type.NotNull { return "int" } @@ -241,9 +241,8 @@ func goTypeCol(col *sqlparser.ColumnDefinition, settings dinosql.GenerateSetting case "boolean" == t: return "bool" default: - // TODO: remove panic here - panic(fmt.Sprintf("Handle this col type directly: %v\n", spew.Sdump(col.Type))) - // return col.Type + log.Printf("unknown MySQL type: %s\n", t) + return "interface{}" } } diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index d7b713995d..6495c76201 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -61,7 +61,7 @@ func parseContents(filename, contents string, s *Schema, settings dinosql.Genera var queries []*Query var start int for { - q, err := sqlparser.ParseNext(t) + q, err := sqlparser.ParseNextStrictDDL(t) if err == io.EOF { break } else if err != nil { diff --git a/internal/mysql/schema.go b/internal/mysql/schema.go index 12900d4cb0..f0f2cf4850 100644 --- a/internal/mysql/schema.go +++ b/internal/mysql/schema.go @@ -49,12 +49,15 @@ func (s *Schema) getColType(col *sqlparser.ColName, tableAliasMap FromTables, de } // Add add a MySQL table definition to the schema map -func (s *Schema) Add(table *sqlparser.DDL) { - name := table.Table.Name.String() - if table.TableSpec == nil { - panic(fmt.Sprintf("Failed to parse table [%v] schema.", name)) +func (s *Schema) Add(ddl *sqlparser.DDL) { + switch ddl.Action { + case "create": + name := ddl.Table.Name.String() + if ddl.TableSpec == nil { + panic(fmt.Sprintf("Failed to parse table [%v] schema.", name)) + } + s.tables[name] = ddl.TableSpec.Columns } - s.tables[name] = table.TableSpec.Columns } func (s *Schema) schemaLookup(table string, col string) (*sqlparser.ColumnDefinition, error) { From 5d740dfe3858b149faf83e11fa459f2b93d3537d Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Mon, 6 Jan 2020 17:05:41 -0800 Subject: [PATCH 23/24] Fix some more one-off errors --- internal/mysql/gen.go | 7 +++++-- internal/mysql/parse.go | 13 ++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/mysql/gen.go b/internal/mysql/gen.go index 9533287c09..9d6d24e7d9 100644 --- a/internal/mysql/gen.go +++ b/internal/mysql/gen.go @@ -236,8 +236,11 @@ func goTypeCol(col *sqlparser.ColumnDefinition, settings dinosql.GenerateSetting return "sql.NullFloat64" case "enum" == t: return enumNameFromColDef(col, settings) - case "date" == t, "timestamp" == t: - return "time.Time" + case "date" == t, "timestamp" == t, "datetime" == t: + if col.Type.NotNull { + return "time.Time" + } + return "sql.NullTime" case "boolean" == t: return "bool" default: diff --git a/internal/mysql/parse.go b/internal/mysql/parse.go index 6495c76201..0a33ed6915 100644 --- a/internal/mysql/parse.go +++ b/internal/mysql/parse.go @@ -312,12 +312,15 @@ func parseInsert(node *sqlparser.Insert, query string, s *Schema, settings dinos case *sqlparser.SQLVal: if v.Type == sqlparser.ValArg { colName := cols[colIx].String() - colDfn, _ := s.schemaLookup(tableName, colName) + colDfn, err := s.schemaLookup(tableName, colName) varName := string(v.Val) - p := &Param{ - OriginalName: varName, - Name: paramName(colDfn.Name, varName), - Typ: goTypeCol(colDfn, settings), + p := &Param{OriginalName: varName} + if err == nil { + p.Name = paramName(colDfn.Name, varName) + p.Typ = goTypeCol(colDfn, settings) + } else { + p.Name = "Unknown" + p.Typ = "interface{}" } params = append(params, p) } From 930afe5144d62c2380c22ad17d6228bea95157af Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Tue, 7 Jan 2020 13:50:16 -0800 Subject: [PATCH 24/24] /s/database/engine/g --- README.md | 14 ++++++++++---- internal/cmd/cmd.go | 20 +++++--------------- internal/dinosql/config.go | 23 ++++++++++++++++------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1b14759bda..8506fbaa5f 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,8 @@ Each package document has the following keys: - Directory of SQL queries or path to single SQL file - `schema`: - Directory of SQL migrations or path to single SQL file +- `engine`: + - Either `postgresql` or `mysql`. Defaults to `postgresql`. MySQL support is experimental ### Type Overrides @@ -411,13 +413,17 @@ Each commit is deployed to the [`devel` channel on Equinox](https://dl.equinox.i - [Linux](https://bin.equinox.io/c/gvM95th6ps1/sqlc-devel-linux-amd64.tgz) - [macOS](https://bin.equinox.io/c/gvM95th6ps1/sqlc-devel-darwin-amd64.zip) -## Other Database Engines +## Other Databases and Languages -sqlc currently only supports PostgreSQL. If you'd like to support another database, we'd welcome a contribution. +sqlc currently only supports PostgreSQL / Go. MySQL support has been merged, +but it's marked as experimental. SQLite and TypeScript support are planned. -## Other Language Backends +| Language | PostgreSQL | MySQL | SQLite | +| ------------ |:----------------:|:----------------:|:----------------:| +| Go |:white_check_mark:|:warning: |:timer_clock: | +| TypeScript |:timer_clock: |:timer_clock: |:timer_clock: | -sqlc currently only generates Go code, but if you'd like to build another language backend, we'd welcome a contribution. +If you'd like to add another database or language, we'd welcome a contribution. ## Acknowledgements diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 2f2205499f..361c1484e3 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -127,26 +127,16 @@ var genCmd = &cobra.Command{ output := map[string]string{} - for i, pkg := range settings.Packages { + for _, pkg := range settings.Packages { name := pkg.Name - if pkg.Path == "" { - fmt.Fprintf(os.Stderr, "package[%d]: path must be set\n", i) - errored = true - continue - } - - if name == "" { - name = filepath.Base(pkg.Path) - } - var result dinosql.Generateable - switch pkg.Database { + switch pkg.Engine { - case dinosql.DatabaseMySQL: + case dinosql.EngineMySQL: // Experimental MySQL support - q, err := mysql.GeneratePkg(pkg.Name, pkg.Schema, pkg.Queries, settings) + q, err := mysql.GeneratePkg(name, pkg.Schema, pkg.Queries, settings) if err != nil { fmt.Fprintf(os.Stderr, "# package %s\n", name) fmt.Fprintf(os.Stderr, "error parsing file: %s\n", err) @@ -155,7 +145,7 @@ var genCmd = &cobra.Command{ } result = q - case dinosql.DatabasePostgreSQL: + case dinosql.EnginePostgreSQL: c, err := dinosql.ParseCatalog(pkg.Schema) if err != nil { fmt.Fprintf(os.Stderr, "# package %s\n", name) diff --git a/internal/dinosql/config.go b/internal/dinosql/config.go index 3d388777cc..999b7ab8f9 100644 --- a/internal/dinosql/config.go +++ b/internal/dinosql/config.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "path/filepath" "strings" "github.com/kyleconroy/sqlc/internal/pg" @@ -33,16 +34,16 @@ type GenerateSettings struct { PackageMap map[string]PackageSettings } -type Database string +type Engine string const ( - DatabaseMySQL Database = "mysql" - DatabasePostgreSQL Database = "postgresql" + EngineMySQL Engine = "mysql" + EnginePostgreSQL Engine = "postgresql" ) type PackageSettings struct { Name string `json:"name"` - Database Database `json:"database,omitempty"` + Engine Engine `json:"engine,omitempty"` Path string `json:"path"` Schema string `json:"schema"` Queries string `json:"queries"` @@ -120,6 +121,8 @@ func (o *Override) Parse() error { var ErrMissingVersion = errors.New("no version number") var ErrUnknownVersion = errors.New("invalid version number") var ErrNoPackages = errors.New("no packages") +var ErrNoPackageName = errors.New("missing package name") +var ErrNoPackagePath = errors.New("missing package path") func ParseConfig(rd io.Reader) (GenerateSettings, error) { dec := json.NewDecoder(rd) @@ -143,13 +146,19 @@ func ParseConfig(rd io.Reader) (GenerateSettings, error) { } } for j := range config.Packages { + if config.Packages[j].Path == "" { + return config, ErrNoPackagePath + } for i := range config.Packages[j].Overrides { if err := config.Packages[j].Overrides[i].Parse(); err != nil { return config, err } } - if config.Packages[j].Database == "" { - config.Packages[j].Database = DatabasePostgreSQL + if config.Packages[j].Name == "" { + config.Packages[j].Name = filepath.Base(config.Packages[j].Path) + } + if config.Packages[j].Engine == "" { + config.Packages[j].Engine = EnginePostgreSQL } } err := config.PopulatePkgMap() @@ -162,7 +171,7 @@ func (s *GenerateSettings) PopulatePkgMap() error { for _, c := range s.Packages { if c.Name == "" { - panic("Package name must be specified in sqlc.json") + return ErrNoPackageName } packageMap[c.Name] = c }