feat(analyzer): Analyze queries using a running PostgreSQL database

94 of the open issues on sqlc are related to the analyzer. There are
many cases where the current analyzer produces false positives or
false negatives.

sqlx and some other projects have proven that it's possible to extract
query metadata from a running database.

This approach is a bit different, in that the database analysis is
layered on top of the existing query analyzer. We use the new analysis
to provide better type information and support a wider set of cases when
the existing analyzer fails.

go.mod

@@ -29,6 +29,7 @@ require (
 
 require (
  github.com/benbjohnson/clock v1.3.5 // indirect
+ github.com/jackc/puddle/v2 v2.2.1 // indirect
  github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
  github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
  github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect

go.sum

@@ -100,7 +100,10 @@ github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSlj
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

internal/analyzer/analyzer.go

@@ -0,0 +1,46 @@
+package analyzer
+
+import (
+ "context"
+
+ "github.com/sqlc-dev/sqlc/internal/sql/ast"
+ "github.com/sqlc-dev/sqlc/internal/sql/named"
+)
+
+type Column struct {
+ Name string
+ OriginalName string
+ DataType string
+ NotNull bool
+ Unsigned bool
+ IsArray bool
+ ArrayDims int
+ Comment string
+ Length *int
+ IsNamedParam bool
+ IsFuncCall bool
+
+ // XXX: Figure out what PostgreSQL calls `foo.id`
+ Scope string
+ Table *ast.TableName
+ TableAlias string
+ Type *ast.TypeName
+ EmbedTable *ast.TableName
+
+ IsSqlcSlice bool // is this sqlc.slice()
+}
+
+type Parameter struct {
+ Number int
+ Column *Column
+}
+
+type Analysis struct {
+ Columns []Column
+ Params []Parameter
+}
+
+type Analyzer interface {
+ Analyze(context.Context, ast.Node, string, []string, *named.ParamSet) (*Analysis, error)
+ Close(context.Context) error
+}

internal/cmd/cmd.go

@@ -197,7 +197,10 @@ var genCmd = &cobra.Command{
  defer trace.StartRegion(cmd.Context(), "generate").End()
  stderr := cmd.ErrOrStderr()
  dir, name := getConfigPath(stderr, cmd.Flag("file"))
- output, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr)
+ output, err := Generate(cmd.Context(), dir, name, &Options{
+ Env: ParseEnv(cmd),
+ Stderr: stderr,
+ })
  if err != nil {
  os.Exit(1)
  }
@@ -219,7 +222,11 @@ var uploadCmd = &cobra.Command{
  RunE: func(cmd *cobra.Command, args []string) error {
  stderr := cmd.ErrOrStderr()
  dir, name := getConfigPath(stderr, cmd.Flag("file"))
- if err := createPkg(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+ opts := &Options{
+ Env: ParseEnv(cmd),
+ Stderr: stderr,
+ }
+ if err := createPkg(cmd.Context(), dir, name, opts); err != nil {
  fmt.Fprintf(stderr, "error uploading: %s\n", err)
  os.Exit(1)
  }
@@ -234,7 +241,11 @@ var checkCmd = &cobra.Command{
  defer trace.StartRegion(cmd.Context(), "compile").End()
  stderr := cmd.ErrOrStderr()
  dir, name := getConfigPath(stderr, cmd.Flag("file"))
- if _, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+ _, err := Generate(cmd.Context(), dir, name, &Options{
+ Env: ParseEnv(cmd),
+ Stderr: stderr,
+ })
+ if err != nil {
  os.Exit(1)
  }
  return nil
@@ -277,7 +288,11 @@ var diffCmd = &cobra.Command{
  defer trace.StartRegion(cmd.Context(), "diff").End()
  stderr := cmd.ErrOrStderr()
  dir, name := getConfigPath(stderr, cmd.Flag("file"))
- if err := Diff(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+ opts := &Options{
+ Env: ParseEnv(cmd),
+ Stderr: stderr,
+ }
+ if err := Diff(cmd.Context(), dir, name, opts); err != nil {
  os.Exit(1)
  }
  return nil

internal/cmd/diff.go

@@ -4,7 +4,6 @@ import (
  "context"
  "errors"
  "fmt"
- "io"
  "os"
  "runtime/trace"
  "sort"
@@ -13,8 +12,9 @@ import (
  "github.com/cubicdaiya/gonp"
 )
 
-func Diff(ctx context.Context, e Env, dir, name string, stderr io.Writer) error {
- output, err := Generate(ctx, e, dir, name, stderr)
+func Diff(ctx context.Context, dir, name string, opts *Options) error {
+ stderr := opts.Stderr
+ output, err := Generate(ctx, dir, name, opts)
  if err != nil {
  return err
  }

internal/cmd/generate.go

@@ -128,8 +128,11 @@ func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config,
  return configPath, &conf, nil
 }
 
-func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) {
- configPath, conf, err := readConfig(stderr, dir, filename)
+func Generate(ctx context.Context, dir, filename string, o *Options) (map[string]string, error) {
+ e := o.Env
+ stderr := o.Stderr
+
+ configPath, conf, err := o.ReadConfig(dir, filename)
  if err != nil {
  return nil, err
  }
@@ -332,7 +335,12 @@ func remoteGenerate(ctx context.Context, configPath string, conf *config.Config,
 
 func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
  defer trace.StartRegion(ctx, "parse").End()
- c := compiler.NewCompiler(sql, combo)
+ c, err := compiler.NewCompiler(sql, combo)
+ defer c.Close(ctx)
+ if err != nil {
+ fmt.Fprintf(stderr, "error creating compiler: %s\n", err)
+ return nil, true
+ }
  if err := c.ParseCatalog(sql.Schema); err != nil {
  fmt.Fprintf(stderr, "# package %s\n", name)
  if parserErr, ok := err.(*multierr.Error); ok {

internal/cmd/options.go

@@ -0,0 +1,24 @@
+package cmd
+
+import (
+ "io"
+
+ "github.com/sqlc-dev/sqlc/internal/config"
+)
+
+type Options struct {
+ Env Env
+ Stderr io.Writer
+ MutateConfig func(*config.Config)
+}
+
+func (o *Options) ReadConfig(dir, filename string) (string, *config.Config, error) {
+ path, conf, err := readConfig(o.Stderr, dir, filename)
+ if err != nil {
+ return path, conf, err
+ }
+ if o.MutateConfig != nil {
+ o.MutateConfig(conf)
+ }
+ return path, conf, nil
+}

internal/cmd/package.go

@@ -2,13 +2,14 @@ package cmd
 
 import (
  "context"
- "io"
  "os"
 
  "github.com/sqlc-dev/sqlc/internal/bundler"
 )
 
-func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
+func createPkg(ctx context.Context, dir, filename string, opts *Options) error {
+ e := opts.Env
+ stderr := opts.Stderr
  configPath, conf, err := readConfig(stderr, dir, filename)
  if err != nil {
  return err
@@ -17,7 +18,7 @@ func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Write
  if err := up.Validate(); err != nil {
  return err
  }
- output, err := Generate(ctx, e, dir, filename, stderr)
+ output, err := Generate(ctx, dir, filename, opts)
  if err != nil {
  os.Exit(1)
  }

internal/cmd/vet.go

@@ -47,8 +47,12 @@ func NewCmdVet() *cobra.Command {
  RunE: func(cmd *cobra.Command, args []string) error {
  defer trace.StartRegion(cmd.Context(), "vet").End()
  stderr := cmd.ErrOrStderr()
+ opts := &Options{
+ Env: ParseEnv(cmd),
+ Stderr: stderr,
+ }
  dir, name := getConfigPath(stderr, cmd.Flag("file"))
- if err := Vet(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+ if err := Vet(cmd.Context(), dir, name, opts); err != nil {
  if !errors.Is(err, ErrFailedChecks) {
  fmt.Fprintf(stderr, "%s\n", err)
  }
@@ -59,7 +63,9 @@ func NewCmdVet() *cobra.Command {
  }
 }
 
-func Vet(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
+func Vet(ctx context.Context, dir, filename string, opts *Options) error {
+ e := opts.Env
+ stderr := opts.Stderr
  configPath, conf, err := readConfig(stderr, dir, filename)
  if err != nil {
  return err

internal/codegen/golang/result.go

@@ -249,9 +249,7 @@ func buildQueries(req *plugin.CodeGenRequest, structs []Struct) ([]Query, error)
  if len(query.Columns) == 1 && query.Columns[0].EmbedTable == nil {
  c := query.Columns[0]
  name := columnName(c, 0)
- if c.IsFuncCall {
- name = strings.Replace(name, "$", "_", -1)
- }
+ name = strings.Replace(name, "$", "_", -1)
  gq.Ret = QueryValue{
  Name: name,
  DBName: name,