feat(typegen): retain intersection types

packages/typegen/readme.md

@@ -26,11 +26,11 @@ Select statements, joins, and updates/inserts/deletes using `returning` are all
 - [Usage](#usage)
 - [Configuration](#configuration)
    - [Example config](#example-config)
-   - [CLI options](#cli-options)
-   - [writeTypes](#writetypes)
+   - [Advanced Configuration](#writetypes)
       - [Controlling write destination](#controlling-write-destination)
       - [Modifying types](#modifying-types)
       - [Modifying source files](#modifying-source-files)
+- [Enhancing Return Types](#enhancing-return-types)
 - [Examples](#examples)
 - [Migration from v0.8.0](#migration-from-v080)
 - [SQL files](#sql-files)
@@ -117,17 +117,44 @@ export declare namespace queries {
 
 ## Configuration
 
-The CLI can run with zero config, but there will usually be customisations needed depending on your project's setup. By default, the CLI will look for `typegen.config.js` file in the working directory. The config file can contain the following options (all are optional):
+The CLI can run with zero config, but there will usually be customisations needed depending on your project's setup.  
+By default, the CLI will look for `typegen.config.js` file in the working directory, exporting an object containing the properties below.
+
+Some options are only available via CLI, some are only available in the config.  
+CLI arguments will always have precedence over config options.
+
+|Option|CLI Argument            |Type|Default|Description|
+|-|-|-|-|-|
+|`rootDir`|`--root-dir`|`string`|`'src'`|Source root that the tool will search for files in.|
+|`include`|`--include`|`string[]`|`['**/*.{ts,sql}']`|Glob patterns for files to include in processing. Repeatable in CLI.|
+|`exclude`|`--exclude`|`string[]`|`['**/node_modules/**']`|Glob patterns for files to exclude from processing. Repeatable in CLI.|
+|`since`|`--since`|`string \| undefined`|`undefined`|Limit matched files to those which have been changed since the given git ref. Use `"HEAD"` for files changed since the last commit, `"main"` for files changed in a branch, etc.|
+|`connectionURI`|`--connection-uri`|`string`|`'postgresql://` `postgres:postgres` `@localhost:5432/` `postgres'`|URI for connecting to psql. Note that if you are using `psql` inside docker, you should make sure that the container and host port match, since this will be used both by `psql` and slonik to connect to the database.|
+|`psqlCommand`|`--psql`|`string`|`'psql'`|The CLI command for running the official postgres `psql` CLI client.<br/>Note that right now this can't contain single quotes. This should also be configured to talk to the same database as the `pool` variable (and it should be a development database - don't run this tool in production!). If you are using docker compose, you can use a command like `docker-compose exec -T postgres psql`|
+|`defaultType`|`--default-type`|`string`|`'unknown'`|TypeScript type when no mapping is found. This should usually be `unknown` (or `any` if you like to live dangerously).|
+|`poolConfig`||`PoolConfig \| undefined`<br/>(see [below](#complex-config-types))|`undefined`|Slonik database pool configuration. Will be used to create a pool which issues queries to the database as the tool is running, and will have its type parsers inspected to ensure the generated types are correct. It's important to pass in a pool confguration which is the same as the one used in your application.|
+|`logger`||`Logger`<br/>(see [below](#complex-config-types))|`console`|Logger object with `debug`, `info`, `warn` and `error` methods. Defaults to `console`.|
+|`writeTypes`<br/>(experimental)||`WriteTypes`<br/>(see [below](#complex-config-types))|`typegen.`&thinsp;`defaultWriteTypes`|Control how files are written to disk. See the [writeTypes](#writetypes) section.|
+||`--config`|`string`|`'typegen.config.js'`|Path to configuration file.|
+||`--migrate`|`'<=0.8.0'`|disabled|Before generating types, attempt to migrate a codebase which has used a prior version of this tool.|
+||`--watch`|CLI argument|disabled|Run in watch mode.|
+||`--lazy`|CLI argument|disabled|Skip initial processing of input files. Only useful with `'--watch'`.|
+||`--skip-check-clean`|CLI argument|disabled|If enabled, the tool will not check the git status to ensure changes are checked in.|
+
+#### Complex config types
+```typescript
+type Logger = Record<'error' | 'warn' | 'info' | 'debug', (msg: unknown) => void>;
+type WriteTypes = (queries: AnalysedQuery[]) => Promise<void>;
+type PoolConfig = slonik.ClientConfigurationInput; // imported from slonik lib
+```
 
-- `rootDir` - Source root that the tool will search for files in. Defaults to `src`. Can be overridden with the `--root-dir` CLI argument.
-- `include` - Array of glob patterns for files to include in processing. Defaults to `['**/*.{ts,sql}']`, matching all `.ts` and `.sql` files. Can be overridden with the `--include` CLI argument.
-- `exclude` - Array of glob patterns for files to exclude from processing. Defaults to `['**/node_modules/**']`, excluding `node_modules`. Can be overridden with the `--exclude` CLI argument.
-- `since` - Limit matched files to those which have been changed since the given git ref. Use `"HEAD"` for files changed since the last commit, `"main"` for files changed in a branch, etc. Can be overridden with the `--since` CLI argument.
-- `connectionURI` - URI for connecting to psql. Defaults to `postgresql://postgres:postgres@localhost:5432/postgres`. Note that if you are using `psql` inside docker, you should make sure that the container and host port match, since this will be used both by `psql` and slonik to connect to the database.
-- `poolConfig` - Slonik database pool configuration. Will be used to create a pool which issues queries to the database as the tool is running, and will have its type parsers inspected to ensure the generated types are correct. It's important to pass in a pool confguration which is the same as the one used in your application.
-- `psqlCommand` - the CLI command for running the official postgres `psql` CLI client. Defaults to `psql`. You can test it's working, and that your postgres version supports `\gdesc` with your connection string using: `echo 'select 123 as abc \gdesc' | psql "postgresql://postgres:postgres@localhost:5432/postgres" -f -`. Note that right now this can't contain single quotes. This should also be configured to talk to the same database as the `pool` variable (and it should be a development database - don't run this tool in production!). If you are using docker compose, you can use a command like `docker-compose exec -T postgres psql`
-- `logger` - Logger object with `debug`, `info`, `warn` and `error` methods. Defaults to `console`.
-- `writeTypes` (advanced/experimental) - Control how files are written to disk. See the [writeTypes](#writetypes) section.
+#### Testing `psqlCommand`
+You can check if your `psql` is working, and that your postgres version supports `\gdesc` with your connection string using this shell command:
+```bash
+echo 'select 123 as abc \gdesc' \| psql "postgresql://postgres:postgres@localhost:5432/postgres" -f -
+```
+
+There are some more configuration options [documented in code](./src/types.ts), but these should be considered experimental, and might change without warning. You can try them out as documented [below](#writetypes), but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way.
 
 ### Example config
 
@@ -148,87 +175,6 @@ module.exports.default = {
 
 Note that the `/** @type {import('@slonik/typegen').Options} */` comment is optional, but will ensure your IDE gives you type hints.
 
-### CLI options
-
-Some of the options above can be overriden by the CLI:
-
-<!-- codegen:start {preset: custom, source: ./docgen.js, export: cliHelpText} -->
-```
-usage: slonik-typegen generate [-h] [--config PATH] [--root-dir PATH]
-                               [--connection-uri URI] [--psql COMMAND]
-                               [--default-type TYPESCRIPT] [--include PATTERN]
-                               [--exclude PATTERN] [--since REF]
-                               [--migrate {<=0.8.0}] [--skip-check-clean]
-                               [--watch] [--lazy]
-
-
-Generates a directory containing with a 'sql' tag wrapper based on found 
-queries found in source files. By default, searches 'src' for source files.
-
-Optional arguments:
-
-  -h, --help            Show this help message and exit.
-
-  --config PATH         Path to a module containing parameters to be passed 
-                        to 'generate'. If specified, it will be required and 
-                        the export will be used as parameters. If not 
-                        specified, defaults will be used. Note: other CLI 
-                        arguments will override values set in this module
-
-  --root-dir PATH       Path to the source directory containing SQL queries. 
-                        Defaults to "src" if no value is provided
-
-  --connection-uri URI  URI for connecting to postgres. Defaults to 
-                        URI for connecting to postgres. Defaults to 
-
-  --psql COMMAND        psql command used to query postgres via CLI client. e.
-                        g. 'psql -h localhost -U postgres postgres' if 
-                        running postgres locally, or 'docker-compose exec -T 
-                        postgres psql -h localhost -U postgres postgres' if 
-                        running with docker-compose. You can test this by 
-                        running "<<your_psql_command>> -c 'select 1 as a, 2 
-                        as b'". Note that this command will be executed 
-                        dynamically, so avoid using any escape characters in 
-                        here.
-
-  --default-type TYPESCRIPT
-                        TypeScript fallback type for when no type is found. 
-                        Most simple types (text, int etc.) are mapped to 
-                        their TypeScript equivalent automatically. This 
-                        should usually be 'unknown', or 'any' if you like to 
-                        live dangerously.
-
-  --include PATTERN     Glob pattern of files to search for SQL queries in. 
-                        By default searches for all .ts and .sql files: '**/*.
-                        {ts,sql}' This option is repeatable to include 
-                        multiple patterns.
-
-  --exclude PATTERN     Glob pattern for files to be excluded from processing.
-                         By default excludes '**/node_modules/**'. This 
-                        option is repeatable to exlude multiple patterns.
-
-  --since REF           Limit affected files to those which have been changed 
-                        since the given git ref. Use "--since HEAD" for files 
-                        changed since the last commit, "--since main for 
-                        files changed in a branch, etc. This option has no 
-                        effect in watch mode.
-
-  --migrate {<=0.8.0}   Before generating types, attempt to migrate a 
-                        codebase which has used a prior version of this tool
-
-  --skip-check-clean    If enabled, the tool will not check the git status to 
-                        ensure changes are checked in.
-
-  --watch               Run the type checker in watch mode. Files will be run 
-                        through the code generator when changed or added.
-
-  --lazy                Skip initial processing of input files. Only useful 
-                        with '--watch'.
-```
-<!-- codegen:end -->
-
-There are some more configuration options [documented in code](./src/types.ts), but these should be considered experimental, and might change without warning. You can try them out as documented below, but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way.
-
 ### writeTypes
 
 The `writeTypes` option allows you to tweak what's written to disk. Note that the usage style isn't finalised and might change in future. If you use it, please create a discussion about it in https://github.com/mmkal/slonik-tools/discussions so that your use-case doesn't get taken away unexpectedly.
@@ -386,6 +332,64 @@ module.exports.default = {
 }
 ```
 
+## Enhancing Return Types
+
+Typgen is designed to output types only to the degree it's certain they are correct.  
+
+Let's say in a complex query it can determine that a specific column will return a `string`, but isn't sure if it is also nullable, it will extract the type as `{ column: string | null }`, just to be on the safe side. When it encounters columns where it is unable to even determine the basic type, i.e. `json` columns, it will return :shrug: (Ok, actually the typescript equivalent, which is `unknown`).
+
+In these cases you likely know more about the actual return type than typegen and you might feel the urge to overwrite the types.  
+Yet you shouldn't touch generated code, as your changes will be removed again on the next run.
+
+Instead what you should do is add (one or more) intersection types to the sql literal, specifying the columns where you want to help typegen out by increasing specificity. The resulting type will be a combination of the extracted types and your enhancements.  
+Check out the [typescript docs on intersection types](https://www.typescriptlang.org/docs/handbook/2/objects.html#intersection-types) to learn more.
+
+Imagine this is your code after running typegen.
+```typescript
+sql<queries.ExtractedResult>`select string_col, json_col from table`
+
+export declare namespace queries {
+  // Generated by @slonik/typegen
+
+  /** - query: `select string_col, json_col from table` */
+  export interface TestTable {
+    /** column: `example_test.table.string_col`, regtype: `character_varying` */
+    string_col: string | null,
+    /** column: `example_test.table.json_col`, regtype: `jsonb` */
+    json_col: unkown
+  }
+}
+```
+
+You can enhance the return type like this:
+
+```typescript
+sql<queries.ExtractedResult & { json_col: string[] }>`[query]`
+```
+\- or, if you prefer -
+```typescript
+interface EnhancedResult {
+  json_col: string[]
+}
+sql<queries.ExtractedResult & EnhancedResult>`[query]`
+```
+
+Either way the resulting type will be this:
+
+```typescript
+type ResultingType = {
+  string_col: string | null,
+  json_col: string[]
+}
+```
+
+**On subsequent runs typegen will only update the first intersection type and <u>leave all subsequent intersections untouched**.</u>
+
+This also means you can make the column `string_col` non-nullable by intersecting it with `{ string_col: string }`.
+
+Note that you can't completely change a type (say from `string` to `number`) this way. This is by design, because if you could, a change of the unterlying table might cause typegen to detect a new type, which would be ignored, if you could overwrite it. It is also why typegen only specifies a type when it's reasonably sure, as stated at the beginning of this paragraph.  
+If you found an example where this is not the case, please [raise an issue](https://github.com/mmkal/slonik-tools/issues/new).
+
 ## Examples
 
 [The tests](./test) and [corresponding fixtures](./test/fixtures) are a good starting point to see what the code-generator will do.

packages/typegen/src/write/inline.ts

@@ -1,15 +1,19 @@
+import * as path from 'path'
+
 import * as lodash from 'lodash'
+import type * as ts from 'typescript'
+
 import {TaggedQuery} from '../types'
 import {relativeUnixPath} from '../util'
 import {tsPrettify} from './prettify'
-import type * as ts from 'typescript'
-import * as path from 'path'
 import {queryInterfaces} from './typescript'
 import {WriteFile} from '.'
 
 // todo: pg-protocol parseError adds all the actually useful information
 // to fields which don't show up in error messages. make a library which patches it to include relevant info.
 
+const queryNamespace = 'queries' // todo: at some point we might want to make this configurable
+
 export const defaultGetQueriesModule = (filepath: string) => filepath
 
 export interface WriteTSFileOptions {
@@ -34,7 +38,7 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w
 
     const edits: Array<Edit> = []
 
-    visit(sourceFile)
+    visitRecursive(sourceFile)
 
     const destPath = getQueriesModulePath(file)
     if (destPath === file) {
@@ -48,7 +52,7 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w
       await writeFile(destPath, content)
 
       const importPath = relativeUnixPath(destPath, path.dirname(file))
-      const importStatement = `import * as queries from './${importPath.replace(/\.(js|ts|tsx)$/, '')}'`
+      const importStatement = `import * as ${queryNamespace} from './${importPath.replace(/\.(js|ts|tsx)$/, '')}'`
 
       const importExists =
         originalSource.includes(importStatement) ||
@@ -69,31 +73,51 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w
 
     await writeFile(file, newSource)
 
-    function visit(node: ts.Node) {
-      if (ts.isModuleDeclaration(node) && node.name.getText() === 'queries') {
+    function visitRecursive(node: ts.Node) {
+      if (ts.isModuleDeclaration(node) && node.name.getText() === queryNamespace) {
+        // remove old import(s) (will get re-added later)
         edits.push({
           start: node.getStart(sourceFile),
           end: node.getEnd(),
           replacement: '',
         })
+        return
       }
 
       if (ts.isTaggedTemplateExpression(node)) {
-        const isSqlIdentifier = (n: ts.Node) => ts.isIdentifier(n) && n.getText() === 'sql'
-        const sqlPropertyAccessor = ts.isPropertyAccessExpression(node.tag) && isSqlIdentifier(node.tag.name)
-        if (isSqlIdentifier(node.tag) || sqlPropertyAccessor) {
-          const match = group.find(q => q.text === node.getFullText())
-          if (match) {
+        const isSqlIdentifier = (e: ts.Node) => ts.isIdentifier(e) && e.getText() === 'sql'
+        const isSqlPropertyAccessor = (e: ts.Expression) => ts.isPropertyAccessExpression(e) && isSqlIdentifier(e.name)
+        if (!isSqlIdentifier(node.tag) && !isSqlPropertyAccessor(node.tag)) {
+          return
+        }
+        const matchingQuery = group.find(q => q.text === node.getFullText())
+        if (!matchingQuery) {
+          return
+        }
+        const typeReference = `${queryNamespace}.${matchingQuery.tag}`
+        if (node.typeArguments && node.typeArguments.length === 1) {
+          // existing type definitions
+          const [typeNode] = node.typeArguments
+          if (ts.isIntersectionTypeNode(typeNode)) {
+            // we want to preserve intersection types
+            const [firstArg] = typeNode.types // We can't be sure the first argument is a generated type, but as the namespace might have been overwritten we're gonna have to assume.
             edits.push({
-              start: node.tag.getStart(sourceFile),
-              end: node.template.getStart(sourceFile),
-              replacement: `${node.tag.getText()}<queries.${match.tag}>`,
+              start: firstArg.getStart(sourceFile),
+              end: firstArg.getEnd(),
+              replacement: typeReference,
             })
+            return
           }
         }
+        // default: replace complete tag to add/overwrite type arguments
+        edits.push({
+          start: node.tag.getStart(sourceFile),
+          end: node.template.getStart(sourceFile),
+          replacement: `${node.tag.getText()}<${typeReference}>`,
+        })
       }
 
-      ts.forEachChild(node, visit)
+      ts.forEachChild(node, visitRecursive)
     }
   }
 }