From f55655874705198a74f5ea56d795f8d4c9c88490 Mon Sep 17 00:00:00 2001 From: Jan Paepke Date: Thu, 14 Apr 2022 17:20:38 +0200 Subject: [PATCH 1/8] retain intersection types --- packages/typegen/readme.md | 190 +++++++++--------- packages/typegen/src/write/inline.ts | 54 +++-- .../test/inline-tag-modification.test.ts | 77 +++++++ 3 files changed, 213 insertions(+), 108 deletions(-) create mode 100644 packages/typegen/test/inline-tag-modification.test.ts diff --git a/packages/typegen/readme.md b/packages/typegen/readme.md index d1b79129..4a7200a0 100644 --- a/packages/typegen/readme.md +++ b/packages/typegen/readme.md @@ -26,11 +26,11 @@ Select statements, joins, and updates/inserts/deletes using `returning` are all - [Usage](#usage) - [Configuration](#configuration) - [Example config](#example-config) - - [CLI options](#cli-options) - - [writeTypes](#writetypes) + - [Advanced Configuration](#writetypes) - [Controlling write destination](#controlling-write-destination) - [Modifying types](#modifying-types) - [Modifying source files](#modifying-source-files) +- [Enhancing Return Types](#enhancing-return-types) - [Examples](#examples) - [Migration from v0.8.0](#migration-from-v080) - [SQL files](#sql-files) @@ -117,17 +117,44 @@ export declare namespace queries { ## Configuration -The CLI can run with zero config, but there will usually be customisations needed depending on your project's setup. By default, the CLI will look for `typegen.config.js` file in the working directory. The config file can contain the following options (all are optional): +The CLI can run with zero config, but there will usually be customisations needed depending on your project's setup. +By default, the CLI will look for `typegen.config.js` file in the working directory, exporting an object containing the properties below. + +Some options are only available via CLI, some are only available in the config. +CLI arguments will always have precedence over config options. + +|Option|CLI Argument            |Type|Default|Description| +|-|-|-|-|-| +|`rootDir`|`--root-dir`|`string`|`'src'`|Source root that the tool will search for files in.| +|`include`|`--include`|`string[]`|`['**/*.{ts,sql}']`|Glob patterns for files to include in processing. Repeatable in CLI.| +|`exclude`|`--exclude`|`string[]`|`['**/node_modules/**']`|Glob patterns for files to exclude from processing. Repeatable in CLI.| +|`since`|`--since`|`string \| undefined`|`undefined`|Limit matched files to those which have been changed since the given git ref. Use `"HEAD"` for files changed since the last commit, `"main"` for files changed in a branch, etc.| +|`connectionURI`|`--connection-uri`|`string`|`'postgresql://` `postgres:postgres` `@localhost:5432/` `postgres'`|URI for connecting to psql. Note that if you are using `psql` inside docker, you should make sure that the container and host port match, since this will be used both by `psql` and slonik to connect to the database.| +|`psqlCommand`|`--psql`|`string`|`'psql'`|The CLI command for running the official postgres `psql` CLI client.
Note that right now this can't contain single quotes. This should also be configured to talk to the same database as the `pool` variable (and it should be a development database - don't run this tool in production!). If you are using docker compose, you can use a command like `docker-compose exec -T postgres psql`| +|`defaultType`|`--default-type`|`string`|`'unknown'`|TypeScript type when no mapping is found. This should usually be `unknown` (or `any` if you like to live dangerously).| +|`poolConfig`||`PoolConfig \| undefined`
(see [below](#complex-config-types))|`undefined`|Slonik database pool configuration. Will be used to create a pool which issues queries to the database as the tool is running, and will have its type parsers inspected to ensure the generated types are correct. It's important to pass in a pool confguration which is the same as the one used in your application.| +|`logger`||`Logger`
(see [below](#complex-config-types))|`console`|Logger object with `debug`, `info`, `warn` and `error` methods. Defaults to `console`.| +|`writeTypes`
(experimental)||`WriteTypes`
(see [below](#complex-config-types))|`typegen.` `defaultWriteTypes`|Control how files are written to disk. See the [writeTypes](#writetypes) section.| +||`--config`|`string`|`'typegen.config.js'`|Path to configuration file.| +||`--migrate`|`'<=0.8.0'`|disabled|Before generating types, attempt to migrate a codebase which has used a prior version of this tool.| +||`--watch`|CLI argument|disabled|Run in watch mode.| +||`--lazy`|CLI argument|disabled|Skip initial processing of input files. Only useful with `'--watch'`.| +||`--skip-check-clean`|CLI argument|disabled|If enabled, the tool will not check the git status to ensure changes are checked in.| + +#### Complex config types +```typescript +type Logger = Record<'error' | 'warn' | 'info' | 'debug', (msg: unknown) => void>; +type WriteTypes = (queries: AnalysedQuery[]) => Promise; +type PoolConfig = slonik.ClientConfigurationInput; // imported from slonik lib +``` -- `rootDir` - Source root that the tool will search for files in. Defaults to `src`. Can be overridden with the `--root-dir` CLI argument. -- `include` - Array of glob patterns for files to include in processing. Defaults to `['**/*.{ts,sql}']`, matching all `.ts` and `.sql` files. Can be overridden with the `--include` CLI argument. -- `exclude` - Array of glob patterns for files to exclude from processing. Defaults to `['**/node_modules/**']`, excluding `node_modules`. Can be overridden with the `--exclude` CLI argument. -- `since` - Limit matched files to those which have been changed since the given git ref. Use `"HEAD"` for files changed since the last commit, `"main"` for files changed in a branch, etc. Can be overridden with the `--since` CLI argument. -- `connectionURI` - URI for connecting to psql. Defaults to `postgresql://postgres:postgres@localhost:5432/postgres`. Note that if you are using `psql` inside docker, you should make sure that the container and host port match, since this will be used both by `psql` and slonik to connect to the database. -- `poolConfig` - Slonik database pool configuration. Will be used to create a pool which issues queries to the database as the tool is running, and will have its type parsers inspected to ensure the generated types are correct. It's important to pass in a pool confguration which is the same as the one used in your application. -- `psqlCommand` - the CLI command for running the official postgres `psql` CLI client. Defaults to `psql`. You can test it's working, and that your postgres version supports `\gdesc` with your connection string using: `echo 'select 123 as abc \gdesc' | psql "postgresql://postgres:postgres@localhost:5432/postgres" -f -`. Note that right now this can't contain single quotes. This should also be configured to talk to the same database as the `pool` variable (and it should be a development database - don't run this tool in production!). If you are using docker compose, you can use a command like `docker-compose exec -T postgres psql` -- `logger` - Logger object with `debug`, `info`, `warn` and `error` methods. Defaults to `console`. -- `writeTypes` (advanced/experimental) - Control how files are written to disk. See the [writeTypes](#writetypes) section. +#### Testing `psqlCommand` +You can check if your `psql` is working, and that your postgres version supports `\gdesc` with your connection string using this shell command: +```bash +echo 'select 123 as abc \gdesc' \| psql "postgresql://postgres:postgres@localhost:5432/postgres" -f - +``` + +There are some more configuration options [documented in code](./src/types.ts), but these should be considered experimental, and might change without warning. You can try them out as documented [below](#writetypes), but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way. ### Example config @@ -148,87 +175,6 @@ module.exports.default = { Note that the `/** @type {import('@slonik/typegen').Options} */` comment is optional, but will ensure your IDE gives you type hints. -### CLI options - -Some of the options above can be overriden by the CLI: - - -``` -usage: slonik-typegen generate [-h] [--config PATH] [--root-dir PATH] - [--connection-uri URI] [--psql COMMAND] - [--default-type TYPESCRIPT] [--include PATTERN] - [--exclude PATTERN] [--since REF] - [--migrate {<=0.8.0}] [--skip-check-clean] - [--watch] [--lazy] - - -Generates a directory containing with a 'sql' tag wrapper based on found -queries found in source files. By default, searches 'src' for source files. - -Optional arguments: - - -h, --help Show this help message and exit. - - --config PATH Path to a module containing parameters to be passed - to 'generate'. If specified, it will be required and - the export will be used as parameters. If not - specified, defaults will be used. Note: other CLI - arguments will override values set in this module - - --root-dir PATH Path to the source directory containing SQL queries. - Defaults to "src" if no value is provided - - --connection-uri URI URI for connecting to postgres. Defaults to - URI for connecting to postgres. Defaults to - - --psql COMMAND psql command used to query postgres via CLI client. e. - g. 'psql -h localhost -U postgres postgres' if - running postgres locally, or 'docker-compose exec -T - postgres psql -h localhost -U postgres postgres' if - running with docker-compose. You can test this by - running "<> -c 'select 1 as a, 2 - as b'". Note that this command will be executed - dynamically, so avoid using any escape characters in - here. - - --default-type TYPESCRIPT - TypeScript fallback type for when no type is found. - Most simple types (text, int etc.) are mapped to - their TypeScript equivalent automatically. This - should usually be 'unknown', or 'any' if you like to - live dangerously. - - --include PATTERN Glob pattern of files to search for SQL queries in. - By default searches for all .ts and .sql files: '**/*. - {ts,sql}' This option is repeatable to include - multiple patterns. - - --exclude PATTERN Glob pattern for files to be excluded from processing. - By default excludes '**/node_modules/**'. This - option is repeatable to exlude multiple patterns. - - --since REF Limit affected files to those which have been changed - since the given git ref. Use "--since HEAD" for files - changed since the last commit, "--since main for - files changed in a branch, etc. This option has no - effect in watch mode. - - --migrate {<=0.8.0} Before generating types, attempt to migrate a - codebase which has used a prior version of this tool - - --skip-check-clean If enabled, the tool will not check the git status to - ensure changes are checked in. - - --watch Run the type checker in watch mode. Files will be run - through the code generator when changed or added. - - --lazy Skip initial processing of input files. Only useful - with '--watch'. -``` - - -There are some more configuration options [documented in code](./src/types.ts), but these should be considered experimental, and might change without warning. You can try them out as documented below, but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way. - ### writeTypes The `writeTypes` option allows you to tweak what's written to disk. Note that the usage style isn't finalised and might change in future. If you use it, please create a discussion about it in https://github.com/mmkal/slonik-tools/discussions so that your use-case doesn't get taken away unexpectedly. @@ -386,6 +332,64 @@ module.exports.default = { } ``` +## Enhancing Return Types + +Typgen is designed to output types only to the degree it's certain they are correct. + +Let's say in a complex query it can determine that a specific column will return a `string`, but isn't sure if it is also nullable, it will extract the type as `{ column: string | null }`, just to be on the safe side. When it encounters columns where it is unable to even determine the basic type, i.e. `json` columns, it will return :shrug: (Ok, actually the typescript equivalent, which is `unknown`). + +In these cases you likely know more about the actual return type than typegen and you might feel the urge to overwrite the types. +Yet you shouldn't touch generated code, as your changes will be removed again on the next run. + +Instead what you should do is add (one or more) intersection types to the sql literal, specifying the columns where you want to help typegen out by increasing specificity. The resulting type will be a combination of the extracted types and your enhancements. +Check out the [typescript docs on intersection types](https://www.typescriptlang.org/docs/handbook/2/objects.html#intersection-types) to learn more. + +Imagine this is your code after running typegen. +```typescript +sql`select string_col, json_col from table` + +export declare namespace queries { + // Generated by @slonik/typegen + + /** - query: `select string_col, json_col from table` */ + export interface TestTable { + /** column: `example_test.table.string_col`, regtype: `character_varying` */ + string_col: string | null, + /** column: `example_test.table.json_col`, regtype: `jsonb` */ + json_col: unkown + } +} +``` + +You can enhance the return type like this: + +```typescript +sql`[query]` +``` +\- or, if you prefer - +```typescript +interface EnhancedResult { + json_col: string[] +} +sql`[query]` +``` + +Either way the resulting type will be this: + +```typescript +type ResultingType = { + string_col: string | null, + json_col: string[] +} +``` + +**On subsequent runs typegen will only update the first intersection type and leave all subsequent intersections untouched**. + +This also means you can make the column `string_col` non-nullable by intersecting it with `{ string_col: string }`. + +Note that you can't completely change a type (say from `string` to `number`) this way. This is by design, because if you could, a change of the unterlying table might cause typegen to detect a new type, which would be ignored, if you could overwrite it. It is also why typegen only specifies a type when it's reasonably sure, as stated at the beginning of this paragraph. +If you found an example where this is not the case, please [raise an issue](https://github.com/mmkal/slonik-tools/issues/new). + ## Examples [The tests](./test) and [corresponding fixtures](./test/fixtures) are a good starting point to see what the code-generator will do. diff --git a/packages/typegen/src/write/inline.ts b/packages/typegen/src/write/inline.ts index 2a5aedd7..8448943e 100644 --- a/packages/typegen/src/write/inline.ts +++ b/packages/typegen/src/write/inline.ts @@ -1,15 +1,19 @@ +import * as path from 'path' + import * as lodash from 'lodash' +import type * as ts from 'typescript' + import {TaggedQuery} from '../types' import {relativeUnixPath} from '../util' import {tsPrettify} from './prettify' -import type * as ts from 'typescript' -import * as path from 'path' import {queryInterfaces} from './typescript' import {WriteFile} from '.' // todo: pg-protocol parseError adds all the actually useful information // to fields which don't show up in error messages. make a library which patches it to include relevant info. +const queryNamespace = 'queries' // todo: at some point we might want to make this configurable + export const defaultGetQueriesModule = (filepath: string) => filepath export interface WriteTSFileOptions { @@ -34,7 +38,7 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w const edits: Array = [] - visit(sourceFile) + visitRecursive(sourceFile) const destPath = getQueriesModulePath(file) if (destPath === file) { @@ -48,7 +52,7 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w await writeFile(destPath, content) const importPath = relativeUnixPath(destPath, path.dirname(file)) - const importStatement = `import * as queries from './${importPath.replace(/\.(js|ts|tsx)$/, '')}'` + const importStatement = `import * as ${queryNamespace} from './${importPath.replace(/\.(js|ts|tsx)$/, '')}'` const importExists = originalSource.includes(importStatement) || @@ -69,31 +73,51 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w await writeFile(file, newSource) - function visit(node: ts.Node) { - if (ts.isModuleDeclaration(node) && node.name.getText() === 'queries') { + function visitRecursive(node: ts.Node) { + if (ts.isModuleDeclaration(node) && node.name.getText() === queryNamespace) { + // remove old import(s) (will get re-added later) edits.push({ start: node.getStart(sourceFile), end: node.getEnd(), replacement: '', }) + return } if (ts.isTaggedTemplateExpression(node)) { - const isSqlIdentifier = (n: ts.Node) => ts.isIdentifier(n) && n.getText() === 'sql' - const sqlPropertyAccessor = ts.isPropertyAccessExpression(node.tag) && isSqlIdentifier(node.tag.name) - if (isSqlIdentifier(node.tag) || sqlPropertyAccessor) { - const match = group.find(q => q.text === node.getFullText()) - if (match) { + const isSqlIdentifier = (e: ts.Expression) => ts.isIdentifier(e) && e.getText() === 'sql' + const isSqlPropertyAccessor = (e: ts.Expression) => ts.isPropertyAccessExpression(e) && isSqlIdentifier(e.name) + if (!isSqlIdentifier(node.tag) && !isSqlPropertyAccessor(node.tag)) { + return + } + const matchingQuery = group.find(q => q.text === node.getFullText()) + if (!matchingQuery) { + return + } + const typeReference = `${queryNamespace}.${matchingQuery.tag}` + if (node.typeArguments && node.typeArguments.length === 1) { + // existing type definitions + const [typeNode] = node.typeArguments + if (ts.isIntersectionTypeNode(typeNode)) { + // we want to preserve intersection types + const [firstArg] = typeNode.types // We can't be sure the first argument is a generated type, but as the namespace might have been overwritten we're gonna have to assume. edits.push({ - start: node.tag.getStart(sourceFile), - end: node.template.getStart(sourceFile), - replacement: `${node.tag.getText()}`, + start: firstArg.getStart(sourceFile), + end: firstArg.getEnd(), + replacement: typeReference, }) + return } } + // default: replace complete tag to add/overwrite type arguments + edits.push({ + start: node.tag.getStart(sourceFile), + end: node.template.getStart(sourceFile), + replacement: `${node.tag.getText()}<${typeReference}>`, + }) } - ts.forEachChild(node, visit) + ts.forEachChild(node, visitRecursive) } } } diff --git a/packages/typegen/test/inline-tag-modification.test.ts b/packages/typegen/test/inline-tag-modification.test.ts new file mode 100644 index 00000000..4b3aa6fb --- /dev/null +++ b/packages/typegen/test/inline-tag-modification.test.ts @@ -0,0 +1,77 @@ +import * as fsSyncer from 'fs-syncer' + +import * as typegen from '../src' +import {getHelper} from './helper' + +export const {typegenOptions, logger, poolHelper: helper} = getHelper({__filename}) + +beforeEach(async () => { + await helper.pool.query(helper.sql` + create table test_table(foo int not null, bar text); + `) +}) + +const createInput = (existingTag: string = '') => ` + import {sql, createPool} from 'slonik' + + export default () => { + const pool = createPool('...connection string...') + return pool.query(sql${existingTag}\`select foo, bar from test_table\`) + } +` +const createSnapshot = (resultingTag: string) => ` +"--- +index.ts: |- + ${createInput(resultingTag).trim()} + + export declare namespace queries { + // Generated by @slonik/typegen + + /** - query: \`select foo, bar from test_table\` */ + export interface TestTable { + /** column: \`inline_tag_modification_test.test_table.foo\`, not null: \`true\`, regtype: \`integer\` */ + foo: number + + /** column: \`inline_tag_modification_test.test_table.bar\`, regtype: \`text\` */ + bar: string | null + } + } + " +` + +const process = async (input: string = '') => { + const syncer = fsSyncer.jestFixture({ + targetState: {'index.ts': input}, + }) + syncer.sync() + await typegen.generate(typegenOptions(syncer.baseDir)) + return syncer.yaml() +} + +const checkModification = async (existingType: string | undefined, resultingType: string) => { + const input = createInput(existingType === undefined ? '' : `<${existingType}>`) + const result = createSnapshot(`<${resultingType}>`) + const processed = await process(input) + expect(processed).toMatchInlineSnapshot(result) +} + +describe('inline tag modification', () => { + test('add tag', () => checkModification(undefined, 'queries.TestTable')) + test('overwrite existing', async () => { + // running sequentially to avoid overlapping fixtures + await checkModification('{col: string}', 'queries.TestTable') + await checkModification('queries.TestTable', 'queries.TestTable') + await checkModification('queries.TestTable | Other', 'queries.TestTable') + await checkModification("Omit", 'queries.TestTable') + }) + test('preserve intersections', async () => { + // running sequentially to avoid overlapping fixtures + await checkModification('queries.TestTable & {col: string}', 'queries.TestTable & {col: string}') + await checkModification('queries.TestTable & Other', 'queries.TestTable & Other') + await checkModification( + 'queries.TestTable & One & Two & {col: string}', + 'queries.TestTable & One & Two & {col: string}', + ) + await checkModification('{col: string} & Other', 'queries.TestTable & Other') // we can't tell if the first intersection type is generated, so it will always be overwritten + }) +}) From aa13da8cb0bd3250310961889ac60838dde4b27e Mon Sep 17 00:00:00 2001 From: Jan Paepke Date: Thu, 14 Apr 2022 17:32:52 +0200 Subject: [PATCH 2/8] shut up, github actions --- packages/typegen/src/write/inline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/typegen/src/write/inline.ts b/packages/typegen/src/write/inline.ts index 8448943e..5ca0b1fa 100644 --- a/packages/typegen/src/write/inline.ts +++ b/packages/typegen/src/write/inline.ts @@ -85,7 +85,7 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w } if (ts.isTaggedTemplateExpression(node)) { - const isSqlIdentifier = (e: ts.Expression) => ts.isIdentifier(e) && e.getText() === 'sql' + const isSqlIdentifier = (e: ts.Node) => ts.isIdentifier(e) && e.getText() === 'sql' const isSqlPropertyAccessor = (e: ts.Expression) => ts.isPropertyAccessExpression(e) && isSqlIdentifier(e.name) if (!isSqlIdentifier(node.tag) && !isSqlPropertyAccessor(node.tag)) { return From be937325b7b3ebe4b92ba02ded94a2f0d959e7b5 Mon Sep 17 00:00:00 2001 From: Jan Paepke Date: Fri, 15 Apr 2022 10:59:29 +0200 Subject: [PATCH 3/8] restructure config info in readme --- packages/typegen/readme.md | 132 +++++++++++-------------------------- 1 file changed, 39 insertions(+), 93 deletions(-) diff --git a/packages/typegen/readme.md b/packages/typegen/readme.md index d1b79129..9ab7f6d8 100644 --- a/packages/typegen/readme.md +++ b/packages/typegen/readme.md @@ -26,11 +26,11 @@ Select statements, joins, and updates/inserts/deletes using `returning` are all - [Usage](#usage) - [Configuration](#configuration) - [Example config](#example-config) - - [CLI options](#cli-options) - - [writeTypes](#writetypes) + - [Advanced Configuration](#writetypes) - [Controlling write destination](#controlling-write-destination) - [Modifying types](#modifying-types) - [Modifying source files](#modifying-source-files) +- [Enhancing Return Types](#enhancing-return-types) - [Examples](#examples) - [Migration from v0.8.0](#migration-from-v080) - [SQL files](#sql-files) @@ -117,17 +117,44 @@ export declare namespace queries { ## Configuration -The CLI can run with zero config, but there will usually be customisations needed depending on your project's setup. By default, the CLI will look for `typegen.config.js` file in the working directory. The config file can contain the following options (all are optional): +The CLI can run with zero config, but there will usually be customisations needed depending on your project's setup. +By default, the CLI will look for `typegen.config.js` file in the working directory, exporting an object containing the properties below. + +Some options are only available via CLI, some are only available in the config. +CLI arguments will always have precedence over config options. + +|Option|CLI Argument            |Type|Default|Description| +|-|-|-|-|-| +|`rootDir`|`--root-dir`|`string`|`'src'`|Source root that the tool will search for files in.| +|`include`|`--include`|`string[]`|`['**/*.{ts,sql}']`|Glob patterns for files to include in processing. Repeatable in CLI.| +|`exclude`|`--exclude`|`string[]`|`['**/node_modules/**']`|Glob patterns for files to exclude from processing. Repeatable in CLI.| +|`since`|`--since`|`string \| undefined`|`undefined`|Limit matched files to those which have been changed since the given git ref. Use `"HEAD"` for files changed since the last commit, `"main"` for files changed in a branch, etc.| +|`connectionURI`|`--connection-uri`|`string`|`'postgresql://` `postgres:postgres` `@localhost:5432/` `postgres'`|URI for connecting to psql. Note that if you are using `psql` inside docker, you should make sure that the container and host port match, since this will be used both by `psql` and slonik to connect to the database.| +|`psqlCommand`|`--psql`|`string`|`'psql'`|The CLI command for running the official postgres `psql` CLI client.
Note that right now this can't contain single quotes. This should also be configured to talk to the same database as the `pool` variable (and it should be a development database - don't run this tool in production!). If you are using docker compose, you can use a command like `docker-compose exec -T postgres psql`| +|`defaultType`|`--default-type`|`string`|`'unknown'`|TypeScript type when no mapping is found. This should usually be `unknown` (or `any` if you like to live dangerously).| +|`poolConfig`||`PoolConfig \| undefined`
(see [below](#complex-config-types))|`undefined`|Slonik database pool configuration. Will be used to create a pool which issues queries to the database as the tool is running, and will have its type parsers inspected to ensure the generated types are correct. It's important to pass in a pool confguration which is the same as the one used in your application.| +|`logger`||`Logger`
(see [below](#complex-config-types))|`console`|Logger object with `debug`, `info`, `warn` and `error` methods. Defaults to `console`.| +|`writeTypes`
(experimental)||`WriteTypes`
(see [below](#complex-config-types))|`typegen.` `defaultWriteTypes`|Control how files are written to disk. See the [writeTypes](#writetypes) section.| +||`--config`|`string`|`'typegen.config.js'`|Path to configuration file.| +||`--migrate`|`'<=0.8.0'`|disabled|Before generating types, attempt to migrate a codebase which has used a prior version of this tool.| +||`--watch`|CLI argument|disabled|Run in watch mode.| +||`--lazy`|CLI argument|disabled|Skip initial processing of input files. Only useful with `'--watch'`.| +||`--skip-check-clean`|CLI argument|disabled|If enabled, the tool will not check the git status to ensure changes are checked in.| + +#### Complex config types +```typescript +type Logger = Record<'error' | 'warn' | 'info' | 'debug', (msg: unknown) => void>; +type WriteTypes = (queries: AnalysedQuery[]) => Promise; +type PoolConfig = slonik.ClientConfigurationInput; // imported from slonik lib +``` + +#### Testing `psqlCommand` +You can check if your `psql` is working, and that your postgres version supports `\gdesc` with your connection string using this shell command: +```bash +echo 'select 123 as abc \gdesc' \| psql "postgresql://postgres:postgres@localhost:5432/postgres" -f - +``` -- `rootDir` - Source root that the tool will search for files in. Defaults to `src`. Can be overridden with the `--root-dir` CLI argument. -- `include` - Array of glob patterns for files to include in processing. Defaults to `['**/*.{ts,sql}']`, matching all `.ts` and `.sql` files. Can be overridden with the `--include` CLI argument. -- `exclude` - Array of glob patterns for files to exclude from processing. Defaults to `['**/node_modules/**']`, excluding `node_modules`. Can be overridden with the `--exclude` CLI argument. -- `since` - Limit matched files to those which have been changed since the given git ref. Use `"HEAD"` for files changed since the last commit, `"main"` for files changed in a branch, etc. Can be overridden with the `--since` CLI argument. -- `connectionURI` - URI for connecting to psql. Defaults to `postgresql://postgres:postgres@localhost:5432/postgres`. Note that if you are using `psql` inside docker, you should make sure that the container and host port match, since this will be used both by `psql` and slonik to connect to the database. -- `poolConfig` - Slonik database pool configuration. Will be used to create a pool which issues queries to the database as the tool is running, and will have its type parsers inspected to ensure the generated types are correct. It's important to pass in a pool confguration which is the same as the one used in your application. -- `psqlCommand` - the CLI command for running the official postgres `psql` CLI client. Defaults to `psql`. You can test it's working, and that your postgres version supports `\gdesc` with your connection string using: `echo 'select 123 as abc \gdesc' | psql "postgresql://postgres:postgres@localhost:5432/postgres" -f -`. Note that right now this can't contain single quotes. This should also be configured to talk to the same database as the `pool` variable (and it should be a development database - don't run this tool in production!). If you are using docker compose, you can use a command like `docker-compose exec -T postgres psql` -- `logger` - Logger object with `debug`, `info`, `warn` and `error` methods. Defaults to `console`. -- `writeTypes` (advanced/experimental) - Control how files are written to disk. See the [writeTypes](#writetypes) section. +There are some more configuration options [documented in code](./src/types.ts), but these should be considered experimental, and might change without warning. You can try them out as documented [below](#writetypes), but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way. ### Example config @@ -148,87 +175,6 @@ module.exports.default = { Note that the `/** @type {import('@slonik/typegen').Options} */` comment is optional, but will ensure your IDE gives you type hints. -### CLI options - -Some of the options above can be overriden by the CLI: - - -``` -usage: slonik-typegen generate [-h] [--config PATH] [--root-dir PATH] - [--connection-uri URI] [--psql COMMAND] - [--default-type TYPESCRIPT] [--include PATTERN] - [--exclude PATTERN] [--since REF] - [--migrate {<=0.8.0}] [--skip-check-clean] - [--watch] [--lazy] - - -Generates a directory containing with a 'sql' tag wrapper based on found -queries found in source files. By default, searches 'src' for source files. - -Optional arguments: - - -h, --help Show this help message and exit. - - --config PATH Path to a module containing parameters to be passed - to 'generate'. If specified, it will be required and - the export will be used as parameters. If not - specified, defaults will be used. Note: other CLI - arguments will override values set in this module - - --root-dir PATH Path to the source directory containing SQL queries. - Defaults to "src" if no value is provided - - --connection-uri URI URI for connecting to postgres. Defaults to - URI for connecting to postgres. Defaults to - - --psql COMMAND psql command used to query postgres via CLI client. e. - g. 'psql -h localhost -U postgres postgres' if - running postgres locally, or 'docker-compose exec -T - postgres psql -h localhost -U postgres postgres' if - running with docker-compose. You can test this by - running "<> -c 'select 1 as a, 2 - as b'". Note that this command will be executed - dynamically, so avoid using any escape characters in - here. - - --default-type TYPESCRIPT - TypeScript fallback type for when no type is found. - Most simple types (text, int etc.) are mapped to - their TypeScript equivalent automatically. This - should usually be 'unknown', or 'any' if you like to - live dangerously. - - --include PATTERN Glob pattern of files to search for SQL queries in. - By default searches for all .ts and .sql files: '**/*. - {ts,sql}' This option is repeatable to include - multiple patterns. - - --exclude PATTERN Glob pattern for files to be excluded from processing. - By default excludes '**/node_modules/**'. This - option is repeatable to exlude multiple patterns. - - --since REF Limit affected files to those which have been changed - since the given git ref. Use "--since HEAD" for files - changed since the last commit, "--since main for - files changed in a branch, etc. This option has no - effect in watch mode. - - --migrate {<=0.8.0} Before generating types, attempt to migrate a - codebase which has used a prior version of this tool - - --skip-check-clean If enabled, the tool will not check the git status to - ensure changes are checked in. - - --watch Run the type checker in watch mode. Files will be run - through the code generator when changed or added. - - --lazy Skip initial processing of input files. Only useful - with '--watch'. -``` - - -There are some more configuration options [documented in code](./src/types.ts), but these should be considered experimental, and might change without warning. You can try them out as documented below, but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way. - ### writeTypes The `writeTypes` option allows you to tweak what's written to disk. Note that the usage style isn't finalised and might change in future. If you use it, please create a discussion about it in https://github.com/mmkal/slonik-tools/discussions so that your use-case doesn't get taken away unexpectedly. From fc68043d2be8cf8b35bc718508566263c16f4298 Mon Sep 17 00:00:00 2001 From: Jan Paepke Date: Fri, 15 Apr 2022 11:53:05 +0200 Subject: [PATCH 4/8] Improved last paragraph. --- packages/typegen/readme.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/typegen/readme.md b/packages/typegen/readme.md index 4a7200a0..78330efe 100644 --- a/packages/typegen/readme.md +++ b/packages/typegen/readme.md @@ -383,12 +383,13 @@ type ResultingType = { } ``` -**On subsequent runs typegen will only update the first intersection type and leave all subsequent intersections untouched**. +**On subsequent runs typegen will only update the first intersection type and leave all following intersections untouched**. This also means you can make the column `string_col` non-nullable by intersecting it with `{ string_col: string }`. -Note that you can't completely change a type (say from `string` to `number`) this way. This is by design, because if you could, a change of the unterlying table might cause typegen to detect a new type, which would be ignored, if you could overwrite it. It is also why typegen only specifies a type when it's reasonably sure, as stated at the beginning of this paragraph. -If you found an example where this is not the case, please [raise an issue](https://github.com/mmkal/slonik-tools/issues/new). +Note that you can't completely change a property type (say from `string` to `number`) this way. +This is by design, because if you could, a change in the underlying table might cause typegen to detect a new type, which would be ignored, had you overwritten it. This would cause type changes to go unnoticed and we can't have that. +With intersections, the resulting property will be of type `never`, when an underlying column type changes. This will alert you to the change, so you can update your manual enhancements. ## Examples From 233ce0f6788817f0bf4592a1f6a27720773b8267 Mon Sep 17 00:00:00 2001 From: Misha Kaletsky Date: Fri, 15 Apr 2022 09:05:04 -0400 Subject: [PATCH 5/8] add high level test --- .../test/inline-tag-modification.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/typegen/test/inline-tag-modification.test.ts b/packages/typegen/test/inline-tag-modification.test.ts index 4b3aa6fb..09486d1b 100644 --- a/packages/typegen/test/inline-tag-modification.test.ts +++ b/packages/typegen/test/inline-tag-modification.test.ts @@ -74,4 +74,42 @@ describe('inline tag modification', () => { ) await checkModification('{col: string} & Other', 'queries.TestTable & Other') // we can't tell if the first intersection type is generated, so it will always be overwritten }) + + test('high-level', async () => { + const syncer = fsSyncer.jestFixture({ + targetState: { + 'index.ts': ` + import {sql} from 'slonik' + + export default sql<{} & {bar: number}>\`select foo, bar from test_table\` + `, + }, + }) + + syncer.sync() + + await typegen.generate(typegenOptions(syncer.baseDir)) + + expect(syncer.yaml()).toMatchInlineSnapshot(` + "--- + index.ts: |- + import {sql} from 'slonik' + + export default sql\`select foo, bar from test_table\` + + export declare namespace queries { + // Generated by @slonik/typegen + + /** - query: \`select foo, bar from test_table\` */ + export interface TestTable { + /** column: \`inline_tag_modification_test.test_table.foo\`, not null: \`true\`, regtype: \`integer\` */ + foo: number + + /** column: \`inline_tag_modification_test.test_table.bar\`, regtype: \`text\` */ + bar: string | null + } + } + " + `) + }) }) From 3cf9a80c9317016715117e0e118477899a3a5be6 Mon Sep 17 00:00:00 2001 From: Misha Kaletsky Date: Fri, 15 Apr 2022 12:55:12 -0400 Subject: [PATCH 6/8] oops --- packages/typegen/test/inline-tag-modification.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/typegen/test/inline-tag-modification.test.ts b/packages/typegen/test/inline-tag-modification.test.ts index 09486d1b..cadd8445 100644 --- a/packages/typegen/test/inline-tag-modification.test.ts +++ b/packages/typegen/test/inline-tag-modification.test.ts @@ -81,7 +81,7 @@ describe('inline tag modification', () => { 'index.ts': ` import {sql} from 'slonik' - export default sql<{} & {bar: number}>\`select foo, bar from test_table\` + export default sql<{} & {bar: string}>\`select foo, bar from test_table\` `, }, }) @@ -95,7 +95,7 @@ describe('inline tag modification', () => { index.ts: |- import {sql} from 'slonik' - export default sql\`select foo, bar from test_table\` + export default sql\`select foo, bar from test_table\` export declare namespace queries { // Generated by @slonik/typegen From 0e517127178b350ba85a2de6d55ea39163c947cb Mon Sep 17 00:00:00 2001 From: Jan Paepke Date: Sat, 16 Apr 2022 07:33:50 +0200 Subject: [PATCH 7/8] Update packages/typegen/src/write/inline.ts --- packages/typegen/src/write/inline.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/typegen/src/write/inline.ts b/packages/typegen/src/write/inline.ts index 5ca0b1fa..dff55ab3 100644 --- a/packages/typegen/src/write/inline.ts +++ b/packages/typegen/src/write/inline.ts @@ -99,8 +99,8 @@ export function getFileWriter({getQueriesModulePath = defaultGetQueriesModule, w // existing type definitions const [typeNode] = node.typeArguments if (ts.isIntersectionTypeNode(typeNode)) { - // we want to preserve intersection types - const [firstArg] = typeNode.types // We can't be sure the first argument is a generated type, but as the namespace might have been overwritten we're gonna have to assume. + // preserve intersection types + const [firstArg] = typeNode.types // Always overwrite the first type in the intersection, leave all subsequent ones alone. edits.push({ start: firstArg.getStart(sourceFile), end: firstArg.getEnd(), From e4b25f2a01cf139b99b15d914f6cd98c0ab87e6b Mon Sep 17 00:00:00 2001 From: Jan Paepke Date: Sat, 16 Apr 2022 09:48:05 +0200 Subject: [PATCH 8/8] Update packages/typegen/readme.md --- packages/typegen/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/typegen/readme.md b/packages/typegen/readme.md index 44596614..06019a5d 100644 --- a/packages/typegen/readme.md +++ b/packages/typegen/readme.md @@ -335,7 +335,7 @@ module.exports.default = { ## Enhancing Return Types -Typgen is designed to output types only to the degree it's certain they are correct. +Typegen is designed to output types only to the degree it's certain they are correct. Let's say in a complex query it can determine that a specific column will return a `string`, but isn't sure if it is also nullable, it will extract the type as `{ column: string | null }`, just to be on the safe side. When it encounters columns where it is unable to even determine the basic type, i.e. `json` columns, it will return :shrug: (Ok, actually the typescript equivalent, which is `unknown`).