Skip to content

Commit 1bffa12

Browse files
oscar60310oscar60310
committed
feat(ext-dbt): load dbt table, view, and ephemeral models
1 parent dedf73f commit 1bffa12

File tree

12 files changed

+221
-41
lines changed

12 files changed

+221
-41
lines changed

‎packages/core/src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ export * from './lib/data-query';
66
export * from './lib/data-source';
77
export * from './models';
88
export * from './containers';
9+
export * from './options';

‎packages/core/src/lib/template-engine/extension-loader/models.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,9 @@ export abstract class TagBuilder extends CompileTimeExtension {
6464

6565
export abstract class TagRunner extends RuntimeExtension {
6666
abstract tags: string[];
67-
abstract run(options: TagRunnerOptions): Promise<string | void>;
67+
abstract run(
68+
options: TagRunnerOptions
69+
): Promise<string | nunjucks.runtime.SafeString | void>;
6870

6971
public __run(...originalArgs: any[]) {
7072
const context = originalArgs[0];

‎packages/extension-dbt/src/index.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
import { DBTTagBuilder } from './lib/dbtTagBuilder';
2+
import { DBTTagRunner } from './lib/dbtTagRunner';
23

3-
export default [DBTTagBuilder];
4+
export default [DBTTagBuilder, DBTTagRunner];

‎packages/extension-dbt/src/lib/dbtTagBuilder.ts

+69-4
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,84 @@
1-
import { TagBuilder } from '@vulcan-sql/core';
1+
import {
2+
ITemplateEngineOptions,
3+
OnInit,
4+
TagBuilder,
5+
TYPES,
6+
} from '@vulcan-sql/core';
7+
import { inject, injectable } from 'inversify';
28
import * as nunjucks from 'nunjucks';
9+
import { promises as fs } from 'fs';
10+
import { chain } from 'lodash';
311

4-
export class DBTTagBuilder extends TagBuilder {
12+
@injectable()
13+
export class DBTTagBuilder extends TagBuilder implements OnInit {
514
public tags = ['dbt'];
15+
private modelFiles: string[] = [];
16+
private models = new Map<string, string>();
17+
18+
constructor(
19+
@inject(TYPES.TemplateEngineOptions) options: ITemplateEngineOptions
20+
) {
21+
super();
22+
this.modelFiles = options['dbt']?.modelFiles || [];
23+
}
24+
25+
public async onInit() {
26+
this.models.clear();
27+
for (const modelFile of this.modelFiles) {
28+
const content = JSON.parse(await fs.readFile(modelFile, 'utf-8'));
29+
chain(content.nodes || [])
30+
.toPairs()
31+
.filter((node) => node[0].startsWith('model'))
32+
.forEach((node) => this.loadModel(node[0], node[1]))
33+
.value();
34+
}
35+
}
636

737
public parse(
838
parser: nunjucks.parser.Parser,
939
nodes: typeof nunjucks.nodes,
1040
lexer: typeof nunjucks.lexer
1141
) {
12-
// {% dbt model-name %}
42+
// {% dbt "model-name" %}
1343
// consume dbt tag
1444
const dbtToken = parser.nextToken();
1545
const args = new nodes.NodeList(dbtToken.lineno, dbtToken.colno);
46+
const modelNameToken = parser.nextToken();
47+
if (modelNameToken.type !== lexer.TOKEN_STRING) {
48+
parser.fail(
49+
`Expect model name as string, but got ${modelNameToken.type}`,
50+
modelNameToken.lineno,
51+
modelNameToken.colno
52+
);
53+
}
54+
const end = parser.nextToken();
55+
if (end.type !== lexer.TOKEN_BLOCK_END) {
56+
parser.fail(
57+
`Expect block end %}, but got ${end.type}`,
58+
end.lineno,
59+
end.colno
60+
);
61+
}
62+
63+
const sql = this.models.get(modelNameToken.value);
64+
if (!sql) {
65+
parser.fail(
66+
`Model ${modelNameToken.value} is not found in modelFiles`,
67+
modelNameToken.lineno,
68+
modelNameToken.colno
69+
);
70+
}
71+
72+
const output = new nodes.Output(dbtToken.lineno, dbtToken.colno);
73+
output.addChild(
74+
new nodes.TemplateData(dbtToken.lineno, dbtToken.colno, sql)
75+
);
76+
return this.createAsyncExtensionNode(args, [output]);
77+
}
1678

17-
return this.createAsyncExtensionNode(args, []);
79+
private loadModel(name: string, node: any) {
80+
if (this.models.has(name))
81+
throw Error(`Model name ${name} is unambiguous.`);
82+
this.models.set(name, node.relation_name || `(${node.compiled_sql})`);
1883
}
1984
}

‎packages/extension-dbt/src/lib/dbtTagRunner.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { TagRunner, TagRunnerOptions } from '@vulcan-sql/core';
2+
import { injectable } from 'inversify';
3+
import * as nunjucks from 'nunjucks';
4+
5+
@injectable()
6+
export class DBTTagRunner extends TagRunner {
7+
public tags = ['dbt'];
8+
9+
public async run({ contentArgs }: TagRunnerOptions) {
10+
const sql = await contentArgs[0]();
11+
return new nunjucks.runtime.SafeString(sql);
12+
}
13+
}

‎packages/extension-dbt/test/dbt.spec.ts

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import { getTestCompiler } from '@vulcan-sql/test-utility';
2+
import * as path from 'path';
3+
4+
it('Should replace with table name of dbt model with type table', async () => {
5+
// Arrange
6+
const { compileAndLoad, execute, getExecutedQueries } = await getTestCompiler(
7+
{
8+
extensionNames: [path.join(__dirname, '..', 'src')],
9+
config: {
10+
dbt: {
11+
modelFiles: [path.join(__dirname, 'test-artifact.json')],
12+
},
13+
},
14+
}
15+
);
16+
17+
// Act.
18+
await compileAndLoad(`select * from {% dbt "model.test.1_table" %}`);
19+
await execute({});
20+
21+
// Assert
22+
const queries = await getExecutedQueries();
23+
expect(queries[0]).toBe('select * from "postgres"."public"."1_table"');
24+
});
25+
26+
it('Should replace with table name of dbt model with type view', async () => {
27+
// Arrange
28+
const { compileAndLoad, execute, getExecutedQueries } = await getTestCompiler(
29+
{
30+
extensionNames: [path.join(__dirname, '..', 'src')],
31+
config: {
32+
dbt: {
33+
modelFiles: [path.join(__dirname, 'test-artifact.json')],
34+
},
35+
},
36+
}
37+
);
38+
39+
// Act.
40+
await compileAndLoad(`select * from {% dbt "model.test.2_view" %}`);
41+
await execute({});
42+
43+
// Assert
44+
const queries = await getExecutedQueries();
45+
expect(queries[0]).toBe('select * from "postgres"."public"."2_view"');
46+
});
47+
48+
it('Should replace with sub-query of dbt model with type ephemeral', async () => {
49+
// Arrange
50+
const { compileAndLoad, execute, getExecutedQueries } = await getTestCompiler(
51+
{
52+
extensionNames: [path.join(__dirname, '..', 'src')],
53+
config: {
54+
dbt: {
55+
modelFiles: [path.join(__dirname, 'test-artifact.json')],
56+
},
57+
},
58+
}
59+
);
60+
61+
// Act.
62+
await compileAndLoad(
63+
`select sub.* from {% dbt "model.test.3_ephemeral" %} as sub`
64+
);
65+
await execute({});
66+
67+
// Assert
68+
const queries = await getExecutedQueries();
69+
expect(queries[0]).toBe(`select sub.* from (
70+
select *
71+
from "postgres"."public"."1_table"
72+
where age <= 18) as sub`);
73+
});

‎packages/extension-dbt/test/dbt_models/1_table.sql

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{{ config(materialized='table') }}
2+
3+
with source_data as (
4+
select 1 as id, 'Ivan' as name, 18 as age
5+
UNION
6+
select 2 as id, 'William' as name, 80 as age
7+
UNION
8+
select 3 as id, 'Eason' as name, 18 as age
9+
)
10+
11+
select * from source_data

‎packages/extension-dbt/test/dbt_models/2_view.sql

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{{ config(materialized='view') }}
2+
3+
select *
4+
from {{ ref('1_table') }}
5+
where age <= 18

‎packages/extension-dbt/test/dbt_models/3_ephemeral.sql

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{{ config(materialized='ephemeral') }}
2+
3+
select *
4+
from {{ ref('1_table') }}
5+
where age <= 18

‎tsconfig.base.json

+35-34
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,43 @@
1717
"types": ["reflect-metadata"],
1818
"paths": {
1919
"@vulcan-sql/build": ["packages/build/src/index"],
20+
"@vulcan-sql/build/containers": ["packages/build/src/containers/index"],
21+
"@vulcan-sql/build/models": ["packages/build/src/models/index"],
22+
"@vulcan-sql/build/options": ["packages/build/src/options/index"],
23+
"@vulcan-sql/build/schema-parser": [
24+
"packages/build/src/lib/schema-parser/index"
25+
],
26+
"@vulcan-sql/build/schema-parser/*": [
27+
"packages/build/src/lib/schema-parser/*"
28+
],
29+
"@vulcan-sql/build/spec-generator": [
30+
"packages/build/src/lib/spec-generator/index"
31+
],
32+
"@vulcan-sql/build/spec-generator/*": [
33+
"packages/build/src/lib/spec-generator/*"
34+
],
2035
"@vulcan-sql/core": ["packages/core/src/index"],
21-
"@vulcan-sql/serve": ["packages/serve/src/index"],
22-
"@vulcan-sql/extension-dbt": ["packages/extension-dbt/src/index"],
2336
"@vulcan-sql/core/artifact-builder": [
2437
"packages/core/src/lib/artifact-builder/index"
2538
],
2639
"@vulcan-sql/core/artifact-builder/*": [
2740
"packages/core/src/lib/artifact-builder/*"
2841
],
42+
"@vulcan-sql/core/containers": ["packages/core/src/containers/index"],
43+
"@vulcan-sql/core/data-query": ["packages/core/src/lib/data-query/index"],
44+
"@vulcan-sql/core/data-query/*": ["packages/core/src/lib/data-query/*"],
45+
"@vulcan-sql/core/data-source": [
46+
"packages/core/src/lib/data-source/index"
47+
],
48+
"@vulcan-sql/core/data-source/*": ["packages/core/src/lib/data-source/*"],
49+
"@vulcan-sql/core/models": ["packages/core/src/models/index"],
2950
"@vulcan-sql/core/template-engine": [
3051
"packages/core/src/lib/template-engine/index"
3152
],
3253
"@vulcan-sql/core/template-engine/*": [
3354
"packages/core/src/lib/template-engine/*"
3455
],
56+
"@vulcan-sql/core/utils": ["packages/core/src/lib/utils/index"],
3557
"@vulcan-sql/core/validators": ["packages/core/src/lib/validators/index"],
3658
"@vulcan-sql/core/validators/*": ["packages/core/src/lib/validators/*"],
3759
"@vulcan-sql/core/validators/built-in-validators": [
@@ -40,45 +62,24 @@
4062
"@vulcan-sql/core/validators/built-in-validators/*": [
4163
"packages/core/src/lib/validators/built-in-validators/*"
4264
],
43-
"@vulcan-sql/core/data-query": ["packages/core/src/lib/data-query/index"],
44-
"@vulcan-sql/core/data-query/*": ["packages/core/src/lib/data-query/*"],
45-
"@vulcan-sql/core/data-source": [
46-
"packages/core/src/lib/data-source/index"
47-
],
48-
"@vulcan-sql/core/data-source/*": ["packages/core/src/lib/data-source/*"],
49-
"@vulcan-sql/core/containers": ["packages/core/src/containers/index"],
50-
"@vulcan-sql/core/models": ["packages/core/src/models/index"],
51-
"@vulcan-sql/core/utils": ["packages/core/src/lib/utils/index"],
52-
"@vulcan-sql/build/schema-parser": [
53-
"packages/build/src/lib/schema-parser/index"
54-
],
55-
"@vulcan-sql/build/schema-parser/*": [
56-
"packages/build/src/lib/schema-parser/*"
57-
],
58-
"@vulcan-sql/build/spec-generator": [
59-
"packages/build/src/lib/spec-generator/index"
60-
],
61-
"@vulcan-sql/build/spec-generator/*": [
62-
"packages/build/src/lib/spec-generator/*"
65+
"@vulcan-sql/extension-dbt": ["packages/extension-dbt/src/index"],
66+
"@vulcan-sql/serve": ["packages/serve/src/index"],
67+
"@vulcan-sql/serve/app": ["packages/serve/src/lib/app.ts"],
68+
"@vulcan-sql/serve/config": ["packages/serve/src/lib/config"],
69+
"@vulcan-sql/serve/containers": ["packages/serve/src/containers/index"],
70+
"@vulcan-sql/serve/middleware": [
71+
"packages/serve/src/lib/middleware/index"
6372
],
64-
"@vulcan-sql/build/models": ["packages/build/src/models/index"],
65-
"@vulcan-sql/build/containers": ["packages/build/src/containers/index"],
66-
"@vulcan-sql/build/options": ["packages/build/src/options/index"],
73+
"@vulcan-sql/serve/middleware/*": ["packages/serve/src/lib/middleware/*"],
6774
"@vulcan-sql/serve/models": ["packages/serve/src/models/index"],
68-
"@vulcan-sql/serve/containers": ["packages/serve/src/containers/index"],
6975
"@vulcan-sql/serve/options": ["packages/serve/src/options/index"],
70-
"@vulcan-sql/serve/route": ["packages/serve/src/lib/route/index"],
71-
"@vulcan-sql/serve/route/*": ["packages/serve/src/lib/route/*"],
72-
"@vulcan-sql/serve/app": ["packages/serve/src/lib/app.ts"],
7376
"@vulcan-sql/serve/pagination": [
7477
"packages/serve/src/lib/pagination/index"
7578
],
7679
"@vulcan-sql/serve/pagination/*": ["packages/serve/src/lib/pagination/*"],
77-
"@vulcan-sql/serve/middleware": [
78-
"packages/serve/src/lib/middleware/index"
79-
],
80-
"@vulcan-sql/serve/middleware/*": ["packages/serve/src/lib/middleware/*"],
81-
"@vulcan-sql/serve/config": ["packages/serve/src/lib/config"]
80+
"@vulcan-sql/serve/route": ["packages/serve/src/lib/route/index"],
81+
"@vulcan-sql/serve/route/*": ["packages/serve/src/lib/route/*"],
82+
"@vulcan-sql/test-utility": ["packages/test-utility/src/index.ts"]
8283
}
8384
},
8485
"exclude": ["node_modules", "tmp"]

‎types/nunjucks.d.ts

+2
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,8 @@ declare module 'nunjucks' {
368368
}
369369

370370
class TemplateData extends Literal {}
371+
372+
class Output extends NodeList {}
371373
}
372374

373375
namespace lexer {

‎workspace.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"build": "packages/build",
55
"core": "packages/core",
66
"extension-dbt": "packages/extension-dbt",
7-
"serve": "packages/serve"
7+
"serve": "packages/serve",
8+
"test-utility": "packages/test-utility"
89
}
910
}

0 commit comments

Comments
 (0)