Skip to content

Commit f85675d

Browse files
cornermancornerman
authored
use schemacrawler and let users define template for code generation with scalate (#2)
1 parent 25c469c commit f85675d

File tree

13 files changed

+624
-393
lines changed

13 files changed

+624
-393
lines changed

‎README.md

Lines changed: 173 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,64 @@
1-
# scala-quillcodegen
1+
# scala-db-codegen
22

3-
This is an sbt-plugin and mill-plugin that uses the [quill-codegen-jdbc](https://zio.dev/zio-quill/code-generation/) to generate case classes and query schemas from a database schema.
3+
A sbt-plugin and mill-plugin to generate boilerplate code from a database schema. Tested with SQLite and Postgresql. Should work for all databases supported by jdbc.
4+
5+
> DbSchema + Template => generate scala code
6+
7+
The plugin can be configured to crawl a database schema to extract all tables/columns/enums. It matches the occurring jdbc/sql types to scala types.
8+
You can provide a [scalate](https://scalate.github.io/scalate/) template to generate scala code out of this information like this:
9+
```scala
10+
<%@ val schema: dbcodegen.DataSchema %>
11+
12+
package kicks.db.${schema.name}
13+
14+
#for (table <- schema.tables)
15+
case class ${table.scalaName}(
16+
#for (column <- table.columns)
17+
${column.scalaName}: ${column.scalaType},
18+
#end
19+
)
20+
#end
21+
```
422

5-
Works with scala 2 and 3.
623

724
## Usage
825

926
### sbt
1027

1128
In `project/plugins.sbt`:
1229
```sbt
13-
addSbtPlugin("com.github.cornerman" % "sbt-quillcodegen" % "0.2.0")
30+
addSbtPlugin("com.github.cornerman" % "sbt-db-codegen" % "0.2.0")
1431
```
1532

1633
In `build.sbt`:
1734
```sbt
1835
lazy val db = project
19-
.enablePlugins(quillcodegen.plugin.CodegenPlugin)
36+
.enablePlugins(dbcodegen.plugin.DbCodegenPlugin)
2037
.settings(
21-
// The package prefix for the generated code
22-
quillcodegenPackagePrefix := "com.example.db",
2338
// The jdbc URL for the database
24-
quillcodegenJdbcUrl := "jdbc:...",
39+
dbcodegenJdbcUrl := "jdbc:...",
40+
// The template file for the code generator
41+
dbcodegenTemplateFiles := Seq(file("schema.scala.ssp"))
2542

2643
// Optional database username
27-
// quillcodegenUsername := None,
44+
// dbcodegenUsername := None,
2845
// Optional database password
29-
// quillcodegenPassword := None,
30-
// The naming parser to use, default is SnakeCaseNames
31-
// quillcodegenNaming := SnakeCaseNames,
32-
// Whether to generate a nested extensions trait, default is false
33-
// quillcodegenNestedTrait := false,
34-
// Whether to generate query schemas, default is true
35-
// quillcodegenGenerateQuerySchema := true,
36-
// Specify which tables to process, default is all
37-
// quillcodegenTableFilter := (_ => true),
38-
// Strategy for unrecognized types
39-
// quillcodegenUnrecognizedType := SkipColumn,
40-
// Map jdbc types to java/scala types
41-
// quillcodegenTypeMapping := ((_, classTag) => classTag),
42-
// Which numeric type preference for numeric types
43-
// quillcodegenNumericType := UseDefaults,
44-
// Timeout for the generate task
45-
// quillcodegenTimeout := Duration.Inf,
46+
// dbcodegenPassword := None,
47+
// Map sql types to java/scala types
48+
// dbcodegenTypeMapping := (sqlType: SQLType, scalaType: Option[String]) => scalaType,
49+
// Filter which schema and table should be processed
50+
// dbcodegenSchemaTableFilter := (schema: String, table: String) => true
4651
// Setup task to be executed before the code generation runs against the database
47-
// quillcodegenSetupTask := {},
52+
// dbcodegenSetupTask := {},
4853
)
4954
```
5055

5156
#### Setup database before codegen
5257

53-
An example for using the `quillcodegenSetupTask` to setup an sqlite database with a `schema.sql` file before the code generation runs:
58+
An example for using the `dbcodegenSetupTask` to setup an sqlite database with a `schema.sql` file before the code generation runs:
5459
```sbt
55-
quillcodegenSetupTask := Def.taskDyn {
56-
IO.delete(file(quillcodegenJdbcUrl.value.stripPrefix("jdbc:sqlite:")))
60+
dbcodegenSetupTask := Def.taskDyn {
61+
IO.delete(file(dbcodegenJdbcUrl.value.stripPrefix("jdbc:sqlite:")))
5762
executeSqlFile(file("./schema.sql"))
5863
}
5964
```
@@ -64,17 +69,148 @@ The functions `executeSql` and `executeSqlFile` are provided for these kind of u
6469
### mill
6570

6671
In `build.sc`:
67-
```
72+
```scala
6873
import mill._, scalalib._
69-
import $ivy.`com.github.cornerman::mill-quillcodegen:0.2.0`, quillcodegen.plugin.QuillCodegenModule
74+
import $ivy.`com.github.cornerman::mill-db-codegen:0.2.0`, dbcodegen.plugin.DbCodegenModule
7075

71-
object backend extends ScalaModule with QuillCodegenModule {
72-
def quillcodegenJdbcUrl = "com.example.db",
73-
def quillcodegenPackagePrefix = "dbtypes"
74-
def quillcodegenSetupTask = T.task {
75-
val dbpath = quillcodegenJdbcUrl.stripPrefix("jdbc:sqlite:")
76+
object backend extends ScalaModule with DbCodegenModule {
77+
// The jdbc URL for the database
78+
def dbcodegenJdbcUrl = "jdbc:sqlite:..."
79+
// The template file for the code generator
80+
def dbcodegenTemplateFiles = Seq(PathRef(os.pwd / "schema.scala.ssp"))
81+
// Setup task to be executed before the code generation runs against the database
82+
def dbcodegenSetupTask = T.task {
83+
val dbpath = dbcodegenJdbcUrl.stripPrefix("jdbc:sqlite:")
7684
os.remove(os.pwd / dbpath)
7785
executeSqlFile(PathRef(os.pwd / "schema.sql"))
7886
}
87+
// Optional database username
88+
// def dbcodegenUsername = None
89+
// Optional database password
90+
// def dbcodegenPassword = None
91+
// Map sql types to java/scala types
92+
// def dbcodegenTypeMapping = (sqlType: SQLType, scalaType: Option[String]) => scalaType
93+
// Filter which schema and table should be processed
94+
// def dbcodegenSchemaTableFilter = (schema: String, table: String) => true
7995
}
8096
```
97+
98+
## Template Examples
99+
100+
Template can be configured by setting `dbcodegenTemplateFiles`.
101+
102+
We are using [scalate](https://scalate.github.io/scalate/) for templates, so you can use anything that is supported there (e.g. `mustache` or `ssp`) - the converter will be picked according to the file extension of the provided template file. Check the [scalate user guide](https://scalate.github.io/scalate/documentation/user-guide.html) for more details.
103+
104+
The template is called on each database schema, and is passed an instance of [`dbcodegen.DataSchema`](codegen/src/main/scala/dbcodegen/DataSchema.scala) (variable name `schema`) which contains all the extracted information.
105+
You can see the declaration in the first line of each `ssp` template.
106+
107+
### Simple
108+
109+
case-classes.scala.ssp:
110+
```scala
111+
<%@ val schema: dbcodegen.DataSchema %>
112+
113+
package kicks.db.${schema.name}
114+
115+
#for (enum <- schema.enums)
116+
type ${enum.scalaName} = ${enum.values.map(v => "\"" + v.name + "\"").mkString(" | ")}
117+
#end
118+
119+
#for (table <- schema.tables)
120+
121+
case class ${table.scalaName}(
122+
#for (column <- table.columns)
123+
${column.scalaName}: ${column.scalaType},
124+
#end
125+
)
126+
127+
#end
128+
```
129+
130+
#### with scala 3 enums
131+
132+
```scala
133+
#for (enum <- schema.enums)
134+
135+
enum ${enum.scalaName}(val sqlValue: String) {
136+
#for (enumValue <- enum.values)
137+
case ${enumValue.scalaName} extends ${enum.scalaName}("${enumValue.name}")
138+
#end
139+
}
140+
object ${enum.scalaName} {
141+
def bySqlValue(searchValue: String): Option[${enum.scalaName}] = values.find(_.sqlValue == searchValue)
142+
}
143+
144+
#end
145+
```
146+
147+
### Library: quill
148+
149+
quill-case-classes.scala.ssp:
150+
```scala
151+
<%@ val schema: dbcodegen.DataSchema %>
152+
153+
package kicks.db.${schema.scalaName}
154+
155+
import io.getquill.*
156+
157+
#for (enum <- schema.enums)
158+
type ${enum.scalaName} = ${enum.values.map(v => "\"" + v.name + "\"").mkString(" | ")}
159+
#end
160+
161+
#for (table <- schema.tables)
162+
163+
case class ${table.scalaName}(
164+
#for (column <- table.columns)
165+
${column.scalaName}: ${column.scalaType},
166+
#end
167+
)
168+
object ${table.scalaName} {
169+
inline def query = querySchema[Person](
170+
"${table.name}",
171+
#for (column <- table.columns)
172+
_.${column.scalaName} -> "${column.name}",
173+
#end
174+
)
175+
}
176+
177+
#end
178+
```
179+
180+
### Library: magnum
181+
182+
magnum-case-classes.scala.ssp:
183+
```scala
184+
<%@ val schema: dbcodegen.DataSchema %>
185+
186+
package kicks.db.${schema.scalaName}
187+
188+
import com.augustnagro.magnum.*
189+
190+
#for (enum <- schema.enums)
191+
type ${enum.scalaName} = ${enum.values.map(v => "\"" + v.name + "\"").mkString(" | ")}
192+
#end
193+
194+
#for (table <- schema.tables)
195+
196+
@Table(SqliteDbType)
197+
case class ${table.scalaName}(
198+
#for (column <- table.columns)
199+
@SqlName("${column.name}") ${column.scalaName}: ${column.scalaType},
200+
#end
201+
) derives DbCodec
202+
object ${table.scalaName} {
203+
#{ val primaryKeyColumns = table.columns.filter(_.isPartOfPrimaryKey)}#
204+
type Id = ${if (primaryKeyColumns.isEmpty) "Null" else primaryKeyColumns.map(_.scalaType).mkString("(", ", ", ")")}
205+
206+
case class Creator(
207+
#for (column <- table.columns if !column.isAutoGenerated)
208+
${column.scalaName}: ${column.scalaType},
209+
#end
210+
)
211+
}
212+
213+
val ${table.scalaName}Repo = Repo[${table.scalaName}.Creator, ${table.scalaName}, ${table.scalaName}.Id]
214+
215+
#end
216+
```

‎build.sbt

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@ inThisBuild(
44
Seq(
55
organization := "com.github.cornerman",
66
licenses := Seq("MIT License" -> url("https://opensource.org/licenses/MIT")),
7-
homepage := Some(url("https://github.com/cornerman/sbt-quillcodegen")),
7+
homepage := Some(url("https://github.com/cornerman/scala-db-codegen")),
88
scmInfo := Some(
99
ScmInfo(
10-
url("https://github.com/cornerman/sbt-quillcodegen"),
11-
"scm:git:git@github.com:cornerman/sbt-quillcodegen.git",
12-
Some("scm:git:git@github.com:cornerman/sbt-quillcodegen.git"),
10+
url("https://github.com/cornerman/scala-db-codegen"),
11+
"scm:git:git@github.com:cornerman/scala-db-codegen.git",
12+
Some("scm:git:git@github.com:cornerman/scala-db-codegen.git"),
1313
)
1414
),
1515
pomExtra :=
@@ -26,16 +26,19 @@ inThisBuild(
2626
// TODO: Use sbt-cross to workaround: https://github.com/sbt/sbt/issues/5586
2727
lazy val codegen = project
2828
.settings(
29-
name := "quillcodegen",
29+
name := "scala-db-codegen",
3030
libraryDependencies ++= Seq(
31-
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
32-
// Should be same as in Codegen.scala for generated code
33-
"io.getquill" %% "quill-codegen-jdbc" % "4.8.1",
34-
"org.xerial" % "sqlite-jdbc" % "3.44.1.0",
35-
"org.postgresql" % "postgresql" % "42.7.1",
36-
"mysql" % "mysql-connector-java" % "8.0.33",
37-
"org.mariadb.jdbc" % "mariadb-java-client" % "3.1.2",
38-
"org.mybatis" % "mybatis" % "3.5.15",
31+
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
32+
"org.scalatra.scalate" %% "scalate-core" % "1.10.1",
33+
"org.mybatis" % "mybatis" % "3.5.15",
34+
"org.xerial" % "sqlite-jdbc" % "3.44.1.0",
35+
"org.postgresql" % "postgresql" % "42.7.1",
36+
"mysql" % "mysql-connector-java" % "8.0.33",
37+
"org.mariadb.jdbc" % "mariadb-java-client" % "3.1.2",
38+
"us.fatehi" % "schemacrawler-tools" % "16.21.1",
39+
"us.fatehi" % "schemacrawler-postgresql" % "16.21.1",
40+
"us.fatehi" % "schemacrawler-sqlite" % "16.21.1",
41+
"us.fatehi" % "schemacrawler-mysql" % "16.21.1",
3942
),
4043
)
4144
.cross
@@ -45,7 +48,7 @@ lazy val codegen213 = codegen("2.13.13")
4548

4649
lazy val pluginSbt = project
4750
.settings(
48-
name := "sbt-quillcodegen",
51+
name := "sbt-db-codegen",
4952
scalaVersion := "2.12.19",
5053
crossScalaVersions := Seq("2.12.19"),
5154
sbtPlugin := true,
@@ -55,7 +58,7 @@ lazy val pluginSbt = project
5558

5659
lazy val pluginMill = project
5760
.settings(
58-
name := "mill-quillcodegen",
61+
name := "mill-db-codegen",
5962
scalaVersion := "2.13.13",
6063
crossScalaVersions := Seq("2.13.13"),
6164
libraryDependencies ++= Seq(

‎codegen/src/main/scala/SchemaCrawlerExt.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package schemacrawler.crawl
2+
3+
import schemacrawler.schema.{ColumnDataType, DataTypeType, Schema}
4+
5+
object SchemaCrawlerExt {
6+
def newColumnDataType(schema: Schema, name: String, tpe: DataTypeType): ColumnDataType =
7+
new MutableColumnDataType(schema, name, tpe)
8+
}

‎codegen/src/main/scala/dbcodegen/CodeGenerator.scala

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package dbcodegen
2+
3+
import schemacrawler.schemacrawler._
4+
import schemacrawler.tools.databaseconnector.{DatabaseConnectorRegistry, DatabaseUrlConnectionOptions}
5+
import schemacrawler.tools.utility.SchemaCrawlerUtility
6+
import us.fatehi.utility.datasource.MultiUseUserCredentials
7+
import org.fusesource.scalate.{TemplateEngine, TemplateSource}
8+
9+
import java.io.File
10+
import java.nio.file.{Files, Path, Paths}
11+
import java.sql.SQLType
12+
import scala.jdk.CollectionConverters._
13+
14+
case class DbConfig(
15+
jdbcUrl: String,
16+
username: Option[String],
17+
password: Option[String],
18+
)
19+
20+
case class CodeGeneratorConfig(
21+
templateFiles: Seq[File],
22+
outDir: File,
23+
typeMapping: (SQLType, Option[String]) => Option[String],
24+
schemaTableFilter: (String, String) => Boolean,
25+
)
26+
27+
object CodeGenerator {
28+
def generate(db: DbConfig, config: CodeGeneratorConfig): Seq[Path] = {
29+
// schema crawler options
30+
31+
val credentials = new MultiUseUserCredentials(db.username.orNull, db.password.orNull)
32+
val connection =
33+
DatabaseConnectorRegistry
34+
.getDatabaseConnectorRegistry()
35+
.findDatabaseConnectorFromUrl(db.jdbcUrl)
36+
.newDatabaseConnectionSource(new DatabaseUrlConnectionOptions(db.jdbcUrl), credentials)
37+
38+
val schemaCrawlerOptions = SchemaCrawlerOptionsBuilder
39+
.newSchemaCrawlerOptions()
40+
.withLoadOptions(LoadOptionsBuilder.builder().withInfoLevel(InfoLevel.maximum).toOptions)
41+
.withLimitOptions(LimitOptionsBuilder.builder().toOptions)
42+
43+
val catalog = SchemaCrawlerUtility.getCatalog(connection, schemaCrawlerOptions)
44+
45+
// scalate
46+
47+
val templateEngine = new TemplateEngine()
48+
val templateSources = config.templateFiles.map(TemplateSource.fromFile)
49+
50+
// run template on schemas
51+
52+
val dataSchemas = catalog.getSchemas.asScala.map { schema =>
53+
SchemaConverter.toDataSchema(schema, connection, catalog.getTables(schema).asScala.toSeq, config)
54+
}
55+
56+
dataSchemas.flatMap { dataSchema =>
57+
val data = Map(
58+
"schema" -> dataSchema
59+
)
60+
61+
templateSources.map { templateSource =>
62+
val output = templateEngine.layout(templateSource, data)
63+
val outputPath = Paths.get(config.outDir.getPath, templateSource.file.getPath, s"${dataSchema.name}.scala")
64+
65+
Files.createDirectories(outputPath.getParent)
66+
Files.write(outputPath, output.getBytes)
67+
68+
outputPath
69+
}
70+
}.toSeq
71+
}
72+
73+
}

0 commit comments

Comments
 (0)