generic template for code generation

README.md

@@ -1,59 +1,64 @@
-# scala-quillcodegen
+# scala-db-codegen
 
-This is an sbt-plugin and mill-plugin that uses the [quill-codegen-jdbc](https://zio.dev/zio-quill/code-generation/) to generate case classes and query schemas from a database schema.
+A sbt-plugin and mill-plugin to generate boilerplate code from a database schema. Tested with SQLite and Postgresql. Should work for all databases supported by jdbc.
+
+> DbSchema + Template => generate scala code
+
+The plugin can be configured to crawl a database schema to extract all tables/columns/enums. It matches the occurring jdbc/sql types to scala types.
+You can provide a [scalate](https://scalate.github.io/scalate/) template to generate scala code out of this information like this:
+```scala
+<%@ val schema: dbcodegen.DataSchema %>
+
+package kicks.db.${schema.name}
+
+#for (table <- schema.tables)
+case class ${table.scalaName}(
+  #for (column <- table.columns)
+  ${column.scalaName}: ${column.scalaType},
+  #end
+)
+#end
+```
 
-Works with scala 2 and 3.
 
 ## Usage
 
 ### sbt
 
 In `project/plugins.sbt`:
 ```sbt
-addSbtPlugin("com.github.cornerman" % "sbt-quillcodegen" % "0.2.0")
+addSbtPlugin("com.github.cornerman" % "sbt-db-codegen" % "0.2.0")
 ```
 
 In `build.sbt`:
 ```sbt
 lazy val db = project
-  .enablePlugins(quillcodegen.plugin.CodegenPlugin)
+  .enablePlugins(dbcodegen.plugin.DbCodegenPlugin)
   .settings(
-    // The package prefix for the generated code
-    quillcodegenPackagePrefix := "com.example.db",
     // The jdbc URL for the database
-    quillcodegenJdbcUrl := "jdbc:...",
+    dbcodegenJdbcUrl := "jdbc:...",
+    // The template file for the code generator
+    dbcodegenTemplateFiles := Seq(file("schema.scala.ssp"))
 
     // Optional database username
-    // quillcodegenUsername            := None,
+    // dbcodegenUsername          := None,
     // Optional database password
-    // quillcodegenPassword            := None,
-    // The naming parser to use, default is SnakeCaseNames
-    // quillcodegenNaming              := SnakeCaseNames,
-    // Whether to generate a nested extensions trait, default is false
-    // quillcodegenNestedTrait         := false,
-    // Whether to generate query schemas, default is true
-    // quillcodegenGenerateQuerySchema := true,
-    // Specify which tables to process, default is all
-    // quillcodegenTableFilter         := (_ => true),
-    // Strategy for unrecognized types
-    // quillcodegenUnrecognizedType    := SkipColumn,
-    // Map jdbc types to java/scala types
-    // quillcodegenTypeMapping         := ((_, classTag) => classTag),
-    // Which numeric type preference for numeric types
-    // quillcodegenNumericType         := UseDefaults,
-    // Timeout for the generate task
-    // quillcodegenTimeout             := Duration.Inf,
+    // dbcodegenPassword          := None,
+    // Map sql types to java/scala types
+    // dbcodegenTypeMapping       := (sqlType: SQLType, scalaType: Option[String]) => scalaType,
+    // Filter which schema and table should be processed
+    // dbcodegenSchemaTableFilter := (schema: String, table: String) => true
     // Setup task to be executed before the code generation runs against the database
-    // quillcodegenSetupTask           := {},
+    // dbcodegenSetupTask         := {},
   )
 ```
 
 #### Setup database before codegen
 
-An example for using the `quillcodegenSetupTask` to setup an sqlite database with a `schema.sql` file before the code generation runs:
+An example for using the `dbcodegenSetupTask` to setup an sqlite database with a `schema.sql` file before the code generation runs:
 ```sbt
-quillcodegenSetupTask := Def.taskDyn {
-    IO.delete(file(quillcodegenJdbcUrl.value.stripPrefix("jdbc:sqlite:")))
+dbcodegenSetupTask := Def.taskDyn {
+    IO.delete(file(dbcodegenJdbcUrl.value.stripPrefix("jdbc:sqlite:")))
     executeSqlFile(file("./schema.sql"))
 }
 ```
@@ -64,17 +69,148 @@ The functions `executeSql` and `executeSqlFile` are provided for these kind of u
 ### mill
 
 In `build.sc`:
-```
+```scala
 import mill._, scalalib._
-import $ivy.`com.github.cornerman::mill-quillcodegen:0.2.0`, quillcodegen.plugin.QuillCodegenModule
+import $ivy.`com.github.cornerman::mill-db-codegen:0.2.0`, dbcodegen.plugin.DbCodegenModule
 
-object backend extends ScalaModule with QuillCodegenModule {
-  def quillcodegenJdbcUrl       = "com.example.db",
-  def quillcodegenPackagePrefix = "dbtypes"
-  def quillcodegenSetupTask = T.task {
-    val dbpath = quillcodegenJdbcUrl.stripPrefix("jdbc:sqlite:")
+object backend extends ScalaModule with DbCodegenModule {
+  // The jdbc URL for the database
+  def dbcodegenJdbcUrl       = "jdbc:sqlite:..."
+  // The template file for the code generator
+  def dbcodegenTemplateFiles = Seq(PathRef(os.pwd / "schema.scala.ssp"))
+  // Setup task to be executed before the code generation runs against the database
+  def dbcodegenSetupTask = T.task {
+    val dbpath = dbcodegenJdbcUrl.stripPrefix("jdbc:sqlite:")
     os.remove(os.pwd / dbpath)
     executeSqlFile(PathRef(os.pwd / "schema.sql"))
   }
+  // Optional database username
+  // def dbcodegenUsername = None
+  // Optional database password
+  // def dbcodegenPassword = None
+  // Map sql types to java/scala types
+  // def dbcodegenTypeMapping = (sqlType: SQLType, scalaType: Option[String]) => scalaType
+  // Filter which schema and table should be processed
+  // def dbcodegenSchemaTableFilter = (schema: String, table: String) => true
 }
 ```
+
+## Template Examples
+
+Template can be configured by setting `dbcodegenTemplateFiles`.
+
+We are using [scalate](https://scalate.github.io/scalate/) for templates, so you can use anything that is supported there (e.g. `mustache` or `ssp`) - the converter will be picked according to the file extension of the provided template file. Check the [scalate user guide](https://scalate.github.io/scalate/documentation/user-guide.html) for more details.
+
+The template is called on each database schema, and is passed an instance of [`dbcodegen.DataSchema`](codegen/src/main/scala/dbcodegen/DataSchema.scala) (variable name `schema`) which contains all the extracted information.
+You can see the declaration in the first line of each `ssp` template.
+
+### Simple
+
+case-classes.scala.ssp:
+```scala
+<%@ val schema: dbcodegen.DataSchema %>
+
+package kicks.db.${schema.name}
+
+#for (enum <- schema.enums)
+type ${enum.scalaName} = ${enum.values.map(v => "\"" + v.name + "\"").mkString(" | ")}
+#end
+
+#for (table <- schema.tables)
+
+case class ${table.scalaName}(
+  #for (column <- table.columns)
+  ${column.scalaName}: ${column.scalaType},
+  #end
+)
+
+#end
+```
+
+#### with scala 3 enums
+
+```scala
+#for (enum <- schema.enums)
+
+enum ${enum.scalaName}(val sqlValue: String) {
+  #for (enumValue <- enum.values)
+  case ${enumValue.scalaName} extends ${enum.scalaName}("${enumValue.name}")
+  #end
+}
+object ${enum.scalaName} {
+  def bySqlValue(searchValue: String): Option[${enum.scalaName}] = values.find(_.sqlValue == searchValue)
+}
+
+#end
+```
+
+### Library: quill
+
+quill-case-classes.scala.ssp:
+```scala
+<%@ val schema: dbcodegen.DataSchema %>
+
+package kicks.db.${schema.scalaName}
+
+import io.getquill.*
+
+#for (enum <- schema.enums)
+type ${enum.scalaName} = ${enum.values.map(v => "\"" + v.name + "\"").mkString(" | ")}
+#end
+
+#for (table <- schema.tables)
+
+case class ${table.scalaName}(
+  #for (column <- table.columns)
+  ${column.scalaName}: ${column.scalaType},
+  #end
+)
+object ${table.scalaName} {
+  inline def query = querySchema[Person](
+    "${table.name}",
+    #for (column <- table.columns)
+    _.${column.scalaName} -> "${column.name}",
+    #end
+  )
+}
+
+#end
+```
+
+### Library: magnum
+
+magnum-case-classes.scala.ssp:
+```scala
+<%@ val schema: dbcodegen.DataSchema %>
+
+package kicks.db.${schema.scalaName}
+
+import com.augustnagro.magnum.*
+
+#for (enum <- schema.enums)
+type ${enum.scalaName} = ${enum.values.map(v => "\"" + v.name + "\"").mkString(" | ")}
+#end
+
+#for (table <- schema.tables)
+
+@Table(SqliteDbType)
+case class ${table.scalaName}(
+  #for (column <- table.columns)
+  @SqlName("${column.name}") ${column.scalaName}: ${column.scalaType},
+  #end
+) derives DbCodec
+object ${table.scalaName} {
+  #{ val primaryKeyColumns = table.columns.filter(_.isPartOfPrimaryKey)}#
+  type Id = ${if (primaryKeyColumns.isEmpty) "Null" else primaryKeyColumns.map(_.scalaType).mkString("(", ", ", ")")}
+
+  case class Creator(
+    #for (column <- table.columns if !column.isAutoGenerated)
+    ${column.scalaName}: ${column.scalaType},
+    #end
+  )
+}
+
+val ${table.scalaName}Repo = Repo[${table.scalaName}.Creator, ${table.scalaName}, ${table.scalaName}.Id]
+
+#end
+```

build.sbt

@@ -4,12 +4,12 @@ inThisBuild(
   Seq(
     organization := "com.github.cornerman",
     licenses     := Seq("MIT License" -> url("https://opensource.org/licenses/MIT")),
-    homepage     := Some(url("https://github.com/cornerman/sbt-quillcodegen")),
+    homepage     := Some(url("https://github.com/cornerman/scala-db-codegen")),
     scmInfo := Some(
       ScmInfo(
-        url("https://github.com/cornerman/sbt-quillcodegen"),
-        "scm:git:git@github.com:cornerman/sbt-quillcodegen.git",
-        Some("scm:git:git@github.com:cornerman/sbt-quillcodegen.git"),
+        url("https://github.com/cornerman/scala-db-codegen"),
+        "scm:git:git@github.com:cornerman/scala-db-codegen.git",
+        Some("scm:git:git@github.com:cornerman/scala-db-codegen.git"),
       )
     ),
     pomExtra :=
@@ -26,16 +26,19 @@ inThisBuild(
 // TODO: Use sbt-cross to workaround: https://github.com/sbt/sbt/issues/5586
 lazy val codegen = project
   .settings(
-    name := "quillcodegen",
+    name := "scala-db-codegen",
     libraryDependencies ++= Seq(
-      "org.scala-lang" % "scala-reflect" % scalaVersion.value,
-      // Should be same as in Codegen.scala for generated code
-      "io.getquill"     %% "quill-codegen-jdbc"   % "4.8.1",
-      "org.xerial"       % "sqlite-jdbc"          % "3.44.1.0",
-      "org.postgresql"   % "postgresql"           % "42.7.1",
-      "mysql"            % "mysql-connector-java" % "8.0.33",
-      "org.mariadb.jdbc" % "mariadb-java-client"  % "3.1.2",
-      "org.mybatis"      % "mybatis"              % "3.5.15",
+      "org.scala-lang"        % "scala-reflect"            % scalaVersion.value,
+      "org.scalatra.scalate" %% "scalate-core"             % "1.10.1",
+      "org.mybatis"           % "mybatis"                  % "3.5.15",
+      "org.xerial"            % "sqlite-jdbc"              % "3.44.1.0",
+      "org.postgresql"        % "postgresql"               % "42.7.1",
+      "mysql"                 % "mysql-connector-java"     % "8.0.33",
+      "org.mariadb.jdbc"      % "mariadb-java-client"      % "3.1.2",
+      "us.fatehi"             % "schemacrawler-tools"      % "16.21.1",
+      "us.fatehi"             % "schemacrawler-postgresql" % "16.21.1",
+      "us.fatehi"             % "schemacrawler-sqlite"     % "16.21.1",
+      "us.fatehi"             % "schemacrawler-mysql"      % "16.21.1",
     ),
   )
   .cross
@@ -45,7 +48,7 @@ lazy val codegen213 = codegen("2.13.13")
 
 lazy val pluginSbt = project
   .settings(
-    name               := "sbt-quillcodegen",
+    name               := "sbt-db-codegen",
     scalaVersion       := "2.12.19",
     crossScalaVersions := Seq("2.12.19"),
     sbtPlugin          := true,
@@ -55,7 +58,7 @@ lazy val pluginSbt = project
 
 lazy val pluginMill = project
   .settings(
-    name               := "mill-quillcodegen",
+    name               := "mill-db-codegen",
     scalaVersion       := "2.13.13",
     crossScalaVersions := Seq("2.13.13"),
     libraryDependencies ++= Seq(

codegen/src/main/scala/SchemaCrawlerExt.scala

@@ -0,0 +1,8 @@
+package schemacrawler.crawl
+
+import schemacrawler.schema.{ColumnDataType, DataTypeType, Schema}
+
+object SchemaCrawlerExt {
+  def newColumnDataType(schema: Schema, name: String, tpe: DataTypeType): ColumnDataType =
+    new MutableColumnDataType(schema, name, tpe)
+}

codegen/src/main/scala/dbcodegen/CodeGenerator.scala

@@ -0,0 +1,73 @@
+package dbcodegen
+
+import schemacrawler.schemacrawler._
+import schemacrawler.tools.databaseconnector.{DatabaseConnectorRegistry, DatabaseUrlConnectionOptions}
+import schemacrawler.tools.utility.SchemaCrawlerUtility
+import us.fatehi.utility.datasource.MultiUseUserCredentials
+import org.fusesource.scalate.{TemplateEngine, TemplateSource}
+
+import java.io.File
+import java.nio.file.{Files, Path, Paths}
+import java.sql.SQLType
+import scala.jdk.CollectionConverters._
+
+case class DbConfig(
+  jdbcUrl: String,
+  username: Option[String],
+  password: Option[String],
+)
+
+case class CodeGeneratorConfig(
+  templateFiles: Seq[File],
+  outDir: File,
+  typeMapping: (SQLType, Option[String]) => Option[String],
+  schemaTableFilter: (String, String) => Boolean,
+)
+
+object CodeGenerator {
+  def generate(db: DbConfig, config: CodeGeneratorConfig): Seq[Path] = {
+    // schema crawler options
+
+    val credentials = new MultiUseUserCredentials(db.username.orNull, db.password.orNull)
+    val connection =
+      DatabaseConnectorRegistry
+        .getDatabaseConnectorRegistry()
+        .findDatabaseConnectorFromUrl(db.jdbcUrl)
+        .newDatabaseConnectionSource(new DatabaseUrlConnectionOptions(db.jdbcUrl), credentials)
+
+    val schemaCrawlerOptions = SchemaCrawlerOptionsBuilder
+      .newSchemaCrawlerOptions()
+      .withLoadOptions(LoadOptionsBuilder.builder().withInfoLevel(InfoLevel.maximum).toOptions)
+      .withLimitOptions(LimitOptionsBuilder.builder().toOptions)
+
+    val catalog = SchemaCrawlerUtility.getCatalog(connection, schemaCrawlerOptions)
+
+    // scalate
+
+    val templateEngine  = new TemplateEngine()
+    val templateSources = config.templateFiles.map(TemplateSource.fromFile)
+
+    // run template on schemas
+
+    val dataSchemas = catalog.getSchemas.asScala.map { schema =>
+      SchemaConverter.toDataSchema(schema, connection, catalog.getTables(schema).asScala.toSeq, config)
+    }
+
+    dataSchemas.flatMap { dataSchema =>
+      val data = Map(
+        "schema" -> dataSchema
+      )
+
+      templateSources.map { templateSource =>
+        val output     = templateEngine.layout(templateSource, data)
+        val outputPath = Paths.get(config.outDir.getPath, templateSource.file.getPath, s"${dataSchema.name}.scala")
+
+        Files.createDirectories(outputPath.getParent)
+        Files.write(outputPath, output.getBytes)
+
+        outputPath
+      }
+    }.toSeq
+  }
+
+}