Bulk import

README.md

@@ -116,6 +116,71 @@ class Site < Granite::ORM::Base
 end
 ```
 
+### Bulk Insertions
+
+#### Import
+
+**Note:  As of now, imports do not trigger callbacks.**
+
+Each model has an `import` class level method to import an array of models in one bulk insert statement.
+```Crystal
+models = [
+  Model.new(id: 1, name: "Fred", age: 14),
+  Model.new(id: 2, name: "Joe", age: 25),
+  Model.new(id: 3, name: "John", age: 30),
+]
+
+Model.import(models)
+```
+
+#### update_on_duplicate
+
+The `import` method has an optional `update_on_duplicate`  + `columns` params that allows you to specify the columns (as an array of strings) that should be updated if primary constraint is violated.
+```Crystal
+models = [
+  Model.new(id: 1, name: "Fred", age: 14),
+  Model.new(id: 2, name: "Joe", age: 25),
+  Model.new(id: 3, name: "John", age: 30),
+]
+
+Model.import(models)
+
+Model.find!(1).name # => Fred
+
+models = [
+  Model.new(id: 1, name: "George", age: 14),
+]
+
+Model.import(models, update_on_duplicate: true, columns: %w(name))
+
+Model.find!(1).name # => George
+```
+
+##### NOTE:  If using PostgreSQL you must have version 9.5+ to have the on_duplicate_key_update feature.
+
+#### ignore_on_duplicate
+
+the `import` method has an optional `ignore_on_duplicate` param, that takes a boolean, which will skip records if the primary constraint is violated.
+```Crystal
+models = [
+  Model.new(id: 1, name: "Fred", age: 14),
+  Model.new(id: 2, name: "Joe", age: 25),
+  Model.new(id: 3, name: "John", age: 30),
+]
+
+Model.import(models)
+
+Model.find!(1).name # => Fred
+
+models = [
+  Model.new(id: 1, name: "George", age: 14),
+]
+
+Model.import(models, ignore_on_duplicate: true)
+
+Model.find!(1).name # => Fred
+```
+
 ### SQL
 
 To clear all the rows in the database:

spec/granite_orm/transactions/import_spec.cr

@@ -0,0 +1,59 @@
+require "../../spec_helper"
+
+{% for adapter in GraniteExample::ADAPTERS %}
+module {{adapter.capitalize.id}}
+  describe "{{ adapter.id }} .import" do
+    it "should import 3 new objects" do
+      to_import = [
+        Parent.new(name: "ImportParent1"),
+        Parent.new(name: "ImportParent2"),
+        Parent.new(name: "ImportParent3"),
+      ]
+      Parent.import(to_import)
+      Parent.all("WHERE name LIKE ?", ["Import%"]).size.should eq 3
+    end
+
+    it "should work with on_duplicate_key_update" do
+      to_import = [
+         Parent.new(id: 111, name: "ImportParent1"),
+         Parent.new(id: 112, name: "ImportParent2"),
+         Parent.new(id: 113, name: "ImportParent3"),
+      ]
+
+      Parent.import(to_import)
+
+      to_import = [
+         Parent.new(id: 112, name: "ImportParent112"),
+      ]
+
+      Parent.import(to_import, update_on_duplicate: true, columns: ["name"])
+
+      if parent = Parent.find 112
+        parent.name.should be "ImportParent112"
+        parent.id.should eq 112
+      end
+    end
+
+    it "should work with on_duplicate_key_ignore" do
+      to_import = [
+         Parent.new(id: 111, name: "ImportParent1"),
+         Parent.new(id: 112, name: "ImportParent2"),
+         Parent.new(id: 113, name: "ImportParent3"),
+      ]
+
+      Parent.import(to_import)
+
+      to_import = [
+         Parent.new(id: 113, name: "ImportParent113"),
+      ]
+
+      Parent.import(to_import, ignore_on_duplicate: true)
+
+      if parent = Parent.find 113
+        parent.name.should be "ImportParent3"
+        parent.id.should eq 113
+      end
+    end
+  end
+end
+{% end %}

spec/spec_helper.cr

@@ -1,7 +1,7 @@
 require "spec"
 
 module GraniteExample
-  ADAPTERS = ["pg","mysql","sqlite"]
+  ADAPTERS = ["pg", "mysql", "sqlite"]
 end
 
 require "../src/granite_orm"

src/adapter/base.cr

@@ -49,6 +49,9 @@ abstract class Granite::Adapter::Base
   # This will insert a row in the database and return the id generated.
   abstract def insert(table_name, fields, params, lastval) : Int64
 
+  # This will insert an array of models as one insert statement
+  abstract def import(table_name : String, primary_name : String, fields, model_array, **options)
+
   # This will update a row in the database.
   abstract def update(table_name, primary_name, fields, params)
 

src/adapter/mysql.cr

@@ -76,6 +76,46 @@ class Granite::Adapter::Mysql < Granite::Adapter::Base
     end
   end
 
+  def import(table_name : String, primary_name : String, fields, model_array, **options)
+    params = [] of DB::Any
+    now = Time.now.to_utc
+    fields.reject! { |field| field === "id" } if primary_name === "id"
+
+    statement = String.build do |stmt|
+      stmt << "INSERT"
+      stmt << " IGNORE" if options["ignore_on_duplicate"]?
+      stmt << " INTO #{quote(table_name)} ("
+      stmt << fields.map { |field| quote(field) }.join(", ")
+      stmt << ") VALUES "
+
+      model_array.each do |model|
+        model.updated_at = now if model.responds_to? :updated_at
+        model.created_at = now if model.responds_to? :created_at
+        next unless model.valid?
+        stmt << '('
+        stmt << Array.new(fields.size, '?').join(',')
+        params.concat fields.map { |field| model.to_h[field] }
+        stmt << "),"
+      end
+    end.chomp(',')
+
+    if options["update_on_duplicate"]?
+      if columns = options["columns"]?
+        statement += " ON DUPLICATE KEY UPDATE "
+        columns.each do |key|
+          statement += "#{quote(key)}=VALUES(#{quote(key)}), "
+        end
+        statement = statement.chomp(", ")
+      end
+    end
+
+    log statement, params
+
+    open do |db|
+      db.exec statement, params
+    end
+  end
+
   private def last_val
     return "SELECT LAST_INSERT_ID()"
   end

src/adapter/pg.cr

@@ -76,6 +76,49 @@ class Granite::Adapter::Pg < Granite::Adapter::Base
     end
   end
 
+  def import(table_name : String, primary_name : String, fields, model_array, **options)
+    params = [] of DB::Any
+    now = Time.now.to_utc
+    fields.reject! { |field| field === "id" } if primary_name === "id"
+    index = 0
+
+    statement = String.build do |stmt|
+      stmt << "INSERT"
+      stmt << " INTO #{quote(table_name)} ("
+      stmt << fields.map { |field| quote(field) }.join(", ")
+      stmt << ") VALUES "
+
+      model_array.each do |model|
+        model.updated_at = now if model.responds_to? :updated_at
+        model.created_at = now if model.responds_to? :created_at
+        next unless model.valid?
+        stmt << '('
+        stmt << fields.map_with_index { |_f, idx| "$#{index + idx + 1}" }.join(',')
+        params.concat fields.map { |field| model.to_h[field] }
+        stmt << "),"
+        index += fields.size
+      end
+    end.chomp(',')
+
+    if options["update_on_duplicate"]?
+      if columns = options["columns"]?
+        statement += " ON CONFLICT (#{quote(primary_name)}) DO UPDATE SET "
+        columns.each do |key|
+          statement += "#{quote(key)}=EXCLUDED.#{quote(key)}, "
+        end
+      end
+      statement = statement.chomp(", ")
+    elsif options["ignore_on_duplicate"]?
+      statement += " ON CONFLICT DO NOTHING"
+    end
+
+    log statement, params
+
+    open do |db|
+      db.exec statement, params
+    end
+  end
+
   private def last_val
     return "SELECT LASTVAL()"
   end

src/adapter/sqlite.cr

@@ -74,6 +74,40 @@ class Granite::Adapter::Sqlite < Granite::Adapter::Base
     end
   end
 
+  def import(table_name : String, primary_name : String, fields, model_array, **options)
+    params = [] of DB::Any
+    now = Time.now.to_utc
+    fields.reject! { |field| field === "id" } if primary_name === "id"
+
+    statement = String.build do |stmt|
+      stmt << "INSERT "
+      if options["update_on_duplicate"]?
+        stmt << "OR REPLACE "
+      elsif options["ignore_on_duplicate"]?
+        stmt << "OR IGNORE "
+      end
+      stmt << "INTO #{quote(table_name)} ("
+      stmt << fields.map { |field| quote(field) }.join(", ")
+      stmt << ") VALUES "
+
+      model_array.each do |model|
+        next unless model.valid?
+        model.updated_at = now if model.responds_to? :updated_at
+        model.created_at = now if model.responds_to? :created_at
+        stmt << '('
+        stmt << Array.new(fields.size, '?').join(',')
+        params.concat fields.map { |field| model.to_h[field] }
+        stmt << "),"
+      end
+    end.chomp(',')
+
+    log statement, params
+
+    open do |db|
+      db.exec statement, params
+    end
+  end
+
   private def last_val
     return "SELECT LAST_INSERT_ROWID()"
   end

src/granite_orm/transactions.cr

@@ -7,9 +7,36 @@ module Granite::ORM::Transactions
     @updated_at : Time?
     @created_at : Time?
 
+    # The import class method will run a batch INSERT statement for each model in the array
+    # the array must contain only one model class
+    # invalid model records will be skipped
+    def self.import(model_array : Array(self))
+      begin
+        @@adapter.import(table_name, primary_name, fields.dup, model_array)
+      rescue err
+        raise DB::Error.new(err.message)
+      end
+    end
+
+    def self.import(model_array : Array(self), update_on_duplicate : Bool, columns : Array(String))
+      begin
+        @@adapter.import(table_name, primary_name, fields.dup, model_array, update_on_duplicate: update_on_duplicate, columns: columns)
+      rescue err
+        raise DB::Error.new(err.message)
+      end
+    end
+
+    def self.import(model_array : Array(self), ignore_on_duplicate : Bool)
+      begin
+        @@adapter.import(table_name, primary_name, fields.dup, model_array, ignore_on_duplicate: ignore_on_duplicate)
+      rescue err
+        raise DB::Error.new(err.message)
+      end
+    end
+
     # The save method will check to see if the primary exists yet. If it does it
     # will call the update method, otherwise it will call the create method.
-    # This will update the timestamps apropriately.
+    # This will update the timestamps appropriately.
     def save
       return false unless valid?
 
@@ -91,7 +118,7 @@ module Granite::ORM::Transactions
     end
   end
 
-  module ClassMethods 
+  module ClassMethods
     def create(**args)
       create(args.to_h)
     end

src/granite_orm/validators.cr

@@ -8,7 +8,7 @@ require "./error"
 #   !user.name.to_s.blank?
 # end
 #
-# validate :name, "can't be blank", -> (user : User) do
+# validate :name, "can't be blank", ->(user : User) do
 #   !user.name.to_s.blank?
 # end
 #