diff --git a/chiv_benchmark_test.go b/chiv_benchmark_test.go index aee495d..643d327 100644 --- a/chiv_benchmark_test.go +++ b/chiv_benchmark_test.go @@ -13,6 +13,7 @@ import ( "github.com/gavincabbage/chiv" + _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" ) @@ -30,12 +31,12 @@ func BenchmarkArchiver_Archive(b *testing.B) { ) var ( + benchmarks = []int{1, 10, 100, 1000, 3000, 5000, 10000} + db = newDB(b, "postgres", os.Getenv("POSTGRES_URL")) s3client = newS3Client(b, os.Getenv("AWS_REGION"), os.Getenv("AWS_ENDPOINT")) uploader = s3manager.NewUploaderWithClient(s3client) - r = rand.New(rand.NewSource(time.Now().Unix())) - - benchmarks = []int{1, 10, 100, 1000, 2000, 3000, 5000, 10000} + random = rand.New(rand.NewSource(time.Now().Unix())) ) for _, count := range benchmarks { @@ -43,7 +44,7 @@ func BenchmarkArchiver_Archive(b *testing.B) { createBucket(b, s3client, bucket) for i := 0; i < count; i++ { - statement := fmt.Sprintf(insertIntoTable, text(r, charset, textLength), i, 1.0) + statement := fmt.Sprintf(insertIntoTable, text(random, charset, textLength), i, 42.42) exec(b, db, statement) } diff --git a/chiv_test.go b/chiv_test.go index b13932b..f90bf31 100644 --- a/chiv_test.go +++ b/chiv_test.go @@ -160,6 +160,23 @@ func TestArchiver_Archive(t *testing.T) { }, }, }, + //{ + // name: "mysql to csv", + // driver: "mysql", + // database: os.Getenv("MYSQL_URL"), + // setup: "./test/data/mysql_setup.sql", + // teardown: "./test/data/mysql_teardown.sql", + // bucket: "mysql_bucket", + // options: []chiv.Option{}, + // calls: []call{ + // { + // expected: "./test/data/mysql.csv", + // table: "mysql_table", + // key: "mysql_table", + // options: []chiv.Option{}, + // }, + // }, + //}, } for _, test := range cases { diff --git a/go.mod b/go.mod index a63b36b..dc1f244 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,11 @@ go 1.12 require ( github.com/aws/aws-sdk-go v1.19.6 github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-sql-driver/mysql v1.4.1 github.com/lib/pq v1.0.0 github.com/stretchr/testify v1.3.0 golang.org/x/net v0.0.0-20190313220215-9f648a60d977 // indirect + golang.org/x/tour v0.0.0-20190318020441-db40fe78fefc // indirect + google.golang.org/appengine v1.5.0 // indirect gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 18ee733..f780774 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,9 @@ github.com/aws/aws-sdk-go v1.19.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= @@ -13,11 +16,19 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190312164927-7b79afddac43 h1:TIYuaOnWereVgpmGdM420uV4ykKf3r5m++5s628rZdE= +golang.org/x/tools v0.0.0-20190312164927-7b79afddac43/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tour v0.0.0-20190318020441-db40fe78fefc h1:z7p7FC77GRAkUMqvG1FESaaikV+A2gpE8FhHbVNBz8I= +golang.org/x/tour v0.0.0-20190318020441-db40fe78fefc/go.mod h1:qMugOFWX59KzC8Nx7f2uvXxKxAqJfi1J6ZUHAWKnrRA= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/support_test.go b/support_test.go index a508adc..45f3cf1 100644 --- a/support_test.go +++ b/support_test.go @@ -13,20 +13,24 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" ) -type errorer interface { - Error(...interface{}) +type fataler interface { + Fatal(...interface{}) } -func newDB(e errorer, driver string, url string) *sql.DB { +func newDB(f fataler, driver string, url string) *sql.DB { db, err := sql.Open(driver, url) if err != nil { - e.Error(err) + f.Fatal(err) + } + + if err := db.Ping(); err != nil { + f.Fatal(err) } return db } -func newS3Client(e errorer, region string, endpoint string) *s3.S3 { +func newS3Client(f fataler, region string, endpoint string) *s3.S3 { awsConfig := aws.NewConfig(). WithRegion(region). WithDisableSSL(true). @@ -34,7 +38,7 @@ func newS3Client(e errorer, region string, endpoint string) *s3.S3 { awsSession, err := session.NewSession(awsConfig) if err != nil { - e.Error(err) + f.Fatal(err) } client := s3.New(awsSession) @@ -43,31 +47,31 @@ func newS3Client(e errorer, region string, endpoint string) *s3.S3 { return client } -func exec(e errorer, db *sql.DB, statements string) { +func exec(f fataler, db *sql.DB, statements string) { s := strings.Split(statements, ";\n\n") for _, statement := range s { if _, err := db.Exec(statement); err != nil { fmt.Println(statement) - e.Error(err) + f.Fatal(err) } } } -func createBucket(e errorer, client *s3.S3, name string) { +func createBucket(f fataler, client *s3.S3, name string) { if _, err := client.CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(name), }); err != nil { - e.Error(err) + f.Fatal(err) } } -func deleteBucket(e errorer, client *s3.S3, name string) { +func deleteBucket(f fataler, client *s3.S3, name string) { // we could do this more cleanly with BatchDeleteIterator, but localstack doesn't like batch deletes :shrug: out, err := client.ListObjects(&s3.ListObjectsInput{ Bucket: aws.String(name), }) if err != nil { - e.Error(err) + f.Fatal(err) } for _, o := range out.Contents { @@ -75,34 +79,34 @@ func deleteBucket(e errorer, client *s3.S3, name string) { Key: o.Key, Bucket: aws.String(name), }); err != nil { - e.Error(err) + f.Fatal(err) } } if _, err := client.DeleteBucket(&s3.DeleteBucketInput{ Bucket: aws.String(name), }); err != nil { - e.Error(err) + f.Fatal(err) } } -func readFile(e errorer, path string) string { +func readFile(f fataler, path string) string { contents, err := ioutil.ReadFile(path) if err != nil { - e.Error(err) + f.Fatal(err) } return string(contents) } -func download(e errorer, downloader *s3manager.Downloader, bucket string, key string) string { +func download(f fataler, downloader *s3manager.Downloader, bucket string, key string) string { b := &aws.WriteAtBuffer{} _, err := downloader.Download(b, &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) if err != nil { - e.Error(err) + f.Fatal(err) } return string(b.Bytes())