Skip to content

Commit

Permalink
forget mysql
Browse files Browse the repository at this point in the history
  • Loading branch information
Gavin Cabbage committed May 21, 2019
1 parent 71aed52 commit 040222b
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 22 deletions.
9 changes: 5 additions & 4 deletions chiv_benchmark_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/gavincabbage/chiv"

_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
)

Expand All @@ -30,20 +31,20 @@ func BenchmarkArchiver_Archive(b *testing.B) {
)

var (
benchmarks = []int{1, 10, 100, 1000, 3000, 5000, 10000}

db = newDB(b, "postgres", os.Getenv("POSTGRES_URL"))
s3client = newS3Client(b, os.Getenv("AWS_REGION"), os.Getenv("AWS_ENDPOINT"))
uploader = s3manager.NewUploaderWithClient(s3client)
r = rand.New(rand.NewSource(time.Now().Unix()))

benchmarks = []int{1, 10, 100, 1000, 2000, 3000, 5000, 10000}
random = rand.New(rand.NewSource(time.Now().Unix()))
)

for _, count := range benchmarks {
exec(b, db, createTable)
createBucket(b, s3client, bucket)

for i := 0; i < count; i++ {
statement := fmt.Sprintf(insertIntoTable, text(r, charset, textLength), i, 1.0)
statement := fmt.Sprintf(insertIntoTable, text(random, charset, textLength), i, 42.42)
exec(b, db, statement)
}

Expand Down
17 changes: 17 additions & 0 deletions chiv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,23 @@ func TestArchiver_Archive(t *testing.T) {
},
},
},
//{
// name: "mysql to csv",
// driver: "mysql",
// database: os.Getenv("MYSQL_URL"),
// setup: "./test/data/mysql_setup.sql",
// teardown: "./test/data/mysql_teardown.sql",
// bucket: "mysql_bucket",
// options: []chiv.Option{},
// calls: []call{
// {
// expected: "./test/data/mysql.csv",
// table: "mysql_table",
// key: "mysql_table",
// options: []chiv.Option{},
// },
// },
//},
}

for _, test := range cases {
Expand Down
3 changes: 3 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ go 1.12
require (
github.com/aws/aws-sdk-go v1.19.6
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-sql-driver/mysql v1.4.1
github.com/lib/pq v1.0.0
github.com/stretchr/testify v1.3.0
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 // indirect
golang.org/x/tour v0.0.0-20190318020441-db40fe78fefc // indirect
google.golang.org/appengine v1.5.0 // indirect
gopkg.in/yaml.v2 v2.2.2
)
11 changes: 11 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ github.com/aws/aws-sdk-go v1.19.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
Expand All @@ -13,11 +16,19 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190312164927-7b79afddac43 h1:TIYuaOnWereVgpmGdM420uV4ykKf3r5m++5s628rZdE=
golang.org/x/tools v0.0.0-20190312164927-7b79afddac43/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tour v0.0.0-20190318020441-db40fe78fefc h1:z7p7FC77GRAkUMqvG1FESaaikV+A2gpE8FhHbVNBz8I=
golang.org/x/tour v0.0.0-20190318020441-db40fe78fefc/go.mod h1:qMugOFWX59KzC8Nx7f2uvXxKxAqJfi1J6ZUHAWKnrRA=
google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
Expand Down
40 changes: 22 additions & 18 deletions support_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,28 +13,32 @@ import (
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

type errorer interface {
Error(...interface{})
type fataler interface {
Fatal(...interface{})
}

func newDB(e errorer, driver string, url string) *sql.DB {
func newDB(f fataler, driver string, url string) *sql.DB {
db, err := sql.Open(driver, url)
if err != nil {
e.Error(err)
f.Fatal(err)
}

if err := db.Ping(); err != nil {
f.Fatal(err)
}

return db
}

func newS3Client(e errorer, region string, endpoint string) *s3.S3 {
func newS3Client(f fataler, region string, endpoint string) *s3.S3 {
awsConfig := aws.NewConfig().
WithRegion(region).
WithDisableSSL(true).
WithCredentials(credentials.NewEnvCredentials())

awsSession, err := session.NewSession(awsConfig)
if err != nil {
e.Error(err)
f.Fatal(err)
}

client := s3.New(awsSession)
Expand All @@ -43,66 +47,66 @@ func newS3Client(e errorer, region string, endpoint string) *s3.S3 {
return client
}

func exec(e errorer, db *sql.DB, statements string) {
func exec(f fataler, db *sql.DB, statements string) {
s := strings.Split(statements, ";\n\n")
for _, statement := range s {
if _, err := db.Exec(statement); err != nil {
fmt.Println(statement)
e.Error(err)
f.Fatal(err)
}
}
}

func createBucket(e errorer, client *s3.S3, name string) {
func createBucket(f fataler, client *s3.S3, name string) {
if _, err := client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(name),
}); err != nil {
e.Error(err)
f.Fatal(err)
}
}

func deleteBucket(e errorer, client *s3.S3, name string) {
func deleteBucket(f fataler, client *s3.S3, name string) {
// we could do this more cleanly with BatchDeleteIterator, but localstack doesn't like batch deletes :shrug:
out, err := client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(name),
})
if err != nil {
e.Error(err)
f.Fatal(err)
}

for _, o := range out.Contents {
if _, err := client.DeleteObject(&s3.DeleteObjectInput{
Key: o.Key,
Bucket: aws.String(name),
}); err != nil {
e.Error(err)
f.Fatal(err)
}
}

if _, err := client.DeleteBucket(&s3.DeleteBucketInput{
Bucket: aws.String(name),
}); err != nil {
e.Error(err)
f.Fatal(err)
}
}

func readFile(e errorer, path string) string {
func readFile(f fataler, path string) string {
contents, err := ioutil.ReadFile(path)
if err != nil {
e.Error(err)
f.Fatal(err)
}

return string(contents)
}

func download(e errorer, downloader *s3manager.Downloader, bucket string, key string) string {
func download(f fataler, downloader *s3manager.Downloader, bucket string, key string) string {
b := &aws.WriteAtBuffer{}
_, err := downloader.Download(b, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
e.Error(err)
f.Fatal(err)
}

return string(b.Bytes())
Expand Down

0 comments on commit 040222b

Please sign in to comment.