Skip to content

Commit

Permalink
benchmark
Browse files Browse the repository at this point in the history
  • Loading branch information
@gavincabbage
gavincabbage committed Apr 17, 2019
1 parent 7f82240 commit e67dd37
Show file tree
Hide file tree
Showing 12 changed files with 148 additions and 77 deletions.
32 changes: 17 additions & 15 deletions chiv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,15 @@ import (

var (
// DefaultFormat is CSV.
DefaultFormatFunc = CSV
DefaultFormat = CSV
// ErrRecordLength does not match the number of columns.
ErrRecordLength = errors.New("record length does not match number of columns")
// ErrParserRegex initialization problem.
ErrParserRegex = errors.New("initializing parser regex")
)

type archiver struct {
// Archiver archives arbitrarily large relational database tables to Amazon S3.
type Archiver struct {
db *sql.DB
s3 *s3manager.Uploader
format FormatterFunc
Expand All @@ -30,13 +31,13 @@ type archiver struct {
null []byte
}

// NewArchiver constructs an archiver with the given database, S3 uploader and options. Options set on
// creation apply to all calls to Archive unless overridden.
func NewArchiver(db *sql.DB, s3 *s3manager.Uploader, options ...Option) *archiver {
a := archiver{
// NewArchiver constructs an archiver with the given database, S3 uploader and options.
// Options set on creation apply to all calls to Archive unless overridden.
func NewArchiver(db *sql.DB, s3 *s3manager.Uploader, options ...Option) *Archiver {
a := Archiver{
db: db,
s3: s3,
format: DefaultFormatFunc,
format: DefaultFormat,
}

for _, option := range options {
Expand All @@ -47,20 +48,21 @@ func NewArchiver(db *sql.DB, s3 *s3manager.Uploader, options ...Option) *archive
}

// Archive a database table to S3.
func (a *archiver) Archive(table, bucket string, options ...Option) error {
func (a *Archiver) Archive(table, bucket string, options ...Option) error {
return a.ArchiveWithContext(context.Background(), table, bucket, options...)
}

// ArchiveWithContext is like Archive, with context.
func (a *archiver) ArchiveWithContext(ctx context.Context, table, bucket string, options ...Option) error {
// ArchiveWithContext is like Archive, with context. Any options provided override those set on creation.
func (a *Archiver) ArchiveWithContext(ctx context.Context, table, bucket string, options ...Option) error {
b := *a
for _, option := range options {
option(a)
option(&b)
}

return a.archive(ctx, table, bucket)
return b.archive(ctx, table, bucket)
}

func (a *archiver) archive(ctx context.Context, table string, bucket string) error {
func (a *Archiver) archive(ctx context.Context, table string, bucket string) error {
errs := make(chan error)
r, w := io.Pipe()
defer r.Close()
Expand All @@ -77,7 +79,7 @@ func (a *archiver) archive(ctx context.Context, table string, bucket string) err
}
}

func (a *archiver) download(ctx context.Context, wc io.WriteCloser, table string, errs chan error) {
func (a *Archiver) download(ctx context.Context, wc io.WriteCloser, table string, errs chan error) {
selectAll := fmt.Sprintf(`select * from "%s";`, table)
rows, err := a.db.QueryContext(ctx, selectAll)
if err != nil {
Expand Down Expand Up @@ -144,7 +146,7 @@ func (a *archiver) download(ctx context.Context, wc io.WriteCloser, table string
}
}

func (a *archiver) upload(ctx context.Context, r io.Reader, table string, bucket string, errs chan error) {
func (a *Archiver) upload(ctx context.Context, r io.Reader, table string, bucket string, errs chan error) {
if a.key == "" {
if a.extension != "" {
a.key = fmt.Sprintf("%s.%s", table, a.extension)
Expand Down
52 changes: 52 additions & 0 deletions chiv_benchmark_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Package chiv_test includes integration tests and benchmarks external to package chiv
// It relies on external services postgres and s3 (localstack) via CodeShip.
package chiv_test

import (
"os"
"testing"

"github.com/aws/aws-sdk-go/service/s3/s3manager"

"github.com/gavincabbage/chiv"

_ "github.com/lib/pq"
)

func BenchmarkArchiver_Archive(b *testing.B) {
var (
db = newDB(b, "postgres", os.Getenv("POSTGRES_URL"))
s3client = newS3Client(b, os.Getenv("AWS_REGION"), os.Getenv("AWS_ENDPOINT"))
uploader = s3manager.NewUploaderWithClient(s3client)
)

const (
createTable = `CREATE TABLE IF NOT EXISTS "benchmark_table" (t TEXT,i INTEGER,f DECIMAL);`

insertIntoTable = `INSERT INTO "benchmark_table" VALUES (%s,
2345,
);`

dropTable = `
DROP TABLE "benchmark_table";`
)

exec(b, db, createTable)
defer exec(b, db, dropTable)

for i := 0; i < 100; i++ {
exec(b, db, insertIntoTable)
}

createBucket(b, s3client, "benchmark_bucket")

subject := chiv.NewArchiver(db, uploader)

b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := subject.Archive("benchmark_table", "benchmark_bucket", chiv.WithKey("benchmark_"+string(i))); err != nil {
b.Error(err)
}
}
}
114 changes: 63 additions & 51 deletions chiv_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Package chiv_test includes integration tests external to package chiv
// and relies on external services postgres and s3 (localstack) via CodeShip.
// Package chiv_test includes integration tests and benchmarks external to package chiv
// It relies on external services postgres and s3 (localstack) via CodeShip.
package chiv_test

import (
Expand All @@ -14,20 +14,13 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
_ "github.com/lib/pq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/gavincabbage/chiv"
)

type call struct {
expected string
table string
bucket string
key string
options []chiv.Option
}
_ "github.com/lib/pq"
)

type test struct {
name string
Expand All @@ -39,18 +32,26 @@ type test struct {
calls []call
}

type call struct {
expected string
table string
bucket string
key string
options []chiv.Option
}

func TestArchiver_Archive(t *testing.T) {
cases := []test{
{
name: "postgres to csv",
driver: "postgres",
database: os.Getenv("POSTGRES_URL"),
setup: "./testdata/postgres_setup.sql",
teardown: "./testdata/postgres_teardown.sql",
setup: "./test/data/postgres_setup.sql",
teardown: "./test/data/postgres_teardown.sql",
options: []chiv.Option{},
calls: []call{
{
expected: "./testdata/postgres.csv",
expected: "./test/data/postgres.csv",
bucket: "postgres_bucket",
table: "postgres_table",
key: "postgres_table",
Expand All @@ -62,14 +63,14 @@ func TestArchiver_Archive(t *testing.T) {
name: "postgres to csv key override",
driver: "postgres",
database: os.Getenv("POSTGRES_URL"),
setup: "./testdata/postgres_setup.sql",
teardown: "./testdata/postgres_teardown.sql",
setup: "./test/data/postgres_setup.sql",
teardown: "./test/data/postgres_teardown.sql",
options: []chiv.Option{
chiv.WithKey("postgres_table.csv"),
},
calls: []call{
{
expected: "./testdata/postgres.csv",
expected: "./test/data/postgres.csv",
bucket: "postgres_bucket",
table: "postgres_table",
key: "postgres_table.csv",
Expand All @@ -81,14 +82,14 @@ func TestArchiver_Archive(t *testing.T) {
name: "postgres to csv null override",
driver: "postgres",
database: os.Getenv("POSTGRES_URL"),
setup: "./testdata/postgres_setup.sql",
teardown: "./testdata/postgres_teardown.sql",
setup: "./test/data/postgres_setup.sql",
teardown: "./test/data/postgres_teardown.sql",
options: []chiv.Option{
chiv.WithNull("custom_null"),
},
calls: []call{
{
expected: "./testdata/postgres_with_null.csv",
expected: "./test/data/postgres_with_null.csv",
bucket: "postgres_bucket",
table: "postgres_table",
key: "postgres_table",
Expand All @@ -100,15 +101,15 @@ func TestArchiver_Archive(t *testing.T) {
name: "postgres to json",
driver: "postgres",
database: os.Getenv("POSTGRES_URL"),
setup: "./testdata/postgres_setup.sql",
teardown: "./testdata/postgres_teardown.sql",
setup: "./test/data/postgres_setup.sql",
teardown: "./test/data/postgres_teardown.sql",
options: []chiv.Option{
chiv.WithFormat(chiv.JSON),
chiv.WithKey("postgres_table.json"),
},
calls: []call{
{
expected: "./testdata/postgres.json",
expected: "./test/data/postgres.json",
bucket: "postgres_bucket",
table: "postgres_table",
key: "postgres_table.json",
Expand All @@ -120,22 +121,53 @@ func TestArchiver_Archive(t *testing.T) {
name: "postgres to yaml",
driver: "postgres",
database: os.Getenv("POSTGRES_URL"),
setup: "./testdata/postgres_setup.sql",
teardown: "./testdata/postgres_teardown.sql",
setup: "./test/data/postgres_setup.sql",
teardown: "./test/data/postgres_teardown.sql",
options: []chiv.Option{
chiv.WithFormat(chiv.YAML),
chiv.WithKey("postgres_table.yaml"),
},
calls: []call{
{
expected: "./testdata/postgres.yaml",
expected: "./test/data/postgres.yaml",
bucket: "postgres_bucket",
table: "postgres_table",
key: "postgres_table.yaml",
options: []chiv.Option{},
},
},
},
{
name: "postgres two formats",
driver: "postgres",
database: os.Getenv("POSTGRES_URL"),
setup: "./test/data/postgres_setup.sql",
teardown: "./test/data/postgres_teardown.sql",
options: []chiv.Option{
chiv.WithFormat(chiv.YAML),
},
calls: []call{
{
expected: "./test/data/postgres.json",
bucket: "postgres_bucket",
table: "postgres_table",
key: "postgres_table.json",
options: []chiv.Option{
chiv.WithFormat(chiv.JSON),
chiv.WithKey("postgres_table.json"),
},
},
{
expected: "./test/data/postgres.yaml",
bucket: "postgres_bucket",
table: "postgres_table",
key: "postgres_table.yaml",
options: []chiv.Option{
chiv.WithKey("postgres_table.yaml"),
},
},
},
},
}

for _, test := range cases {
Expand All @@ -147,8 +179,8 @@ func TestArchiver_Archive(t *testing.T) {
downloader = s3manager.NewDownloaderWithClient(s3client)
)

exec(t, db, test.setup)
defer exec(t, db, test.teardown)
exec(t, db, readFile(t, test.setup))
defer exec(t, db, readFile(t, test.teardown))

subject := chiv.NewArchiver(db, uploader, test.options...)
assert.NotNil(t, subject)
Expand All @@ -166,25 +198,6 @@ func TestArchiver_Archive(t *testing.T) {
}
}

func BenchmarkArchiver_Archive(b *testing.B) {
var (
db = newDB(b, "postgres", os.Getenv("POSTGRES_URL"))
s3client = newS3Client(b, os.Getenv("AWS_REGION"), os.Getenv("AWS_ENDPOINT"))
uploader = s3manager.NewUploaderWithClient(s3client)
)

exec(b, db, "./testdata/repeated_setup.sql")
defer exec(b, db, "./testdata/repeated_teardown.sql")

subject := chiv.NewArchiver(db, uploader, chiv.WithKey("repeated.csv"))

createBucket(b, s3client, "repeated_bucket")

if err := subject.Archive("repeated_table", "repeated_bucket"); err != nil {
b.Error()
}
}

func newDB(e errorer, driver string, url string) *sql.DB {
db, err := sql.Open(driver, url)
if err != nil {
Expand All @@ -211,10 +224,9 @@ func newS3Client(e errorer, region string, endpoint string) *s3.S3 {
return client
}

func exec(e errorer, db *sql.DB, path string) {
file := readFile(e, path)
statements := strings.Split(string(file), ";\n\n")
for _, statement := range statements {
func exec(e errorer, db *sql.DB, statements string) {
s := strings.Split(statements, ";\n\n")
for _, statement := range s {
if _, err := db.Exec(statement); err != nil {
e.Error(err)
}
Expand Down
8 changes: 5 additions & 3 deletions codeship-services.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ test:
- AWS_REGION=us-east-1
- AWS_ENDPOINT=http://s3:4572
- POSTGRES_URL=postgres://postgres:password@postgres/test?sslmode=disable
- AWS_ACCESS_KEY_FILE=access_key
AWS_SECRET_KEY_FILE=secret_key
depends_on:
- postgres
- s3
Expand All @@ -16,14 +18,14 @@ lint:
cached: true

postgres:
image: postgres:11.2
image: healthcheck/postgres
environment:
- POSTGRES_DB=test
- POSTGRES_PASSWORD=password
cached: true

s3:
image: localstack/localstack:0.9.0
image: localstack/localstack:latest
environment:
- SERVICES=s3
cached: true
cached: true
Loading

0 comments on commit e67dd37

Please sign in to comment.