Skip to content

Commit

Permalink
feat: avoid updating identiqual airtable record (using another golang…
Browse files Browse the repository at this point in the history 
… dependency)
  • Loading branch information
@moul
moul committed Oct 7, 2018
1 parent abf5914 commit 1eb2274
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 36 deletions.
69 changes: 37 additions & 32 deletions cmd_airtable.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ package main
import (
"encoding/json"
"reflect"
"sort"
"time"

airtable "github.com/fabioberger/airtable-go"
"github.com/brianloveswords/airtable"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
Expand Down Expand Up @@ -67,15 +68,17 @@ func airtableSync(opts *airtableOptions) error {
issues.filterByTargets(opts.Targets)
logger().Debug("fetch db entries", zap.Int("count", len(issues)))

at, err := airtable.New(opts.AirtableToken, opts.AirtableBaseID)
if err != nil {
return err
at := airtable.Client{
APIKey: opts.AirtableToken,
BaseID: opts.AirtableBaseID,
Limiter: airtable.RateLimiter(5),
}
table := at.Table(opts.AirtableTableName)

alreadyInAirtable := map[string]bool{}

records := []airtableRecord{}
if err := at.ListRecords(opts.AirtableTableName, &records); err != nil {
if err := table.List(&records, &airtable.Options{}); err != nil {
return err
}
logger().Debug("fetched airtable records", zap.Int("count", len(records)))
Expand All @@ -95,16 +98,17 @@ func airtableSync(opts *airtableOptions) error {
r := issue.ToAirtableRecord()
r.Fields.Labels = []string{}
r.Fields.Assignees = []string{}
if err := at.CreateRecord(opts.AirtableTableName, r); err != nil {
if err := table.Create(&r); err != nil {
return err
}
records = append(records, r)
}

// update/destroy existing ones
for _, record := range records {
if issue, found := issues[record.Fields.URL]; !found {
logger().Debug("destroying airtable record", zap.String("URL", record.Fields.URL))
if err := at.DestroyRecord(opts.AirtableTableName, record.ID); err != nil {
if err := table.Delete(&record); err != nil {
return errors.Wrap(err, "failed to destroy record")
}
} else {
Expand All @@ -117,13 +121,17 @@ func airtableSync(opts *airtableOptions) error {
}

logger().Debug("updating airtable record", zap.String("URL", issue.URL))
m := issue.ToAirtableRecord().Fields.Map()
if err := at.UpdateRecord(opts.AirtableTableName, record.ID, m, &record); err != nil {
record.Fields = issue.ToAirtableRecord().Fields
if err := table.Update(&record); err != nil {
logger().Warn("failed to update record, retrying without slices", zap.String("URL", issue.URL), zap.Error(err))
m["Labels"] = []string{}
m["Assignees"] = []string{}
m["Errors"] = err.Error()
if err := at.UpdateRecord(opts.AirtableTableName, record.ID, m, &record); err != nil {
record.Fields.Labels = []string{}
record.Fields.Assignees = []string{}
if typedErr, ok := err.(airtable.ErrClientRequest); ok {
record.Fields.Errors = typedErr.Err.Error()
} else {
record.Fields.Errors = err.Error()
}
if err := table.Update(&record); err != nil {
logger().Error("failed to update record without slices", zap.String("URL", issue.URL), zap.Error(err))
}
}
Expand Down Expand Up @@ -171,6 +179,7 @@ func (i Issue) ToAirtableRecord() airtableRecord {
RepoURL: i.RepoURL,
Body: i.Body,
State: i.State,
Errors: "",
},
}
}
Expand All @@ -187,9 +196,21 @@ type airtableIssue struct {
Type string
Labels []string
Assignees []string
Errors string
}

func (ai airtableIssue) Equals(other airtableIssue) bool {
sameSlice := func(a, b []string) bool {
if a == nil {
a = []string{}
}
if b == nil {
b = []string{}
}
sort.Strings(a)
sort.Strings(b)
return reflect.DeepEqual(a, b)
}
return ai.URL == other.URL &&
ai.Created.Truncate(time.Millisecond).UTC() == other.Created.Truncate(time.Millisecond).UTC() &&
ai.Updated.Truncate(time.Millisecond).UTC() == other.Updated.Truncate(time.Millisecond).UTC() &&
Expand All @@ -199,23 +220,7 @@ func (ai airtableIssue) Equals(other airtableIssue) bool {
ai.Body == other.Body &&
ai.RepoURL == other.RepoURL &&
ai.Type == other.Type &&
reflect.DeepEqual(ai.Labels, other.Labels) &&
reflect.DeepEqual(ai.Assignees, other.Assignees)
}

func (a airtableIssue) Map() map[string]interface{} {
return map[string]interface{}{
"URL": a.URL,
"Created": a.Created,
"Updated": a.Updated,
"Title": a.Title,
"Provider": a.Provider,
"State": a.State,
"Body": a.Body,
"RepoURL": a.RepoURL,
"Type": a.Type,
"Labels": a.Labels,
"Assignees": a.Assignees,
"Errors": "",
}
sameSlice(ai.Labels, other.Labels) &&
sameSlice(ai.Assignees, other.Assignees) &&
ai.Errors == other.Errors
}
6 changes: 4 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ require (
cloud.google.com/go v0.29.0 // indirect
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/awalterschulze/gographviz v0.0.0-20180927133620-e69668a01397
github.com/brianloveswords/airtable v0.0.0-20180329193050-a39294038dd9
github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6 // indirect
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
github.com/fabioberger/airtable-go v3.1.0+incompatible // indirect
github.com/go-chi/chi v3.3.3+incompatible
github.com/go-chi/docgen v1.0.2 // indirect
github.com/go-chi/docgen v1.0.2
github.com/go-chi/render v1.0.1
github.com/go-sql-driver/mysql v1.4.0 // indirect
github.com/google/go-cmp v0.2.0 // indirect
Expand All @@ -24,7 +24,9 @@ require (
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.2.1
github.com/uber-go/atomic v1.3.2 // indirect
github.com/xanzy/go-gitlab v0.11.1
go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
go.uber.org/zap v1.9.1
golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 // indirect
golang.org/x/net v0.0.0-20181003013248-f5e5bdd77824 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/awalterschulze/gographviz v0.0.0-20180927133620-e69668a01397 h1:SzTF/aqwdSBijZUpXvw1ditdkwfU+ONMyocC2CyRtrM=
github.com/awalterschulze/gographviz v0.0.0-20180927133620-e69668a01397/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
github.com/brianloveswords/airtable v0.0.0-20180329193050-a39294038dd9 h1:2oAZVcirE7xEkkA3lQqEB6ElATe5r60zwQoeTQ64WNI=
github.com/brianloveswords/airtable v0.0.0-20180329193050-a39294038dd9/go.mod h1:jXij3SzY3HghKUmTjQYGRIXyCRBkFK+0wztOaI6a9Bk=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6 h1:BZGp1dbKFjqlGmxEpwkDpCWNxVwEYnUPoncIzLiHlPo=
github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/fabioberger/airtable-go v3.1.0+incompatible h1:n5dw+HWBc+hytrVL75xe94EGt7FtNFGDII1tNoWTCAE=
github.com/fabioberger/airtable-go v3.1.0+incompatible/go.mod h1:EoKuSh7EefzhMCyVr6iXPlgFzDgHyZCZ3E5Sg8Cy9GM=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-chi/chi v3.3.3+incompatible h1:KHkmBEMNkwKuK4FdQL7N2wOeB9jnIx7jR5wsuSBEFI8=
Expand Down Expand Up @@ -69,12 +69,16 @@ github.com/spf13/viper v1.2.1 h1:bIcUwXqLseLF3BDAZduuNfekWG87ibtFxi59Bq+oI9M=
github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/xanzy/go-gitlab v0.11.1 h1:kgVxG9YFerbzMnLuwcFio5nwdv/5bCDHgm7WviT+EEE=
github.com/xanzy/go-gitlab v0.11.1/go.mod h1:CRKHkvFWNU6C3AEfqLWjnCNnAs4nj8Zk95rX2S3X6Mw=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 h1:d9qaMM+ODpCq+9We41//fu/sHsTnXcrqd1en3x+GKy4=
go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 h1:Vk3wNqEZwyGyei9yq5ekj7frek2u7HUfffJ1/opblzc=
Expand Down

0 comments on commit 1eb2274

Please sign in to comment.