Skip to content

Commit

Permalink
postgresql integration
Browse files Browse the repository at this point in the history
Signed-off-by: lwsanty <lwsanty@gmail.com>
  • Loading branch information
lwsanty committed Sep 17, 2019
1 parent a83a0bf commit 5bf0bdf
Show file tree
Hide file tree
Showing 6 changed files with 336 additions and 18 deletions.
3 changes: 3 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ script:
- GO111MODULE=on
- make ci-script

services:
- docker

jobs:
include:
- os: linux
Expand Down
44 changes: 26 additions & 18 deletions cmd/gitcollector/subcmd/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,25 @@ func (c *DownloadCmd) Execute(args []string) error {
}

info, err := os.Stat(c.LibPath)
check(err, "wrong path to locate the library")
if err != nil {
log.Errorf(err, "wrong path to locate the library")
return err
}

if !info.IsDir() {
check(
fmt.Errorf("%s isn't a directory", c.LibPath),
"wrong path to locate the library",
)
err := fmt.Errorf("%s isn't a directory", c.LibPath)
log.Errorf(err, "wrong path to locate the library")
return err
}

fs := osfs.New(c.LibPath)

tmpPath, err := ioutil.TempDir(
c.TmpPath, "gitcollector-downloader")
check(err, "unable to create temporal directory")
if err != nil {
log.Errorf(err, "unable to create temporal directory")
return err
}
defer func() {
if err := os.RemoveAll(tmpPath); err != nil {
log.Warningf(
Expand All @@ -88,7 +93,10 @@ func (c *DownloadCmd) Execute(args []string) error {
Transactional: true,
TempFS: temp,
})
check(err, "unable to create borges siva library")
if err != nil {
log.Errorf(err, "unable to create borges siva library")
return err
}

authTokens := map[string]string{}
if c.Token != "" {
Expand Down Expand Up @@ -124,12 +132,16 @@ func (c *DownloadCmd) Execute(args []string) error {

var mc gitcollector.MetricsCollector
if c.MetricsDBURI != "" {
mc = setupMetrics(
mc, err = setupMetrics(
c.MetricsDBURI,
c.MetricsDBTable,
orgs,
c.MetricsSync,
)
if err != nil {
log.Errorf(err, "failed to setup metrics")
return err
}

log.Debugf("metrics collection activated: sync timeout %d",
c.MetricsSync)
Expand Down Expand Up @@ -158,20 +170,16 @@ func (c *DownloadCmd) Execute(args []string) error {
return nil
}

func check(err error, message string) {
if err != nil {
log.Errorf(err, message)
os.Exit(1)
}
}

func setupMetrics(
uri, table string,
orgs []string,
metricSync int64,
) gitcollector.MetricsCollector {
) (gitcollector.MetricsCollector, error) {
db, err := metrics.PrepareDB(uri, table, orgs)
check(err, "metrics database")
if err != nil {
log.Errorf(err, "metrics database")
return nil, err
}

mcs := make(map[string]*metrics.Collector, len(orgs))
for _, org := range orgs {
Expand All @@ -184,7 +192,7 @@ func setupMetrics(
mcs[org] = mc
}

return metrics.NewCollectorByOrg(mcs)
return metrics.NewCollectorByOrg(mcs), nil
}

func runGHOrgProviders(
Expand Down
13 changes: 13 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,18 @@ module github.com/src-d/gitcollector
go 1.12

require (
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/gliderlabs/ssh v0.2.2 // indirect
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-querystring v1.0.0 // indirect
github.com/google/uuid v1.1.1
github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d // indirect
github.com/kevinburke/ssh_config v0.0.0-20190630040420-2e50c441276c // indirect
Expand All @@ -15,6 +23,10 @@ require (
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/opencontainers/runc v0.1.1 // indirect
github.com/ory/dockertest v3.3.5+incompatible
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/src-d/envconfig v1.0.0 // indirect
github.com/src-d/go-borges v0.0.0-20190704083038-44867e8f2a2a
Expand All @@ -30,4 +42,5 @@ require (
gopkg.in/src-d/go-errors.v1 v1.0.0
gopkg.in/src-d/go-git.v4 v4.12.0
gopkg.in/src-d/go-log.v1 v1.0.2
gotest.tools v2.2.0+incompatible // indirect
)
29 changes: 29 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,13 +1,27 @@
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw=
github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
Expand Down Expand Up @@ -36,6 +50,8 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI=
github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
Expand Down Expand Up @@ -75,6 +91,14 @@ github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y=
github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA=
github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs=
github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand All @@ -84,6 +108,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/src-d/envconfig v1.0.0 h1:/AJi6DtjFhZKNx3OB2qMsq7y4yT5//AeSZIe7rk+PX8=
Expand Down Expand Up @@ -132,13 +157,15 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190609082536-301114b31cce/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -183,3 +210,5 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
144 changes: 144 additions & 0 deletions integration/helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package integration

import (
"database/sql"
"io/ioutil"
"os"
"path/filepath"

"github.com/src-d/gitcollector/cmd/gitcollector/subcmd"

"github.com/ory/dockertest"
)

const (
dockerImage = "postgres"
dockerVersion = "9.6"
)

type helper struct {
address string
cmd subcmd.DownloadCmd
cli *sql.DB
closers []func()
}

func NewHelper(orgs string) (*helper, error) {
lib, err := ioutil.TempDir("", "gcol-lib")
if err != nil {
removePaths(lib)
return nil, err
}

tmp, err := ioutil.TempDir("", "gcol-tmp")
if err != nil {
removePaths(tmp, lib)
return nil, err
}

addr, contClose, err := preparePostgres()
if err != nil {
return nil, err
}

cli, err := sql.Open("postgres", addr)
if err != nil {
contClose()
return nil, err
}

return &helper{
address: addr,
cmd: subcmd.DownloadCmd{
Orgs: orgs,
LibPath: lib,
TmpPath: tmp,
// Tests may fail due to request rate limit until https://github.com/src-d/gitcollector/issues/74 is fixed
Token: os.Getenv("GITHUB_TOKEN"),
},
cli: cli,
closers: []func(){
func() {
cli.Close()
contClose()
},
contClose,
},
}, nil
}

func (h *helper) Exec() error { return h.cmd.Execute(nil) }

func (h *helper) Cleanup() {
removeDirsContents(h.cmd.TmpPath, h.cmd.LibPath)
}

func (h *helper) Close() {
for _, c := range h.closers {
c()
}
removePaths(h.cmd.TmpPath, h.cmd.LibPath)
}

func preparePostgres() (string, func(), error) {
pool, err := dockertest.NewPool("")
if err != nil {
return "", func() {}, err
}

cont, err := pool.Run(dockerImage, dockerVersion, []string{
"POSTGRES_PASSWORD=postgres",
})
if err != nil {
return "", func() {}, err
}

const port = "5432/tcp"
addr := `postgres://postgres:postgres@` + cont.GetHostPort(port) + "?sslmode=disable"
if err := pool.Retry(func() error {
cli, err := sql.Open("postgres", addr)
if err != nil {
return err
}
defer cli.Close()
return cli.Ping()
}); err != nil {
cont.Close()
return "", func() {}, err
}

return addr, func() {
cont.Close()
}, nil
}

func removePaths(paths ...string) {
for _, p := range paths {
os.RemoveAll(p)
}
}

func removeDirsContents(dirs ...string) {
for _, d := range dirs {
removeContents(d)
}
}

func removeContents(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
err = os.RemoveAll(filepath.Join(dir, name))
if err != nil {
return err
}
}
return nil
}
Loading

0 comments on commit 5bf0bdf

Please sign in to comment.