From a3b8b57687758d3fcbb9673d7cc8289c26bd418b Mon Sep 17 00:00:00 2001 From: Mario Lima Date: Wed, 16 Oct 2019 11:13:09 +0100 Subject: [PATCH] Renamed repo from repocrawl to repocrawler --- README.md | 14 +++++++------- cmd/crawler-cli/main.go | 6 +++--- cmd/crawler-cli/main_test.go | 6 +++--- cmd/utils/webserver/server.go | 2 +- configs/certs/server.crt | 35 ++++++++++++++++++----------------- configs/certs/server.csr | 30 +++++++++++++++--------------- configs/certs/server.csr.cnf | 2 +- configs/certs/v3.ext | 2 +- docker.sh | 6 +++--- go.mod | 7 ++----- pkg/bitbucket/service.go | 2 +- pkg/crawler/crawler.go | 8 ++++---- pkg/crawler/deepcrawl.go | 2 +- pkg/crawler/graph.go | 2 +- pkg/crawler/match.go | 2 +- pkg/crawler/notification.go | 3 ++- pkg/crawler/task.go | 2 +- pkg/github/service.go | 2 +- 18 files changed, 66 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 11e8d27fd..d4c76c342 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RepoCrawl +# RepoCrawler Crawl GitHub/Bitbucket/Gitlab/Git repositories in search for unsafely stored secrets. Completely written in Go ## Overview @@ -42,8 +42,8 @@ This tool crawls repositories on various Git services using a variety of methods ## Instalation ### From source ```sht -go get github.com/mariolima/repocrawl -cd ~/go/src/github.com/mariolima/repocrawl/cmd/crawler-cli +go get github.com/mariolima/repocrawler +cd ~/go/src/github.com/mariolima/repocrawler/cmd/crawler-cli go build . export LOG_LEVEL=info export GITHUB_ACCESS_TOKEN=TOKEN @@ -51,10 +51,10 @@ export GITHUB_ACCESS_TOKEN=TOKEN ``` ### Using Docker ```sh -git clone github.com/mariolima/repocrawl -cd repocrawl -docker build . -t repocrawl -docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' -e 'SLACK_WEBHOOK=YOURWEBHOOK' repocrawl -h +git clone github.com/mariolima/repocrawler +cd repocrawler +docker build . -t repocrawler +docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' -e 'SLACK_WEBHOOK=YOURWEBHOOK' repocrawler -h ``` diff --git a/cmd/crawler-cli/main.go b/cmd/crawler-cli/main.go index a08657967..955bd333d 100644 --- a/cmd/crawler-cli/main.go +++ b/cmd/crawler-cli/main.go @@ -8,9 +8,9 @@ import ( "fmt" "strings" - "github.com/mariolima/repocrawl/cmd/utils" // used to Highlight matches with colors - "github.com/mariolima/repocrawl/cmd/utils/webserver" - "github.com/mariolima/repocrawl/pkg/crawler" + "github.com/mariolima/repocrawler/cmd/utils" // used to Highlight matches with colors + "github.com/mariolima/repocrawler/cmd/utils/webserver" + "github.com/mariolima/repocrawler/pkg/crawler" ) var ( diff --git a/cmd/crawler-cli/main_test.go b/cmd/crawler-cli/main_test.go index df4b833d4..18b0b494a 100644 --- a/cmd/crawler-cli/main_test.go +++ b/cmd/crawler-cli/main_test.go @@ -10,9 +10,9 @@ import ( "fmt" - "github.com/mariolima/repocrawl/cmd/utils" // used to Highlight matches with colors - "github.com/mariolima/repocrawl/cmd/utils/webserver" - "github.com/mariolima/repocrawl/pkg/crawler" + "github.com/mariolima/repocrawler/cmd/utils" // used to Highlight matches with colors + "github.com/mariolima/repocrawler/cmd/utils/webserver" + "github.com/mariolima/repocrawler/pkg/crawler" "github.com/stretchr/testify/assert" ) diff --git a/cmd/utils/webserver/server.go b/cmd/utils/webserver/server.go index 5e29a5f0b..3cf5ae10c 100644 --- a/cmd/utils/webserver/server.go +++ b/cmd/utils/webserver/server.go @@ -6,7 +6,7 @@ import ( "net/http" "time" - "github.com/mariolima/repocrawl/pkg/crawler" + "github.com/mariolima/repocrawler/pkg/crawler" log "github.com/sirupsen/logrus" ) diff --git a/configs/certs/server.crt b/configs/certs/server.crt index c76af4416..543d63743 100644 --- a/configs/certs/server.crt +++ b/configs/certs/server.crt @@ -1,23 +1,24 @@ -----BEGIN CERTIFICATE----- -MIID7DCCAtSgAwIBAgIUew++RBcC6T80HgJdLLmjxmG8Ky0wDQYJKoZIhvcNAQEL +MIID+jCCAuKgAwIBAgIUew++RBcC6T80HgJdLLmjxmG8Ky4wDQYJKoZIhvcNAQEL BQAwVDELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDENMAsGA1UEAwwEZ29iaDAeFw0xOTA3 -MDgxMDQxNTVaFw0yMDExMTkxMDQxNTVaMIG9MQswCQYDVQQGEwJVUzERMA8GA1UE +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDENMAsGA1UEAwwEZ29iaDAeFw0xOTEw +MTYxMDAzMDZaFw0yMTAyMjcxMDAzMDZaMIHEMQswCQYDVQQGEwJVUzERMA8GA1UE CAwITmV3IFlvcmsxEjAQBgNVBAcMCVJvY2hlc3RlcjESMBAGA1UECgwJRW5kIFBv aW50MRcwFQYDVQQLDA5UZXN0aW5nIERvbWFpbjFLMEkGCSqGSIb3DQEJARY8eW91 ci1hZG1pbmlzdHJhdGl2ZS1hZGRyZXNzQHlvdXItYXdlc29tZS1leGlzdGluZy1k -b21haW4uY29tMQ0wCwYDVQQDDARnb2JoMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAprSkTxCXKQPq0F8EvYFzEupStANyPKXjt5nLORA1ss/PDEODpz+R -m5PAoRtlBnA3McJFu6o0dLfm6fxS93Mr2JOFhaeg7aKCSaVjYGEQNIKYcv6SKx+y -pAH3Qwsga3KIrFeaLwMQPw7WmRv0pCrJ0KkS3jCSh0x+yiq3NtRz3C3DufZPM0WT -SDZQAarU5B4u9sl7NGrWapOCOkoxahwgcdCRNiV1/wDrN5n3+6BKnVWRDuy1IKBs -IG5ECNKqc29lPmjDYr0eqK3IpMlpC74lHU1y9pXbTMYzWCK1s26Sn0ni5VkIPeEg -NLzkhoEAtCK1eTGb0hS9xx21aLdqLfxJLwIDAQABo0wwSjAfBgNVHSMEGDAWgBTn -+OspY9tHYupMTQYA7OMDhsDl/TAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE8DAPBgNV -HREECDAGggRnb2JoMA0GCSqGSIb3DQEBCwUAA4IBAQDHEAmPTtDDGXndfC7gr6Fp -YAlLlRTstfn8H/Pic3ULL+/Xy9SpklJ5Em15nZMGvaHtO2v2K1y3rsr1BTKVVZkj -YJqWQkMm0TyRn6ViE2Ts7jkZKw/2lR/FQ2iaD4D4NnWQpUkXxv49TYXCkbRDYDK5 -rV2f9f1nqMp0frk85g9ZXBwErijX2YK+VLWxj4YxPC1JoznhezgaIgjLUae9GDSe -k7odk1lr5AOGjpSUlzViAjH06jmwyPkGXkrY9l+lU6Oe57ZurQ8LK40kHxZ7Q6Ze -CooZZPVumCPKelzAywCwjulsjkKRMvsPdWpS3f19TGLxty+FwrihsUJ1zpUjs0nz +b21haW4uY29tMRQwEgYDVQQDDAtyZXBvY3Jhd2xlcjCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAMTosdtEoVQrzDj4+hwUcLT++tkkbO/zFQ11dI3b9JCU +M2DEv4LVCYRmDnzNrHmyGhD8R9oqQfFDRut++spI4lMtgDJslpqVIyugCpsv/8ey +NAUEFDdJ+CMXCoHuB1GiCpyNtVC/i4Bv/QcbSmeZJe5qcHepRa4VhBZPLZVxlhYC +Evfq2JY+NARKY6r0GsfNgP7RIq4M0lrauLU/8+TOuc5ghXnslPqm9NuvaEpY5oba +vxcdk14Sny5lB1Y/9QBgbH2l8kACYVQDwavNUhAY/mCJW/ivAoChntdDcdkHecrW +uy6TPCSD+I+TfjWmySYbSDvirU0k1t8R9D9fSdTLKIUCAwEAAaNTMFEwHwYDVR0j +BBgwFoAU5/jrKWPbR2LqTE0GAOzjA4bA5f0wCQYDVR0TBAIwADALBgNVHQ8EBAMC +BPAwFgYDVR0RBA8wDYILcmVwb2NyYXdsZXIwDQYJKoZIhvcNAQELBQADggEBABJ0 +WUuV5Vl9FFOHjNjtgQ7fq3xMEibORxAZXh4lXSwXTFeU/VaYmHQS0ibLAkS+9w6D ++OQh/YN4gA+Xb/T4DOejXIw0Ulu4wfzw3QHrqij27WVyWjXr8HGZD1NbkpVJWHt0 +xbkICZ5o6X/yv8J3KqmX8IkN+tVZRY7p3jBPAXXih8HdCsZdXaUeMi62uFOrgOmM +WjUb+dOLd3Q5dxmWnuVM2U9p1t/Xt79AlVqY6JY8cF6x1zzZCR/yWwD8EhcaYu07 +9jnyd5CKmsb+01ZSBR/ZDzxiFuOuSEPrdnZ34YHk613LFiH6YXZfSt9dRheMxC3O +c//qmzE3Ttje2Q8j5S8= -----END CERTIFICATE----- diff --git a/configs/certs/server.csr b/configs/certs/server.csr index ee6574279..f0af6f174 100644 --- a/configs/certs/server.csr +++ b/configs/certs/server.csr @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE REQUEST----- -MIIDAzCCAesCAQAwgb0xCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazES +MIIDCjCCAfICAQAwgcQxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazES MBAGA1UEBwwJUm9jaGVzdGVyMRIwEAYDVQQKDAlFbmQgUG9pbnQxFzAVBgNVBAsM DlRlc3RpbmcgRG9tYWluMUswSQYJKoZIhvcNAQkBFjx5b3VyLWFkbWluaXN0cmF0 -aXZlLWFkZHJlc3NAeW91ci1hd2Vzb21lLWV4aXN0aW5nLWRvbWFpbi5jb20xDTAL -BgNVBAMMBGdvYmgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmtKRP -EJcpA+rQXwS9gXMS6lK0A3I8peO3mcs5EDWyz88MQ4OnP5Gbk8ChG2UGcDcxwkW7 -qjR0t+bp/FL3cyvYk4WFp6DtooJJpWNgYRA0gphy/pIrH7KkAfdDCyBrcoisV5ov -AxA/DtaZG/SkKsnQqRLeMJKHTH7KKrc21HPcLcO59k8zRZNINlABqtTkHi72yXs0 -atZqk4I6SjFqHCBx0JE2JXX/AOs3mff7oEqdVZEO7LUgoGwgbkQI0qpzb2U+aMNi -vR6orcikyWkLviUdTXL2ldtMxjNYIrWzbpKfSeLlWQg94SA0vOSGgQC0IrV5MZvS -FL3HHbVot2ot/EkvAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAcfdtZ4/muAhK -hbLUvsj9HJbK1sypwdrV30UydmxdGJ80lSpgK/wy4CPuXetX/glvmOWvkf4PpTmq -Kwm4LW1Ug2KSTu6TPExrEC0IrGfoZH1J+GMmkVjlovNcj/J3TJzZipkB62bGZ/Mo -np4wzWmsv3sRR8WORL3WL3/ff7nkqlHkTCzm+V5lPve1s9QKVIgx2CSPmLWbbNiX -T4Br7KodmQG51R/GiKN8rHLWBZD2HdrOzznMllRIT5yS5/xjrwiEMue16u/gEYBe -xqAADqSFJVq+szGnvJ0lWmEbqFZMWokBCuUurjiEhYmPLzMFV+jf+NLgZGT1yR+l -UUViyPoLkw== +aXZlLWFkZHJlc3NAeW91ci1hd2Vzb21lLWV4aXN0aW5nLWRvbWFpbi5jb20xFDAS +BgNVBAMMC3JlcG9jcmF3bGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAxOix20ShVCvMOPj6HBRwtP762SRs7/MVDXV0jdv0kJQzYMS/gtUJhGYOfM2s +ebIaEPxH2ipB8UNG6376ykjiUy2AMmyWmpUjK6AKmy//x7I0BQQUN0n4IxcKge4H +UaIKnI21UL+LgG/9BxtKZ5kl7mpwd6lFrhWEFk8tlXGWFgIS9+rYlj40BEpjqvQa +x82A/tEirgzSWtq4tT/z5M65zmCFeeyU+qb0269oSljmhtq/Fx2TXhKfLmUHVj/1 +AGBsfaXyQAJhVAPBq81SEBj+YIlb+K8CgKGe10Nx2Qd5yta7LpM8JIP4j5N+NabJ +JhtIO+KtTSTW3xH0P19J1MsohQIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBADbj +6iFVjeTN61BIVUlW8ZzKLMI5+1Jm2sU91Gq8nkfVjq1JmE4I8bRKHyr5bpwke95l +ZEsl9m4MazM45VzW6n6aBbESXbrGaxzHVoR9U9dHAAjc/my34ErsHAHLHPdtis/c +M9QEJbQ/84AfXIF3xAOvT/Bt0R/AMJyQiTlPCcYyCw03avVyqjNfl7MEOXXP+Hd2 +m6XhkvNc2mYqtWl4/dRyFl9qe/9lwKB7sTW2/Mn9zSf6NUoJKsnUnBZ+h97LaE4k +OAXxYo5XBInowbDnJG5EyQrBPBF9b+jXJVtGL4PnC0Yd7IasEOV/jQPTf3o4RkII +2d3meMHkO1eiZYv08L8= -----END CERTIFICATE REQUEST----- diff --git a/configs/certs/server.csr.cnf b/configs/certs/server.csr.cnf index 3fb6a13d8..2337897de 100755 --- a/configs/certs/server.csr.cnf +++ b/configs/certs/server.csr.cnf @@ -11,4 +11,4 @@ L=Rochester O=End Point OU=Testing Domain emailAddress=your-administrative-address@your-awesome-existing-domain.com -CN = gobh +CN = repocrawler diff --git a/configs/certs/v3.ext b/configs/certs/v3.ext index b23e2e040..614e970db 100755 --- a/configs/certs/v3.ext +++ b/configs/certs/v3.ext @@ -4,4 +4,4 @@ keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment subjectAltName = @alt_names [alt_names] -DNS.1 = gobh +DNS.1 = repocrawler diff --git a/docker.sh b/docker.sh index bfc6ca1ff..f45807590 100755 --- a/docker.sh +++ b/docker.sh @@ -1,3 +1,3 @@ -docker image rm -f repocrawl -docker build . -t repocrawl -docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' repocrawl -h +docker image rm -f repocrawler +docker build . -t repocrawler +docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' repocrawler -h diff --git a/go.mod b/go.mod index 2ab2fd53a..4eb860b12 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,13 @@ -module github.com/mariolima/repocrawl +module github.com/mariolima/repocrawler -go 1.12 +go 1.13 require ( github.com/bndr/gopencils v0.0.0-20161113114152-22e283ad7611 - github.com/gammazero/workerpool v0.0.0-20190608213748-0ed5e40ec55e github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.0.0 // indirect github.com/gorilla/websocket v1.4.1 github.com/logrusorgru/aurora v0.0.0-20190803045625-94edacc10f9b - github.com/onsi/ginkgo v1.10.1 - github.com/onsi/gomega v1.7.0 github.com/pkg/profile v1.3.0 github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.4.2 diff --git a/pkg/bitbucket/service.go b/pkg/bitbucket/service.go index 073026046..e61b8cc88 100644 --- a/pkg/bitbucket/service.go +++ b/pkg/bitbucket/service.go @@ -3,7 +3,7 @@ package bitbucket import ( log "github.com/sirupsen/logrus" - "github.com/mariolima/repocrawl/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc + "github.com/mariolima/repocrawler/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc "github.com/bndr/gopencils" diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go index d619abccf..d869d8732 100644 --- a/pkg/crawler/crawler.go +++ b/pkg/crawler/crawler.go @@ -3,10 +3,10 @@ package crawler import ( "sync" - "github.com/mariolima/repocrawl/internal/entities" - "github.com/mariolima/repocrawl/pkg/bitbucket" - "github.com/mariolima/repocrawl/pkg/github" - _ "github.com/mariolima/repocrawl/pkg/gitlab" //TODO + "github.com/mariolima/repocrawler/internal/entities" + "github.com/mariolima/repocrawler/pkg/bitbucket" + "github.com/mariolima/repocrawler/pkg/github" + _ "github.com/mariolima/repocrawler/pkg/gitlab" //TODO "bufio" "regexp" diff --git a/pkg/crawler/deepcrawl.go b/pkg/crawler/deepcrawl.go index 3ddd01e7f..f7ee99df4 100644 --- a/pkg/crawler/deepcrawl.go +++ b/pkg/crawler/deepcrawl.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/mariolima/repocrawl/internal/entities" + "github.com/mariolima/repocrawler/internal/entities" _ "gopkg.in/src-d/go-billy.v4/memfs" //??????????????????? "gopkg.in/src-d/go-git.v4" //It's def heavy but gets the job done - any alternatives for commit crawling? "gopkg.in/src-d/go-git.v4/plumbing/object" diff --git a/pkg/crawler/graph.go b/pkg/crawler/graph.go index d945ed39a..8d03dc1d0 100644 --- a/pkg/crawler/graph.go +++ b/pkg/crawler/graph.go @@ -5,7 +5,7 @@ import ( "fmt" "sync" - "github.com/mariolima/repocrawl/internal/entities" + "github.com/mariolima/repocrawler/internal/entities" _ "github.com/twmb/algoimpl/go/graph" // Graph library /w most of Algos and Structs ) diff --git a/pkg/crawler/match.go b/pkg/crawler/match.go index 6e1751f72..b80744bc4 100644 --- a/pkg/crawler/match.go +++ b/pkg/crawler/match.go @@ -1,7 +1,7 @@ package crawler import ( - "github.com/mariolima/repocrawl/internal/entities" + "github.com/mariolima/repocrawler/internal/entities" ) // Match Secret found while crawling a repository diff --git a/pkg/crawler/notification.go b/pkg/crawler/notification.go index 89b48a547..258fa96b9 100644 --- a/pkg/crawler/notification.go +++ b/pkg/crawler/notification.go @@ -2,7 +2,8 @@ package crawler import ( "fmt" - "github.com/mariolima/repocrawl/cmd/utils" + + "github.com/mariolima/repocrawler/cmd/utils" // log "github.com/sirupsen/logrus" ) diff --git a/pkg/crawler/task.go b/pkg/crawler/task.go index 61207a0b1..5b8973de0 100644 --- a/pkg/crawler/task.go +++ b/pkg/crawler/task.go @@ -1,7 +1,7 @@ package crawler import ( - "github.com/mariolima/repocrawl/internal/entities" + "github.com/mariolima/repocrawler/internal/entities" log "github.com/sirupsen/logrus" ) diff --git a/pkg/github/service.go b/pkg/github/service.go index 3bd76d20b..7dd79ef3c 100644 --- a/pkg/github/service.go +++ b/pkg/github/service.go @@ -8,7 +8,7 @@ import ( log "github.com/sirupsen/logrus" - "github.com/mariolima/repocrawl/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc + "github.com/mariolima/repocrawler/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc ) // GitHubCrawler Contains API_KEY and Client for API calls