Renamed repo from repocrawl to repocrawler

README.md

@@ -1,4 +1,4 @@
-# RepoCrawl
+# RepoCrawler
 Crawl GitHub/Bitbucket/Gitlab/Git repositories in search for unsafely stored secrets. Completely written in Go
 
 ## Overview
@@ -42,19 +42,19 @@ This tool crawls repositories on various Git services using a variety of methods
 ## Instalation
 ### From source
 ```sht
-go get github.com/mariolima/repocrawl
-cd ~/go/src/github.com/mariolima/repocrawl/cmd/crawler-cli
+go get github.com/mariolima/repocrawler
+cd ~/go/src/github.com/mariolima/repocrawler/cmd/crawler-cli
 go build .
 export LOG_LEVEL=info
 export GITHUB_ACCESS_TOKEN=TOKEN
 ./crawler-cli -h
 ```
 ### Using Docker
 ```sh
-git clone github.com/mariolima/repocrawl
-cd repocrawl
-docker build . -t repocrawl
-docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' -e 'SLACK_WEBHOOK=YOURWEBHOOK' repocrawl -h
+git clone github.com/mariolima/repocrawler
+cd repocrawler
+docker build . -t repocrawler
+docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' -e 'SLACK_WEBHOOK=YOURWEBHOOK' repocrawler -h
 ```
 
 

cmd/crawler-cli/main.go

@@ -8,9 +8,9 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/mariolima/repocrawl/cmd/utils" // used to Highlight matches with colors
-	"github.com/mariolima/repocrawl/cmd/utils/webserver"
-	"github.com/mariolima/repocrawl/pkg/crawler"
+	"github.com/mariolima/repocrawler/cmd/utils" // used to Highlight matches with colors
+	"github.com/mariolima/repocrawler/cmd/utils/webserver"
+	"github.com/mariolima/repocrawler/pkg/crawler"
 )
 
 var (

cmd/crawler-cli/main_test.go

@@ -10,9 +10,9 @@ import (
 
 	"fmt"
 
-	"github.com/mariolima/repocrawl/cmd/utils" // used to Highlight matches with colors
-	"github.com/mariolima/repocrawl/cmd/utils/webserver"
-	"github.com/mariolima/repocrawl/pkg/crawler"
+	"github.com/mariolima/repocrawler/cmd/utils" // used to Highlight matches with colors
+	"github.com/mariolima/repocrawler/cmd/utils/webserver"
+	"github.com/mariolima/repocrawler/pkg/crawler"
 	"github.com/stretchr/testify/assert"
 )
 

cmd/utils/webserver/server.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/mariolima/repocrawl/pkg/crawler"
+	"github.com/mariolima/repocrawler/pkg/crawler"
 	log "github.com/sirupsen/logrus"
 )
 

configs/certs/server.crt

@@ -1,23 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIID7DCCAtSgAwIBAgIUew++RBcC6T80HgJdLLmjxmG8Ky0wDQYJKoZIhvcNAQEL
+MIID+jCCAuKgAwIBAgIUew++RBcC6T80HgJdLLmjxmG8Ky4wDQYJKoZIhvcNAQEL
 BQAwVDELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDENMAsGA1UEAwwEZ29iaDAeFw0xOTA3
-MDgxMDQxNTVaFw0yMDExMTkxMDQxNTVaMIG9MQswCQYDVQQGEwJVUzERMA8GA1UE
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDENMAsGA1UEAwwEZ29iaDAeFw0xOTEw
+MTYxMDAzMDZaFw0yMTAyMjcxMDAzMDZaMIHEMQswCQYDVQQGEwJVUzERMA8GA1UE
 CAwITmV3IFlvcmsxEjAQBgNVBAcMCVJvY2hlc3RlcjESMBAGA1UECgwJRW5kIFBv
 aW50MRcwFQYDVQQLDA5UZXN0aW5nIERvbWFpbjFLMEkGCSqGSIb3DQEJARY8eW91
 ci1hZG1pbmlzdHJhdGl2ZS1hZGRyZXNzQHlvdXItYXdlc29tZS1leGlzdGluZy1k
-b21haW4uY29tMQ0wCwYDVQQDDARnb2JoMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
-MIIBCgKCAQEAprSkTxCXKQPq0F8EvYFzEupStANyPKXjt5nLORA1ss/PDEODpz+R
-m5PAoRtlBnA3McJFu6o0dLfm6fxS93Mr2JOFhaeg7aKCSaVjYGEQNIKYcv6SKx+y
-pAH3Qwsga3KIrFeaLwMQPw7WmRv0pCrJ0KkS3jCSh0x+yiq3NtRz3C3DufZPM0WT
-SDZQAarU5B4u9sl7NGrWapOCOkoxahwgcdCRNiV1/wDrN5n3+6BKnVWRDuy1IKBs
-IG5ECNKqc29lPmjDYr0eqK3IpMlpC74lHU1y9pXbTMYzWCK1s26Sn0ni5VkIPeEg
-NLzkhoEAtCK1eTGb0hS9xx21aLdqLfxJLwIDAQABo0wwSjAfBgNVHSMEGDAWgBTn
-+OspY9tHYupMTQYA7OMDhsDl/TAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE8DAPBgNV
-HREECDAGggRnb2JoMA0GCSqGSIb3DQEBCwUAA4IBAQDHEAmPTtDDGXndfC7gr6Fp
-YAlLlRTstfn8H/Pic3ULL+/Xy9SpklJ5Em15nZMGvaHtO2v2K1y3rsr1BTKVVZkj
-YJqWQkMm0TyRn6ViE2Ts7jkZKw/2lR/FQ2iaD4D4NnWQpUkXxv49TYXCkbRDYDK5
-rV2f9f1nqMp0frk85g9ZXBwErijX2YK+VLWxj4YxPC1JoznhezgaIgjLUae9GDSe
-k7odk1lr5AOGjpSUlzViAjH06jmwyPkGXkrY9l+lU6Oe57ZurQ8LK40kHxZ7Q6Ze
-CooZZPVumCPKelzAywCwjulsjkKRMvsPdWpS3f19TGLxty+FwrihsUJ1zpUjs0nz
+b21haW4uY29tMRQwEgYDVQQDDAtyZXBvY3Jhd2xlcjCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAMTosdtEoVQrzDj4+hwUcLT++tkkbO/zFQ11dI3b9JCU
+M2DEv4LVCYRmDnzNrHmyGhD8R9oqQfFDRut++spI4lMtgDJslpqVIyugCpsv/8ey
+NAUEFDdJ+CMXCoHuB1GiCpyNtVC/i4Bv/QcbSmeZJe5qcHepRa4VhBZPLZVxlhYC
+Evfq2JY+NARKY6r0GsfNgP7RIq4M0lrauLU/8+TOuc5ghXnslPqm9NuvaEpY5oba
+vxcdk14Sny5lB1Y/9QBgbH2l8kACYVQDwavNUhAY/mCJW/ivAoChntdDcdkHecrW
+uy6TPCSD+I+TfjWmySYbSDvirU0k1t8R9D9fSdTLKIUCAwEAAaNTMFEwHwYDVR0j
+BBgwFoAU5/jrKWPbR2LqTE0GAOzjA4bA5f0wCQYDVR0TBAIwADALBgNVHQ8EBAMC
+BPAwFgYDVR0RBA8wDYILcmVwb2NyYXdsZXIwDQYJKoZIhvcNAQELBQADggEBABJ0
+WUuV5Vl9FFOHjNjtgQ7fq3xMEibORxAZXh4lXSwXTFeU/VaYmHQS0ibLAkS+9w6D
++OQh/YN4gA+Xb/T4DOejXIw0Ulu4wfzw3QHrqij27WVyWjXr8HGZD1NbkpVJWHt0
+xbkICZ5o6X/yv8J3KqmX8IkN+tVZRY7p3jBPAXXih8HdCsZdXaUeMi62uFOrgOmM
+WjUb+dOLd3Q5dxmWnuVM2U9p1t/Xt79AlVqY6JY8cF6x1zzZCR/yWwD8EhcaYu07
+9jnyd5CKmsb+01ZSBR/ZDzxiFuOuSEPrdnZ34YHk613LFiH6YXZfSt9dRheMxC3O
+c//qmzE3Ttje2Q8j5S8=
 -----END CERTIFICATE-----

configs/certs/server.csr

@@ -1,19 +1,19 @@
 -----BEGIN CERTIFICATE REQUEST-----
-MIIDAzCCAesCAQAwgb0xCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazES
+MIIDCjCCAfICAQAwgcQxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazES
 MBAGA1UEBwwJUm9jaGVzdGVyMRIwEAYDVQQKDAlFbmQgUG9pbnQxFzAVBgNVBAsM
 DlRlc3RpbmcgRG9tYWluMUswSQYJKoZIhvcNAQkBFjx5b3VyLWFkbWluaXN0cmF0
-aXZlLWFkZHJlc3NAeW91ci1hd2Vzb21lLWV4aXN0aW5nLWRvbWFpbi5jb20xDTAL
-BgNVBAMMBGdvYmgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmtKRP
-EJcpA+rQXwS9gXMS6lK0A3I8peO3mcs5EDWyz88MQ4OnP5Gbk8ChG2UGcDcxwkW7
-qjR0t+bp/FL3cyvYk4WFp6DtooJJpWNgYRA0gphy/pIrH7KkAfdDCyBrcoisV5ov
-AxA/DtaZG/SkKsnQqRLeMJKHTH7KKrc21HPcLcO59k8zRZNINlABqtTkHi72yXs0
-atZqk4I6SjFqHCBx0JE2JXX/AOs3mff7oEqdVZEO7LUgoGwgbkQI0qpzb2U+aMNi
-vR6orcikyWkLviUdTXL2ldtMxjNYIrWzbpKfSeLlWQg94SA0vOSGgQC0IrV5MZvS
-FL3HHbVot2ot/EkvAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAcfdtZ4/muAhK
-hbLUvsj9HJbK1sypwdrV30UydmxdGJ80lSpgK/wy4CPuXetX/glvmOWvkf4PpTmq
-Kwm4LW1Ug2KSTu6TPExrEC0IrGfoZH1J+GMmkVjlovNcj/J3TJzZipkB62bGZ/Mo
-np4wzWmsv3sRR8WORL3WL3/ff7nkqlHkTCzm+V5lPve1s9QKVIgx2CSPmLWbbNiX
-T4Br7KodmQG51R/GiKN8rHLWBZD2HdrOzznMllRIT5yS5/xjrwiEMue16u/gEYBe
-xqAADqSFJVq+szGnvJ0lWmEbqFZMWokBCuUurjiEhYmPLzMFV+jf+NLgZGT1yR+l
-UUViyPoLkw==
+aXZlLWFkZHJlc3NAeW91ci1hd2Vzb21lLWV4aXN0aW5nLWRvbWFpbi5jb20xFDAS
+BgNVBAMMC3JlcG9jcmF3bGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAxOix20ShVCvMOPj6HBRwtP762SRs7/MVDXV0jdv0kJQzYMS/gtUJhGYOfM2s
+ebIaEPxH2ipB8UNG6376ykjiUy2AMmyWmpUjK6AKmy//x7I0BQQUN0n4IxcKge4H
+UaIKnI21UL+LgG/9BxtKZ5kl7mpwd6lFrhWEFk8tlXGWFgIS9+rYlj40BEpjqvQa
+x82A/tEirgzSWtq4tT/z5M65zmCFeeyU+qb0269oSljmhtq/Fx2TXhKfLmUHVj/1
+AGBsfaXyQAJhVAPBq81SEBj+YIlb+K8CgKGe10Nx2Qd5yta7LpM8JIP4j5N+NabJ
+JhtIO+KtTSTW3xH0P19J1MsohQIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBADbj
+6iFVjeTN61BIVUlW8ZzKLMI5+1Jm2sU91Gq8nkfVjq1JmE4I8bRKHyr5bpwke95l
+ZEsl9m4MazM45VzW6n6aBbESXbrGaxzHVoR9U9dHAAjc/my34ErsHAHLHPdtis/c
+M9QEJbQ/84AfXIF3xAOvT/Bt0R/AMJyQiTlPCcYyCw03avVyqjNfl7MEOXXP+Hd2
+m6XhkvNc2mYqtWl4/dRyFl9qe/9lwKB7sTW2/Mn9zSf6NUoJKsnUnBZ+h97LaE4k
+OAXxYo5XBInowbDnJG5EyQrBPBF9b+jXJVtGL4PnC0Yd7IasEOV/jQPTf3o4RkII
+2d3meMHkO1eiZYv08L8=
 -----END CERTIFICATE REQUEST-----

configs/certs/server.csr.cnf

@@ -11,4 +11,4 @@ L=Rochester
 O=End Point
 OU=Testing Domain
 emailAddress=your-administrative-address@your-awesome-existing-domain.com
-CN = gobh
+CN = repocrawler

configs/certs/v3.ext

@@ -4,4 +4,4 @@ keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
 subjectAltName = @alt_names
 
 [alt_names]
-DNS.1 = gobh
+DNS.1 = repocrawler

docker.sh

@@ -1,3 +1,3 @@
-docker image rm -f repocrawl
-docker build . -t repocrawl
-docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' repocrawl -h
+docker image rm -f repocrawler
+docker build . -t repocrawler
+docker run -it -e 'GITHUB_ACCESS_TOKEN=TOKEN' repocrawler -h

go.mod

@@ -1,16 +1,13 @@
-module github.com/mariolima/repocrawl
+module github.com/mariolima/repocrawler
 
-go 1.12
+go 1.13
 
 require (
 	github.com/bndr/gopencils v0.0.0-20161113114152-22e283ad7611
-	github.com/gammazero/workerpool v0.0.0-20190608213748-0ed5e40ec55e
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/gorilla/websocket v1.4.1
 	github.com/logrusorgru/aurora v0.0.0-20190803045625-94edacc10f9b
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
 	github.com/pkg/profile v1.3.0
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.4.2

pkg/bitbucket/service.go

@@ -3,7 +3,7 @@ package bitbucket
 import (
 	log "github.com/sirupsen/logrus"
 
-	"github.com/mariolima/repocrawl/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc
+	"github.com/mariolima/repocrawler/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc
 
 	"github.com/bndr/gopencils"
 

pkg/crawler/crawler.go

@@ -3,10 +3,10 @@ package crawler
 import (
 	"sync"
 
-	"github.com/mariolima/repocrawl/internal/entities"
-	"github.com/mariolima/repocrawl/pkg/bitbucket"
-	"github.com/mariolima/repocrawl/pkg/github"
-	_ "github.com/mariolima/repocrawl/pkg/gitlab" //TODO
+	"github.com/mariolima/repocrawler/internal/entities"
+	"github.com/mariolima/repocrawler/pkg/bitbucket"
+	"github.com/mariolima/repocrawler/pkg/github"
+	_ "github.com/mariolima/repocrawler/pkg/gitlab" //TODO
 
 	"bufio"
 	"regexp"

pkg/crawler/deepcrawl.go

@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mariolima/repocrawl/internal/entities"
+	"github.com/mariolima/repocrawler/internal/entities"
 	_ "gopkg.in/src-d/go-billy.v4/memfs" //???????????????????
 	"gopkg.in/src-d/go-git.v4"           //It's def heavy but gets the job done - any alternatives for commit crawling?
 	"gopkg.in/src-d/go-git.v4/plumbing/object"

pkg/crawler/graph.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/mariolima/repocrawl/internal/entities"
+	"github.com/mariolima/repocrawler/internal/entities"
 	_ "github.com/twmb/algoimpl/go/graph" // Graph library /w most of Algos and Structs
 )
 

pkg/crawler/match.go

@@ -1,7 +1,7 @@
 package crawler
 
 import (
-	"github.com/mariolima/repocrawl/internal/entities"
+	"github.com/mariolima/repocrawler/internal/entities"
 )
 
 // Match Secret found while crawling a repository

pkg/crawler/notification.go

@@ -2,7 +2,8 @@ package crawler
 
 import (
 	"fmt"
-	"github.com/mariolima/repocrawl/cmd/utils"
+
+	"github.com/mariolima/repocrawler/cmd/utils"
 	// log "github.com/sirupsen/logrus"
 )
 

pkg/crawler/task.go

@@ -1,7 +1,7 @@
 package crawler
 
 import (
-	"github.com/mariolima/repocrawl/internal/entities"
+	"github.com/mariolima/repocrawler/internal/entities"
 	log "github.com/sirupsen/logrus"
 )
 

pkg/github/service.go

@@ -8,7 +8,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/mariolima/repocrawl/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc
+	"github.com/mariolima/repocrawler/internal/entities" //structs common in GitHub/GitLab/BitBucket - RepoData/UserData etc
 )
 
 // GitHubCrawler Contains API_KEY and Client for API calls