Skip to content

Commit

Permalink
Updating to the latest phishdetect refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@botherder
botherder committed Nov 26, 2020
1 parent bfb852d commit 65d1f85
Show file tree
Hide file tree
Showing 8 changed files with 81 additions and 76 deletions.
105 changes: 54 additions & 51 deletions analysis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ func checkIfBlocklisted(target string) (phishdetect.Warning, error) {
if phishdetect.SliceContains(toCheck, ioc.Hashed) {
log.Debug("Target ", target, " is blocklisted by indicator with hash ", ioc.Hashed)
return phishdetect.Warning{
Score: 100,
Name: "blocklisted",
Score: 100,
Name: "blocklisted",
Description: fmt.Sprintf("The domain was blocklisted in PhishDetect Node by indicator with hash %s", ioc.Hashed),
}, nil
}
Expand All @@ -55,13 +55,13 @@ func checkIfBlocklisted(target string) (phishdetect.Warning, error) {
// analyzeDomain is used to statically analyze a domain name.
func analyzeDomain(domain string) (*AnalysisResults, error) {
urlNormalized := phishdetect.NormalizeURL(domain)
urlFinal := urlNormalized
finalURL := urlNormalized

if !validateURL(urlNormalized) {
return nil, errors.New(ErrorMsgInvalidURL)
}

analysis := phishdetect.NewAnalysis(urlFinal, "")
analysis := phishdetect.NewAnalysis(finalURL, "")
loadBrands(*analysis)

err := analysis.AnalyzeDomain()
Expand All @@ -73,7 +73,7 @@ func analyzeDomain(domain string) (*AnalysisResults, error) {

results := AnalysisResults{
URL: domain,
URLFinal: urlFinal,
FinalURL: finalURL,
Safelisted: analysis.Safelisted,
Dangerous: analysis.Dangerous,
Score: analysis.Score,
Expand All @@ -93,13 +93,13 @@ func analyzeDomain(domain string) (*AnalysisResults, error) {
// analyzeURL is used to statically analyze a URL.
func analyzeURL(url string) (*AnalysisResults, error) {
urlNormalized := phishdetect.NormalizeURL(url)
urlFinal := urlNormalized
finalURL := urlNormalized

if !validateURL(urlNormalized) {
return nil, errors.New(ErrorMsgInvalidURL)
}

analysis := phishdetect.NewAnalysis(urlFinal, "")
analysis := phishdetect.NewAnalysis(finalURL, "")
loadBrands(*analysis)

err := analysis.AnalyzeURL()
Expand All @@ -111,15 +111,15 @@ func analyzeURL(url string) (*AnalysisResults, error) {

results := AnalysisResults{
URL: url,
URLFinal: urlFinal,
FinalURL: finalURL,
Safelisted: analysis.Safelisted,
Dangerous: analysis.Dangerous,
Score: analysis.Score,
Brand: brand,
Warnings: analysis.Warnings,
}

blocklisted, err := checkIfBlocklisted(urlFinal)
blocklisted, err := checkIfBlocklisted(finalURL)
if err == nil && blocklisted.Score > 0 {
results.Score += blocklisted.Score
results.Warnings = append(results.Warnings, blocklisted)
Expand All @@ -128,34 +128,28 @@ func analyzeURL(url string) (*AnalysisResults, error) {
return &results, nil
}

// analyzeLink is used to dynamically analyze a URL.
func analyzeLink(url string) (*AnalysisResults, error) {
urlNormalized := phishdetect.NormalizeURL(url)
urlFinal := urlNormalized

var screenshot string
// analyzeHTML is used to statically analyze an HTML page.
func analyzeHTML(url, htmlEncoded string) (*AnalysisResults, error) {
finalURL := url

if !validateURL(urlNormalized) {
if !validateURL(url) {
return nil, errors.New(ErrorMsgInvalidURL)
}

// Setting Docker API version.
os.Setenv("DOCKER_API_VERSION", apiVersion)
// Instantiate new browser and open the link.
browser := phishdetect.NewBrowser(urlNormalized, "", false, false, "")
err := browser.Run()
if err != nil {
log.Error("Failed to instantiate browser: ", err)
return nil, errors.New(ErrorMsgAnalysisFailed)
if htmlEncoded == "" {
return nil, errors.New(ErrorMsgInvalidHTML)
}
urlFinal = browser.FinalURL

screenshot = fmt.Sprintf("data:image/png;base64,%s", browser.ScreenshotData)
analysis := phishdetect.NewAnalysis(urlFinal, browser.HTML)
htmlData, err := base64.StdEncoding.DecodeString(htmlEncoded)
if err != nil {
return nil, errors.New(ErrorMsgInvalidHTML)
}
html := string(htmlData)

analysis := phishdetect.NewAnalysis(finalURL, html)
loadBrands(*analysis)

err = analysis.AnalyzeBrowserResults(browser.Requests)
err = analysis.AnalyzeHTML()
if err != nil {
log.Error("Failed to analyze HTML: ", err)
return nil, errors.New(ErrorMsgAnalysisFailed)
Expand All @@ -169,20 +163,17 @@ func analyzeLink(url string) (*AnalysisResults, error) {

results := AnalysisResults{
URL: url,
URLFinal: urlFinal,
FinalURL: finalURL,
Safelisted: analysis.Safelisted,
Dangerous: analysis.Dangerous,
Score: analysis.Score,
Brand: brand,
Screenshot: screenshot,
Warnings: analysis.Warnings,
Requests: browser.Requests,
Dialogs: browser.Dialogs,
Downloads: browser.Downloads,
HTML: browser.HTML,
HTML: html,
HTMLSHA256: phishdetect.GetSHA256Hash(html),
}

blocklisted, err := checkIfBlocklisted(urlFinal)
blocklisted, err := checkIfBlocklisted(finalURL)
if err == nil && blocklisted.Score > 0 {
results.Score += blocklisted.Score
results.Warnings = append(results.Warnings, blocklisted)
Expand All @@ -191,28 +182,34 @@ func analyzeLink(url string) (*AnalysisResults, error) {
return &results, nil
}

// analyzeHTML is used to statically analyze an HTML page.
func analyzeHTML(url, htmlEncoded string) (*AnalysisResults, error) {
urlFinal := url
// analyzeURLDynamic is used to dynamically analyze a URL.
func analyzeURLDynamic(url string) (*AnalysisResults, error) {
urlNormalized := phishdetect.NormalizeURL(url)
finalURL := urlNormalized

if !validateURL(url) {
return nil, errors.New(ErrorMsgInvalidURL)
}
var screenshot string

if htmlEncoded == "" {
return nil, errors.New(ErrorMsgInvalidHTML)
if !validateURL(urlNormalized) {
return nil, errors.New(ErrorMsgInvalidURL)
}

htmlData, err := base64.StdEncoding.DecodeString(htmlEncoded)
// Setting Docker API version.
os.Setenv("DOCKER_API_VERSION", apiVersion)
// Instantiate new browser and open the link.
browser := phishdetect.NewBrowser(urlNormalized, "", false, false, "")
err := browser.Run()
if err != nil {
return nil, errors.New(ErrorMsgInvalidHTML)
log.Error("Failed to instantiate browser: ", err)
return nil, errors.New(ErrorMsgAnalysisFailed)
}
html := string(htmlData)
finalURL = browser.FinalURL

screenshot = fmt.Sprintf("data:image/png;base64,%s", browser.ScreenshotData)
analysis := phishdetect.NewAnalysis(finalURL, browser.HTML)

analysis := phishdetect.NewAnalysis(urlFinal, html)
loadBrands(*analysis)

err = analysis.AnalyzeHTML()
err = analysis.AnalyzeBrowserResults(browser.Resources)
if err != nil {
log.Error("Failed to analyze HTML: ", err)
return nil, errors.New(ErrorMsgAnalysisFailed)
Expand All @@ -226,16 +223,22 @@ func analyzeHTML(url, htmlEncoded string) (*AnalysisResults, error) {

results := AnalysisResults{
URL: url,
URLFinal: urlFinal,
FinalURL: finalURL,
Safelisted: analysis.Safelisted,
Dangerous: analysis.Dangerous,
Score: analysis.Score,
Brand: brand,
Screenshot: screenshot,
Warnings: analysis.Warnings,
HTML: html,
Visits: browser.Visits,
Resources: browser.Resources,
Dialogs: browser.Dialogs,
Downloads: browser.Downloads,
HTML: browser.HTML,
HTMLSHA256: browser.HTMLSHA256,
}

blocklisted, err := checkIfBlocklisted(urlFinal)
blocklisted, err := checkIfBlocklisted(finalURL)
if err == nil && blocklisted.Score > 0 {
results.Score += blocklisted.Score
results.Warnings = append(results.Warnings, blocklisted)
Expand Down
8 changes: 4 additions & 4 deletions api_analysis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func apiAnalyzeURL(w http.ResponseWriter, r *http.Request) {
responseWithJSON(w, results)
}

func apiAnalyzeLink(w http.ResponseWriter, r *http.Request) {
func apiAnalyzeHTML(w http.ResponseWriter, r *http.Request) {
if !enableAnalysis {
errorWithJSON(w, ErrorMsgAnalysisDisabled, http.StatusForbidden, nil)
return
Expand All @@ -87,7 +87,7 @@ func apiAnalyzeLink(w http.ResponseWriter, r *http.Request) {
return
}

results, err := analyzeLink(req.URL)
results, err := analyzeHTML(req.URL, req.HTML)
if err != nil {
errorWithJSON(w, err.Error(), http.StatusInternalServerError, err)
return
Expand All @@ -96,7 +96,7 @@ func apiAnalyzeLink(w http.ResponseWriter, r *http.Request) {
responseWithJSON(w, results)
}

func apiAnalyzeHTML(w http.ResponseWriter, r *http.Request) {
func apiAnalyzeLink(w http.ResponseWriter, r *http.Request) {
if !enableAnalysis {
errorWithJSON(w, ErrorMsgAnalysisDisabled, http.StatusForbidden, nil)
return
Expand All @@ -110,7 +110,7 @@ func apiAnalyzeHTML(w http.ResponseWriter, r *http.Request) {
return
}

results, err := analyzeHTML(req.URL, req.HTML)
results, err := analyzeURLDynamic(req.URL)
if err != nil {
errorWithJSON(w, err.Error(), http.StatusInternalServerError, err)
return
Expand Down
6 changes: 4 additions & 2 deletions database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,19 @@ type Review struct {

type AnalysisResults struct {
URL string `json:"url"`
URLFinal string `json:"url_final" bson:"url_final`
FinalURL string `json:"final_url" bson:"final_url`
Safelisted bool `json:"safelisted"`
Dangerous bool `json:"dangerous"`
Brand string `json:"brand"`
Score int `json:"score"`
Screenshot string `json:"screenshot"`
Visits []phishdetect.Visit `json:"visits"`
Resources []phishdetect.Resource `json:"resources"`
Warnings []phishdetect.Warning `json:"warnings"`
Requests []phishdetect.Request `json:"requests"`
Dialogs []phishdetect.Dialog `json:"dialogs"`
Downloads []phishdetect.Download `json:"downloads"`
HTML string `json:"html"`
HTMLSHA256 string `json:"html_sha256"`
AlertUUID string `json:"uuid"`
}

Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ require (
github.com/manifoldco/promptui v0.7.0
github.com/mattn/go-colorable v0.1.8
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d
github.com/phishdetect/phishdetect v1.16.0
github.com/phishdetect/phishdetect v1.17.0
github.com/rogpeppe/go-internal v1.6.2 // indirect
github.com/sirupsen/logrus v1.7.0
github.com/spf13/pflag v1.0.5
Expand All @@ -34,7 +34,7 @@ require (
golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect
golang.org/x/sys v0.0.0-20201126144705-a4b67b81d3d2 // indirect
golang.org/x/text v0.3.4 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,8 @@ github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FW
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/phishdetect/phishdetect v1.16.0 h1:AJ2gL543crFMpl6Q9RXxonglswLGcHDHg3lN01Ak9SU=
github.com/phishdetect/phishdetect v1.16.0/go.mod h1:78rW9trV0rUlWBCkMnzTX5gYf8MjFUNJ3rMZFFz8bu8=
github.com/phishdetect/phishdetect v1.17.0 h1:Nw+gu0T6YgvLMcwnJ8yQ8NYiKAPoT2RQukSI7ciu5/8=
github.com/phishdetect/phishdetect v1.17.0/go.mod h1:78rW9trV0rUlWBCkMnzTX5gYf8MjFUNJ3rMZFFz8bu8=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -265,8 +265,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200722175500-76b94024e4b6 h1:X9xIZ1YU8bLZA3l6gqDUHSFiD0GFI9S548h6C8nDtOY=
golang.org/x/sys v0.0.0-20200722175500-76b94024e4b6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201126144705-a4b67b81d3d2 h1:WFCmm2Hi9I2gYf1kv7LQ8ajKA5x9heC2v9xuUKwvf68=
golang.org/x/sys v0.0.0-20201126144705-a4b67b81d3d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
16 changes: 8 additions & 8 deletions gui_link.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,8 @@ func guiLinkAnalyze(w http.ResponseWriter, r *http.Request) {
htmlEncoded := r.PostFormValue("html")
screenshot := r.PostFormValue("screenshot")

// For the moment, urlFinal will be the original URL.
urlFinal := url
// For the moment, finalURL will be the original URL.
finalURL := url

var alertType string
var results *AnalysisResults
Expand All @@ -101,13 +101,13 @@ func guiLinkAnalyze(w http.ResponseWriter, r *http.Request) {
// If there is no specified HTML string, it means we need to open the link.
if htmlEncoded == "" {
alertType = "analysis_link"
results, err = analyzeLink(url)
results, err = analyzeURLDynamic(url)
if err != nil {
errorMessage(w, err.Error())
return
}

urlFinal = results.URLFinal
finalURL = results.FinalURL
screenshot = results.Screenshot
} else {
alertType = "analysis_html"
Expand All @@ -123,12 +123,12 @@ func guiLinkAnalyze(w http.ResponseWriter, r *http.Request) {
// If the site is safelisted, or the final score is low, we offer
// to continue to the original link.
if (results.Safelisted || results.Score < 30) && !results.Dangerous {
urlFinalEncoded := base64.StdEncoding.EncodeToString([]byte(urlFinal))
finalURLEncoded := base64.StdEncoding.EncodeToString([]byte(finalURL))
tpl, err := tmplSet.FromCache("continue.html")
err = tpl.ExecuteWriter(pongo.Context{
"url": url,
"urlFinalEncoded": urlFinalEncoded,
"urlFinal": urlFinal,
"finalURLEncoded": finalURLEncoded,
"finalURL": finalURL,
"sha1": urlSHA1,
"brand": results.Brand,
"safelisted": results.Safelisted,
Expand Down Expand Up @@ -169,7 +169,7 @@ func guiLinkAnalyze(w http.ResponseWriter, r *http.Request) {
tpl, err := tmplSet.FromCache("warning.html")
err = tpl.ExecuteWriter(pongo.Context{
"url": url,
"urlFinal": urlFinal,
"finalURL": finalURL,
"sha1": urlSHA1,
"warnings": results.Warnings,
"brand": results.Brand,
Expand Down
6 changes: 3 additions & 3 deletions templates/continue.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ <h1 class="font-normal mt-4"><b>PhishDetect</b> OK</h1>

<div class="border-l-8 border-blue-lighter mb-8 bg-blue-lightest text-blue-darker p-6 rounded-lg leading-normal">
<div style="word-break: break-all;">We analyzed the link: <span class="font-mono bg-blue-lighter text-blue-darkest">{{url}}</span>. No suspicious elements have been found in this page.</div>
<div class="mt-4"><b>Please notice:</b> this does not guarantee that the page is completely safe</b> (for example, it might evade our detection or identify our service and redirect instead to a legitimate page), please always be cautious. <b>If you believe this assessment is wrong, and suspect this page to be suspicious, please <a class="text-blue hover:no-underline hover:text-blue-dark" href="/report/{{urlFinalEncoded}}?key={{key}}">report it to us!</a></b></div>
<div class="mt-4"><b>Please notice:</b> this does not guarantee that the page is completely safe</b> (for example, it might evade our detection or identify our service and redirect instead to a legitimate page), please always be cautious. <b>If you believe this assessment is wrong, and suspect this page to be suspicious, please <a class="text-blue hover:no-underline hover:text-blue-dark" href="/report/{{finalURLEncoded}}?key={{key}}">report it to us!</a></b></div>
</div>

{% if urlFinal and url != urlFinal %}
<div class="mb-8">The original URL <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{url}}</span> redirected to the final URL <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{urlFinal}}</span></div>
{% if finalURL and url != finalURL %}
<div class="mb-8">The original URL <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{url}}</span> redirected to the final URL <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{finalURL}}</span></div>
{% endif %}

<div class="mb-4">Following is a screenshot preview of the website.</div>
Expand Down
4 changes: 2 additions & 2 deletions templates/warning.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ <h4 class="mt-4">Warnings</h4>
</div>
{% endif %}

{% if urlFinal and url != urlFinal %}
<div class="mb-8">The original link <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{url}}</span> redirected to the final URL <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{urlFinal}}</span></div>
{% if finalURL and url != finalURL %}
<div class="mb-8">The original link <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{url}}</span> redirected to the final URL <span class="font-mono bg-grey-lighter" style="word-break: break-all;">{{finalURL}}</span></div>
{% endif %}

<div class="mb-4">Following is a screenshot preview of the website.{% if brand and not safelisted %} Bear in mind, <b>this website is not a legitimate {{brand|capfirst}} site.</b>{% endif %}</div>
Expand Down

0 comments on commit 65d1f85

Please sign in to comment.