completed the integration of asset-db

cmd/amass/db.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"net"
 	"os"
+	"time"
 
 	"github.com/caffix/netmap"
 	"github.com/caffix/stringset"
@@ -224,7 +225,8 @@ func showEventData(args *dbArgs, asninfo bool, db *netmap.Graph) {
 }
 
 func fillCache(cache *requests.ASNCache, db *netmap.Graph) error {
-	assets, err := db.DB.FindByType(oam.ASN)
+	start := time.Now().Add(-730 * time.Hour)
+	assets, err := db.DB.FindByType(oam.ASN, start)
 	if err != nil {
 		return err
 	}
@@ -235,12 +237,12 @@ func fillCache(cache *requests.ASNCache, db *netmap.Graph) error {
 			continue
 		}
 
-		desc := db.ReadASDescription(context.Background(), as.Number)
+		desc := db.ReadASDescription(context.Background(), as.Number, start)
 		if desc == "" {
 			continue
 		}
 
-		for _, prefix := range db.ReadASPrefixes(context.Background(), as.Number) {
+		for _, prefix := range db.ReadASPrefixes(context.Background(), as.Number, start) {
 			first, cidr, err := net.ParseCIDR(prefix)
 			if err != nil {
 				continue

cmd/amass/io.go

@@ -7,6 +7,7 @@ package main
 import (
 	"context"
 	"net"
+	"time"
 
 	"github.com/caffix/netmap"
 	"github.com/caffix/stringset"
@@ -20,16 +21,16 @@ import (
 // ExtractOutput is a convenience method for obtaining new discoveries made by the enumeration process.
 func ExtractOutput(ctx context.Context, g *netmap.Graph, e *enum.Enumeration, filter *stringset.Set, asinfo bool) []*requests.Output {
 	if e.Config.Passive {
-		return EventNames(ctx, g, e.Config.Domains(), filter)
+		return EventNames(ctx, g, e.Config.Domains(), e.Config.CollectionStartTime, filter)
 	}
-	return EventOutput(ctx, g, e.Config.Domains(), filter, asinfo, e.Sys.Cache())
+	return EventOutput(ctx, g, e.Config.Domains(), e.Config.CollectionStartTime, filter, asinfo, e.Sys.Cache())
 }
 
 type outLookup map[string]*requests.Output
 
 // EventOutput returns findings within the receiver Graph within the scope identified by the provided domain names.
 // The filter is updated by EventOutput.
-func EventOutput(ctx context.Context, g *netmap.Graph, domains []string, f *stringset.Set, asninfo bool, cache *requests.ASNCache) []*requests.Output {
+func EventOutput(ctx context.Context, g *netmap.Graph, domains []string, since time.Time, f *stringset.Set, asninfo bool, cache *requests.ASNCache) []*requests.Output {
 	var res []*requests.Output
 
 	if len(domains) == 0 {
@@ -46,7 +47,7 @@ func EventOutput(ctx context.Context, g *netmap.Graph, domains []string, f *stri
 		fqdns = append(fqdns, domain.FQDN{Name: d})
 	}
 
-	assets, err := g.DB.FindByScope(fqdns...)
+	assets, err := g.DB.FindByScope(fqdns, since)
 	if err != nil {
 		return res
 	}
@@ -73,7 +74,7 @@ func EventOutput(ctx context.Context, g *netmap.Graph, domains []string, f *stri
 		lookup[n] = o
 	}
 	// Build the lookup map used to create the final result set
-	if pairs, err := g.NamesToAddrs(ctx, names...); err == nil {
+	if pairs, err := g.NamesToAddrs(ctx, since, names...); err == nil {
 		for _, p := range pairs {
 			addr := p.Addr.Address.String()
 
@@ -137,7 +138,7 @@ func addInfrastructureInfo(lookup outLookup, filter *stringset.Set, cache *reque
 
 // EventNames returns findings within the receiver Graph within the scope identified by the provided domain names.
 // The filter is updated by EventNames.
-func EventNames(ctx context.Context, g *netmap.Graph, domains []string, f *stringset.Set) []*requests.Output {
+func EventNames(ctx context.Context, g *netmap.Graph, domains []string, since time.Time, f *stringset.Set) []*requests.Output {
 	var res []*requests.Output
 
 	if len(domains) == 0 {
@@ -154,7 +155,7 @@ func EventNames(ctx context.Context, g *netmap.Graph, domains []string, f *strin
 		fqdns = append(fqdns, domain.FQDN{Name: d})
 	}
 
-	assets, err := g.DB.FindByScope(fqdns...)
+	assets, err := g.DB.FindByScope(fqdns, since)
 	if err != nil {
 		return res
 	}

cmd/amass/main.go

@@ -32,6 +32,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/caffix/netmap"
 	"github.com/caffix/service"
@@ -204,7 +205,7 @@ func getEventOutput(ctx context.Context, domains []string, asninfo bool, db *net
 	filter := stringset.New()
 	defer filter.Close()
 
-	return EventOutput(ctx, db, domains, filter, asninfo, cache)
+	return EventOutput(ctx, db, domains, time.Time{}, filter, asninfo, cache)
 }
 
 func domainNameInScope(name string, scope []string) bool {

config/config.go

@@ -50,6 +50,9 @@ type Config struct {
 	// Logger for error messages
 	Log *log.Logger
 
+	// The date/time that discoveries must be active since to be included in the findings
+	CollectionStartTime time.Time
+
 	// The directory that stores the bolt db and other files created
 	Dir string `ini:"output_directory"`
 
@@ -146,10 +149,11 @@ type Config struct {
 // NewConfig returns a default configuration object.
 func NewConfig() *Config {
 	return &Config{
-		Rand:            rand.New(rand.NewSource(time.Now().UTC().UnixNano())),
-		Log:             log.New(io.Discard, "", 0),
-		Ports:           []int{80, 443},
-		MinForRecursive: 1,
+		Rand:                rand.New(rand.NewSource(time.Now().UTC().UnixNano())),
+		Log:                 log.New(io.Discard, "", 0),
+		CollectionStartTime: time.Now(),
+		Ports:               []int{80, 443},
+		MinForRecursive:     1,
 		// The following is enum-only, but intel will just ignore them anyway
 		FlipWords:      true,
 		FlipNumbers:    true,

doc/user_guide.md

@@ -49,8 +49,6 @@ The amass tool has several subcommands shown below for handling your Internet ex
 |------------|-------------|
 | intel | Collect open source intelligence for investigation of the target organization |
 | enum | Perform DNS enumeration and network mapping of systems exposed to the Internet |
-| viz | Generate visualizations of enumerations for exploratory analysis |
-| track | Compare results of enumerations against common target organizations |
 | db | Manage the graph databases storing the enumeration results |
 
 All subcommands have some default global arguments that can be seen below.
@@ -87,13 +85,11 @@ The intel subcommand can help you discover additional root domain names associat
 | -ipv6 | Show the IPv6 addresses for discovered names | amass intel -ipv6 -whois -d example.com |
 | -list | Print the names of all available data sources | amass intel -list |
 | -log | Path to the log file where errors will be written | amass intel -log amass.log -whois -d example.com |
-| -max-dns-queries | Maximum number of concurrent DNS queries | amass intel -max-dns-queries 200 -whois -d example.com |
 | -o | Path to the text output file | amass intel -o out.txt -whois -d example.com |
 | -org | Search string provided against AS description information | amass intel -org Facebook |
 | -p | Ports separated by commas (default: 80, 443) | amass intel -cidr 104.154.0.0/15 -p 443,8080 |
 | -r | IP addresses of preferred DNS resolvers (can be used multiple times) | amass intel -r 8.8.8.8,1.1.1.1 -whois -d example.com |
 | -rf | Path to a file providing preferred DNS resolvers | amass intel -rf data/resolvers.txt -whois -d example.com |
-| -src | Print data sources for the discovered names | amass intel -src -whois -d example.com |
 | -timeout | Number of minutes to execute the enumeration | amass intel -timeout 30 -d example.com |
 | -v | Output status / debug / troubleshooting info | amass intel -v -whois -d example.com |
 | -whois | All discovered domains are run through reverse whois | amass intel -whois -d example.com |
@@ -138,11 +134,9 @@ This subcommand will perform DNS enumeration and network mapping while populatin
 | -ip | Show the IP addresses for discovered names | amass enum -ip -d example.com |
 | -ipv4 | Show the IPv4 addresses for discovered names | amass enum -ipv4 -d example.com |
 | -ipv6 | Show the IPv6 addresses for discovered names | amass enum -ipv6 -d example.com |
-| -json | Path to the JSON output file | amass enum -json out.json -d example.com |
 | -list | Print the names of all available data sources | amass enum -list |
 | -log | Path to the log file where errors will be written | amass enum -log amass.log -d example.com |
 | -max-depth | Maximum number of subdomain labels for brute forcing | amass enum -brute -max-depth 3 -d example.com |
-| -max-dns-queries | Deprecated flag to be replaced by dns-qps in version 4.0 | amass enum -max-dns-queries 200 -d example.com |
 | -min-for-recursive | Subdomain labels seen before recursive brute forcing (Default: 1) | amass enum -brute -min-for-recursive 3 -d example.com |
 | -nf | Path to a file providing already known subdomain names (from other tools/sources) | amass enum -nf names.txt -d example.com |
 | -norecursive | Turn off recursive brute forcing | amass enum -brute -norecursive -d example.com |
@@ -154,7 +148,6 @@ This subcommand will perform DNS enumeration and network mapping while populatin
 | -rf | Path to a file providing untrusted DNS resolvers | amass enum -rf data/resolvers.txt -d example.com |
 | -rqps | Maximum number of DNS queries per second for each untrusted resolver | amass enum -rqps 10 -d example.com |
 | -scripts | Path to a directory containing ADS scripts | amass enum -scripts PATH -d example.com |
-| -src | Print data sources for the discovered names | amass enum -src -d example.com |
 | -timeout | Number of minutes to execute the enumeration | amass enum -timeout 30 -d example.com |
 | -tr | IP addresses of trusted DNS resolvers (can be used multiple times) | amass enum -tr 8.8.8.8,1.1.1.1 -d example.com |
 | -trf | Path to a file providing trusted DNS resolvers | amass enum -trf data/trusted.txt -d example.com |
@@ -163,40 +156,6 @@ This subcommand will perform DNS enumeration and network mapping while populatin
 | -w | Path to a different wordlist file for brute forcing | amass enum -brute -w wordlist.txt -d example.com |
 | -wm | "hashcat-style" wordlist masks for DNS brute forcing | amass enum -brute -wm ?l?l -d example.com |
 
-### The 'viz' Subcommand
-
-Create enlightening network graph visualizations that add structure to the information gathered. This subcommand only leverages the 'output_directory' and remote graph database settings from the configuration file.
-
-The files generated for visualization are created in the current working directory and named amass_TYPE
-
-Switches for outputting the DNS and infrastructure findings as a network graph:
-
-| Flag | Description | Example |
-|------|-------------|---------|
-| -d | Domain names separated by commas (can be used multiple times) | amass viz -d3 -d example.com |
-| -d3 | Output a D3.js v4 force simulation HTML file | amass viz -d3 -d example.com |
-| -df | Path to a file providing root domain names | amass viz -d3 -df domains.txt |
-| -dot | Generate the DOT output file | amass viz -dot -d example.com |
-| -enum | Identify an enumeration via an index from the db listing | amass viz -enum 1 -d3 -d example.com |
-| -gexf | Output to Graph Exchange XML Format (GEXF) | amass viz -gexf -d example.com |
-| -graphistry | Output Graphistry JSON | amass viz -graphistry -d example.com |
-| -i | Path to the Amass data operations JSON input file | amass viz -d3 -d example.com |
-| -maltego | Output a Maltego Graph Table CSV file | amass viz -maltego -d example.com |
-| -o | Path to a pre-existing directory that will hold output files | amass viz -d3 -o OUTPATH -d example.com |
-| -oA | Prefix used for naming all output files | amass viz -d3 -oA example -d example.com |
-
-### The 'track' Subcommand
-
-Shows differences between enumerations that included the same target(s) for monitoring a target's attack surface. This subcommand only leverages the 'output_directory' and remote graph database settings from the configuration file. Flags for performing Internet exposure monitoring across the enumerations in the graph database:
-
-| Flag | Description | Example |
-|------|-------------|---------|
-| -d | Domain names separated by commas (can be used multiple times) | amass track -d example.com |
-| -df | Path to a file providing root domain names | amass track -df domains.txt |
-| -history | Show the difference between all enumeration pairs | amass track -history |
-| -last | The number of recent enumerations to include in the tracking | amass track -last NUM |
-| -since | Exclude all enumerations before a specified date (format: 01/02 15:04:05 2006 MST) | amass track -since DATE |
-
 ### The 'db' Subcommand
 
 Performs viewing and manipulation of the graph database. This subcommand only leverages the 'output_directory' and remote graph database settings from the configuration file. Flags for interacting with the enumeration findings in the graph database include:
@@ -220,7 +179,7 @@ Performs viewing and manipulation of the graph database. This subcommand only le
 
 ## The Output Directory
 
-Amass has several files that it outputs during an enumeration (e.g. the log file). If you are not using a database server to store the network graph information, then Amass creates a file based graph database in the output directory. These files are used again during future enumerations, and when leveraging features like tracking and visualization.
+Amass has several files that it outputs during an enumeration (e.g. the log file). If you are not using a database server to store the network graph information, then Amass creates a file based graph database in the output directory. These files are used again during future enumerations.
 
 By default, the output directory is created in the operating system default root directory to use for user-specific configuration data and named *amass*. If this is not suitable for your needs, then the subcommands can be instructed to create the output directory in an alternative location using the **'-dir'** flag.
 
@@ -289,12 +248,6 @@ Note that these locations are based on the [output directory](#the-output-direct
 | url | URL in the form of "postgres://[username:password@]host[:port]/database-name?sslmode=disable" where Amass will connect to a PostgreSQL database |
 | options | Additional PostgreSQL database options |
 
-#### The `graphdbs.mysql` Section
-
-| Option | Description |
-|--------|-------------|
-| url | URL in the form of "[username:password@]tcp(host[:3306])/database-name?timeout=10s" where Amass will connect to a MySQL database |
-
 ### The `bruteforce` Section
 
 | Option | Description |
@@ -349,30 +302,4 @@ All Amass enumeration findings are stored in a graph database. This database is
 
 When a new enumeration begins and a graph database already exists with previous findings for the same target(s), the subdomain names from those previous enumerations are utilized in the new enumeration. New DNS queries are performed against those subdomain names to ensure that they are still legitimate and to obtain current IP addresses.
 
-The results from each enumeration is stored separately in the graph database, which allows the tracking subcommand to look for differences across the enumerations and provide the user with highlights about the target.
-
 There is nothing preventing multiple users from sharing a single (remote) graph database and leveraging each others findings across enumerations.
-
-### Cayley Graph Schema
-
-The GraphDB is storing all the domains that were found for a given enumeration. It stores the associated information such as the ip, ns_record, a_record, cname, ip block and associated source for each one of them as well. Each enumeration is identified by a uuid.
-
-Here is an example of graph for an enumeration run on example.com:
-
-![GraphDB](../images/example_graphDB.png)
-
-## Importing OWASP Amass Results into Maltego
-
-1. Convert the Amass data into a Maltego graph table CSV file:
-
-```bash
-amass viz -maltego
-```
-
-2. Import the CSV file with the correct Connectivity Table settings:
-
-![Connectivity table](../images/maltego_graph_import_wizard.png "Connectivity Table Settings")
-
-3. All the Amass findings will be brought into your Maltego Graph:
-
-![Maltego results](../images/maltego_results.png "Maltego Results")

enum/enum.go

@@ -7,6 +7,7 @@ package enum
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/caffix/netmap"
 	"github.com/caffix/pipeline"
@@ -16,6 +17,7 @@ import (
 	"github.com/owasp-amass/amass/v3/datasrcs"
 	"github.com/owasp-amass/amass/v3/requests"
 	"github.com/owasp-amass/amass/v3/systems"
+	oam "github.com/owasp-amass/open-asset-model"
 	"github.com/owasp-amass/open-asset-model/domain"
 )
 
@@ -239,7 +241,7 @@ func (e *Enumeration) submitKnownNames() {
 
 func (e *Enumeration) readNamesFromDatabase(db *netmap.Graph) {
 	for _, d := range e.Config.Domains() {
-		assets, err := db.DB.FindByScope(domain.FQDN{Name: d})
+		assets, err := db.DB.FindByScope([]oam.Asset{domain.FQDN{Name: d}}, time.Time{})
 		if err != nil {
 			continue
 		}

enum/input.go

@@ -6,14 +6,12 @@ package enum
 
 import (
 	"context"
-	"regexp"
 	"sync"
 	"time"
 
 	"github.com/caffix/pipeline"
 	"github.com/caffix/queue"
 	"github.com/caffix/service"
-	"github.com/owasp-amass/amass/v3/net/dns"
 	"github.com/owasp-amass/amass/v3/requests"
 	bf "github.com/tylertreat/BoomFilters"
 )
@@ -26,7 +24,6 @@ type enumSource struct {
 	enum     *Enumeration
 	queue    queue.Queue
 	filter   *bf.StableBloomFilter
-	subre    *regexp.Regexp
 	done     chan struct{}
 	doneOnce sync.Once
 	release  chan struct{}
@@ -42,7 +39,6 @@ func newEnumSource(p *pipeline.Pipeline, e *Enumeration) *enumSource {
 		enum:     e,
 		queue:    queue.NewQueue(),
 		filter:   bf.NewDefaultStableBloomFilter(1000000, 0.01),
-		subre:    dns.AnySubdomainRegex(),
 		done:     make(chan struct{}),
 		release:  make(chan struct{}, size),
 		max:      size,
@@ -91,11 +87,7 @@ func (r *enumSource) newName(req *requests.DNSRequest) {
 	}
 	// Clean up the newly discovered name and domain
 	requests.SanitizeDNSRequest(req)
-	// Check that the name is valid
-	if r.subre.FindString(req.Name) != req.Name {
-		r.releaseOutput(1)
-		return
-	}
+
 	if r.enum.Config.Blacklisted(req.Name) {
 		r.releaseOutput(1)
 		return