feat: add indexes to assets and relations tables

.github/workflows/ci.yml

@@ -7,6 +7,7 @@ on:
     branches: develop
 
 jobs:
+
   golangci:
     name: lint
     runs-on: ubuntu-20.04
@@ -20,21 +21,39 @@ jobs:
         with:
           args: --timeout=5m
 
-  unit:
-    name: unit
+  integration:
+    name: integration
     runs-on: ubuntu-20.04
-    env:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: postgres
-      SQLITE3_DB: test.db
+
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
+      - name: Add database extensions
+        run: |
+          psql -h localhost -U postgres -tc "CREATE EXTENSION pg_trgm;"
+        env:
+          PGPASSWORD: postgres
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Run Tests
+        uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
-      - name: Setup PostgreSQL
-        uses: ikalnytskyi/action-setup-postgres@v4
-        id: postgres
       - run: |
-          go test -race -timeout 240s ./...
+          go test -v -cover ./...

README.md → docs/README.md

@@ -12,3 +12,7 @@ This repository offers services and repositories to interact with an asset datab
 ## Contributing
 
 If you have interest in contributing, please refer to the [contributing doc](CONTRIBUTING.md)
+
+## User Guide
+
+For important information on how to get started with this project, please refer to the [user guide](USER_GUIDE.md).

docs/USER_GUIDE.md

@@ -0,0 +1,49 @@
+# User Guide
+
+## Postgres
+
+This project relies upon several Postgres features that require elevated privileges.
+If you plan to use a _superuser_ when running migrations and working with the database,
+you can skip this section.
+
+It is best practice to _not_ use a superuser in most enterprise environments.
+For this reason, if you are configuring this in an environment
+where you do not possess these privileges,
+please work with your DBA to configure a new database in the following way:
+
+```sql
+-- Create a new database to store assets and relations.
+CREATE DATABASE IF NOT EXISTS assetdb;
+
+-- set the timezone to UTC
+ALTER DATABASE assetdb SET timezone TO 'UTC';
+```
+
+Reconnect to the `assetdb` database with the privileged user and run the following:
+
+```sql
+
+-- pg_trgm is required for to improve the performance of queries that use the LIKE operator.
+-- If you already have pg_trgm installed (extensions are global), you can skip this step
+-- If you don't know, you can run the following query to check:
+-- SELECT * FROM pg_extension where extname = 'pg_trgm';
+
+-- Install the pg_trgm extension on assetdb
+CREATE EXTENSION IF NOT EXISTS pg_trgm SCHEMA public;
+
+-- Create a user
+CREATE USER your_username WITH PASSWORD 'your_password';
+
+-- on Postgres 15, the public schema is not available except to superusers.
+GRANT USAGE ON schema public to your_username;
+
+-- Grant create permissions to your user on the public schema.
+GRANT CREATE ON schema public to your_username;
+
+-- Grant table modification permissions to your user on the public schema.
+GRANT ALL ON ALL TABLES IN SCHEMA public to your_username;
+
+```
+
+If you would like to keep the schema modifications separate from the collection user,
+you can create a separate user for this purpose.

migrations/postgres/005_assets_indexes.sql

@@ -0,0 +1,23 @@
+-- +migrate Up
+
+-- Index the `name` field of the `content` jsonb when type is `FQDN`
+-- Assumes the pg_trgm extension is created in the database
+CREATE INDEX idx_fqdn_content_name ON assets USING gin ((content->>'name') gin_trgm_ops) WHERE type = 'FQDN';
+
+-- Index assets.type
+CREATE INDEX idx_assets_type_hash ON assets USING hash (type);
+
+-- Index created_at
+CREATE INDEX idx_as_created_at ON assets (created_at);
+
+-- Index last_seen
+CREATE INDEX idx_as_last_seen ON assets (last_seen);
+
+
+-- +migrate Down
+
+-- drop all the indexes we just created
+DROP INDEX idx_fqdn_content_name;
+DROP INDEX idx_assets_type_hash;
+DROP INDEX idx_as_created_at;
+DROP INDEX idx_as_last_seen;

migrations/postgres/006_relations_indexes.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+
+CREATE INDEX idx_rel_created_at ON relations (created_at);
+CREATE INDEX idx_rel_last_seen ON assets (last_seen);
+
+-- +migrate Down
+
+DROP INDEX idx_rel_created_at;
+DROP INDEX idx_rel_last_seen;

migrations/postgres/example_test.go

@@ -2,6 +2,7 @@ package postgres_test
 
 import (
 	"fmt"
+	"log"
 	"os"
 
 	"github.com/owasp-amass/asset-db/migrations/postgres"
@@ -11,9 +12,23 @@ import (
 )
 
 func ExampleMigrations() {
-	user := os.Getenv("POSTGRES_USER")
-	password := os.Getenv("POSTGRES_PASSWORD")
-	dbname := os.Getenv("POSTGRES_DB")
+
+	user := "postgres"
+	if u, ok := os.LookupEnv("POSTGRES_USER"); ok {
+		user = u
+	}
+
+	password := "postgres"
+	if p, ok := os.LookupEnv("POSTGRES_PASSWORD"); ok {
+		password = p
+	}
+
+	dbname := "postgres"
+	if db, ok := os.LookupEnv("POSTGRES_DB"); ok {
+		dbname = db
+	}
+
+	log.Printf("DSN: %s", fmt.Sprintf("host=localhost port=5432 user=%s password=%s dbname=%s", user, password, dbname))
 
 	dsn := fmt.Sprintf("host=localhost port=5432 user=%s password=%s dbname=%s", user, password, dbname)
 	db, err := gorm.Open(pg.Open(dsn), &gorm.Config{})

migrations/sqlite3/004_assets_indexes.sql

@@ -0,0 +1,23 @@
+-- +migrate Up
+
+-- Index the `name` field of the `content` jsonb when type is `FQDN`
+-- Assumes the pg_trgm extension is created in the database
+CREATE INDEX idx_fqdn_content_name ON assets (content->>'name' COLLATE NOCASE) WHERE type = 'FQDN';
+
+-- Index assets.type
+CREATE INDEX idx_assets_type ON assets (type);
+
+-- Index created_at
+CREATE INDEX idx_as_created_at ON assets (created_at);
+
+-- Index last_seen
+CREATE INDEX idx_as_last_seen ON assets (last_seen);
+
+
+-- +migrate Down
+
+-- drop all the indexes we just created
+DROP INDEX idx_fqdn_content_name;
+DROP INDEX idx_assets_type;
+DROP INDEX idx_as_created_at;
+DROP INDEX idx_as_last_seen;

migrations/sqlite3/005_relations_indexes.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+
+CREATE INDEX idx_rel_created_at ON relations (created_at);
+CREATE INDEX idx_rel_last_seen ON relations (last_seen);
+
+-- +migrate Down
+
+DROP INDEX idx_rel_created_at;
+DROP INDEX idx_rel_last_seen;

migrations/sqlite3/example_test.go

@@ -11,7 +11,11 @@ import (
 )
 
 func ExampleMigrations() {
-	dsn := os.Getenv("SQLITE3_DB")
+
+	dsn := "test.db"
+	if v, ok := os.LookupEnv("SQLITE3_DB"); ok {
+		dsn = v
+	}
 
 	db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
 	if err != nil {

repository/sql_test.go

@@ -20,11 +20,6 @@ import (
 	"gorm.io/gorm"
 )
 
-var user = os.Getenv("POSTGRES_USER")
-var password = os.Getenv("POSTGRES_PASSWORD")
-var pgdbname = os.Getenv("POSTGRES_DB")
-var sqlitedbname = os.Getenv("SQLITE3_DB")
-
 var store *sqlRepository
 
 type testSetup struct {
@@ -112,6 +107,26 @@ func teardownPostgres(dsn string) {
 }
 
 func TestMain(m *testing.M) {
+	user := "postgres"
+	if u, ok := os.LookupEnv("POSTGRES_USER"); ok {
+		user = u
+	}
+
+	password := "postgres"
+	if p, ok := os.LookupEnv("POSTGRES_PASSWORD"); ok {
+		password = p
+	}
+
+	pgdbname := "postgres"
+	if pdb, ok := os.LookupEnv("POSTGRES_DB"); ok {
+		pgdbname = pdb
+	}
+
+	sqlitedbname := "test.db"
+	if sdb, ok := os.LookupEnv("SQLITE3_DB"); ok {
+		sqlitedbname = sdb
+	}
+
 	wrappers := []testSetup{
 		{
 			name:     Postgres,