diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 306df32..1c88bef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: branches: develop jobs: + golangci: name: lint runs-on: ubuntu-20.04 @@ -20,21 +21,39 @@ jobs: with: args: --timeout=5m - unit: - name: unit + integration: + name: integration runs-on: ubuntu-20.04 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - SQLITE3_DB: test.db + + services: + postgres: + image: postgres:latest + env: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - name: Add database extensions + run: | + psql -h localhost -U postgres -tc "CREATE EXTENSION pg_trgm;" + env: + PGPASSWORD: postgres + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Run Tests + uses: actions/setup-go@v4 with: go-version-file: 'go.mod' - - name: Setup PostgreSQL - uses: ikalnytskyi/action-setup-postgres@v4 - id: postgres - run: | - go test -race -timeout 240s ./... + go test -v -cover ./... diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/README.md b/docs/README.md similarity index 88% rename from README.md rename to docs/README.md index 39331c1..76ace6d 100644 --- a/README.md +++ b/docs/README.md @@ -12,3 +12,7 @@ This repository offers services and repositories to interact with an asset datab ## Contributing If you have interest in contributing, please refer to the [contributing doc](CONTRIBUTING.md) + +## User Guide + +For important information on how to get started with this project, please refer to the [user guide](USER_GUIDE.md). \ No newline at end of file diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md new file mode 100644 index 0000000..e2b5d9b --- /dev/null +++ b/docs/USER_GUIDE.md @@ -0,0 +1,49 @@ +# User Guide + +## Postgres + +This project relies upon several Postgres features that require elevated privileges. +If you plan to use a _superuser_ when running migrations and working with the database, +you can skip this section. + +It is best practice to _not_ use a superuser in most enterprise environments. +For this reason, if you are configuring this in an environment +where you do not possess these privileges, +please work with your DBA to configure a new database in the following way: + +```sql +-- Create a new database to store assets and relations. +CREATE DATABASE IF NOT EXISTS assetdb; + +-- set the timezone to UTC +ALTER DATABASE assetdb SET timezone TO 'UTC'; +``` + +Reconnect to the `assetdb` database with the privileged user and run the following: + +```sql + +-- pg_trgm is required for to improve the performance of queries that use the LIKE operator. +-- If you already have pg_trgm installed (extensions are global), you can skip this step +-- If you don't know, you can run the following query to check: +-- SELECT * FROM pg_extension where extname = 'pg_trgm'; + +-- Install the pg_trgm extension on assetdb +CREATE EXTENSION IF NOT EXISTS pg_trgm SCHEMA public; + +-- Create a user +CREATE USER your_username WITH PASSWORD 'your_password'; + +-- on Postgres 15, the public schema is not available except to superusers. +GRANT USAGE ON schema public to your_username; + +-- Grant create permissions to your user on the public schema. +GRANT CREATE ON schema public to your_username; + +-- Grant table modification permissions to your user on the public schema. +GRANT ALL ON ALL TABLES IN SCHEMA public to your_username; + +``` + +If you would like to keep the schema modifications separate from the collection user, +you can create a separate user for this purpose. diff --git a/migrations/postgres/005_assets_indexes.sql b/migrations/postgres/005_assets_indexes.sql new file mode 100644 index 0000000..4bee932 --- /dev/null +++ b/migrations/postgres/005_assets_indexes.sql @@ -0,0 +1,23 @@ +-- +migrate Up + +-- Index the `name` field of the `content` jsonb when type is `FQDN` +-- Assumes the pg_trgm extension is created in the database +CREATE INDEX idx_fqdn_content_name ON assets USING gin ((content->>'name') gin_trgm_ops) WHERE type = 'FQDN'; + +-- Index assets.type +CREATE INDEX idx_assets_type_hash ON assets USING hash (type); + +-- Index created_at +CREATE INDEX idx_as_created_at ON assets (created_at); + +-- Index last_seen +CREATE INDEX idx_as_last_seen ON assets (last_seen); + + +-- +migrate Down + +-- drop all the indexes we just created +DROP INDEX idx_fqdn_content_name; +DROP INDEX idx_assets_type_hash; +DROP INDEX idx_as_created_at; +DROP INDEX idx_as_last_seen; \ No newline at end of file diff --git a/migrations/postgres/006_relations_indexes.sql b/migrations/postgres/006_relations_indexes.sql new file mode 100644 index 0000000..fc70ad7 --- /dev/null +++ b/migrations/postgres/006_relations_indexes.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +CREATE INDEX idx_rel_created_at ON relations (created_at); +CREATE INDEX idx_rel_last_seen ON assets (last_seen); + +-- +migrate Down + +DROP INDEX idx_rel_created_at; +DROP INDEX idx_rel_last_seen; \ No newline at end of file diff --git a/migrations/postgres/example_test.go b/migrations/postgres/example_test.go index 0805bbc..1a821fe 100644 --- a/migrations/postgres/example_test.go +++ b/migrations/postgres/example_test.go @@ -2,6 +2,7 @@ package postgres_test import ( "fmt" + "log" "os" "github.com/owasp-amass/asset-db/migrations/postgres" @@ -11,9 +12,23 @@ import ( ) func ExampleMigrations() { - user := os.Getenv("POSTGRES_USER") - password := os.Getenv("POSTGRES_PASSWORD") - dbname := os.Getenv("POSTGRES_DB") + + user := "postgres" + if u, ok := os.LookupEnv("POSTGRES_USER"); ok { + user = u + } + + password := "postgres" + if p, ok := os.LookupEnv("POSTGRES_PASSWORD"); ok { + password = p + } + + dbname := "postgres" + if db, ok := os.LookupEnv("POSTGRES_DB"); ok { + dbname = db + } + + log.Printf("DSN: %s", fmt.Sprintf("host=localhost port=5432 user=%s password=%s dbname=%s", user, password, dbname)) dsn := fmt.Sprintf("host=localhost port=5432 user=%s password=%s dbname=%s", user, password, dbname) db, err := gorm.Open(pg.Open(dsn), &gorm.Config{}) diff --git a/migrations/sqlite3/004_assets_indexes.sql b/migrations/sqlite3/004_assets_indexes.sql new file mode 100644 index 0000000..5266384 --- /dev/null +++ b/migrations/sqlite3/004_assets_indexes.sql @@ -0,0 +1,23 @@ +-- +migrate Up + +-- Index the `name` field of the `content` jsonb when type is `FQDN` +-- Assumes the pg_trgm extension is created in the database +CREATE INDEX idx_fqdn_content_name ON assets (content->>'name' COLLATE NOCASE) WHERE type = 'FQDN'; + +-- Index assets.type +CREATE INDEX idx_assets_type ON assets (type); + +-- Index created_at +CREATE INDEX idx_as_created_at ON assets (created_at); + +-- Index last_seen +CREATE INDEX idx_as_last_seen ON assets (last_seen); + + +-- +migrate Down + +-- drop all the indexes we just created +DROP INDEX idx_fqdn_content_name; +DROP INDEX idx_assets_type; +DROP INDEX idx_as_created_at; +DROP INDEX idx_as_last_seen; \ No newline at end of file diff --git a/migrations/sqlite3/005_relations_indexes.sql b/migrations/sqlite3/005_relations_indexes.sql new file mode 100644 index 0000000..480e69a --- /dev/null +++ b/migrations/sqlite3/005_relations_indexes.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +CREATE INDEX idx_rel_created_at ON relations (created_at); +CREATE INDEX idx_rel_last_seen ON relations (last_seen); + +-- +migrate Down + +DROP INDEX idx_rel_created_at; +DROP INDEX idx_rel_last_seen; \ No newline at end of file diff --git a/migrations/sqlite3/example_test.go b/migrations/sqlite3/example_test.go index 7914c9e..bad1ac7 100644 --- a/migrations/sqlite3/example_test.go +++ b/migrations/sqlite3/example_test.go @@ -11,7 +11,11 @@ import ( ) func ExampleMigrations() { - dsn := os.Getenv("SQLITE3_DB") + + dsn := "test.db" + if v, ok := os.LookupEnv("SQLITE3_DB"); ok { + dsn = v + } db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) if err != nil { diff --git a/repository/sql_test.go b/repository/sql_test.go index 0ac265b..a0f570e 100644 --- a/repository/sql_test.go +++ b/repository/sql_test.go @@ -20,11 +20,6 @@ import ( "gorm.io/gorm" ) -var user = os.Getenv("POSTGRES_USER") -var password = os.Getenv("POSTGRES_PASSWORD") -var pgdbname = os.Getenv("POSTGRES_DB") -var sqlitedbname = os.Getenv("SQLITE3_DB") - var store *sqlRepository type testSetup struct { @@ -112,6 +107,26 @@ func teardownPostgres(dsn string) { } func TestMain(m *testing.M) { + user := "postgres" + if u, ok := os.LookupEnv("POSTGRES_USER"); ok { + user = u + } + + password := "postgres" + if p, ok := os.LookupEnv("POSTGRES_PASSWORD"); ok { + password = p + } + + pgdbname := "postgres" + if pdb, ok := os.LookupEnv("POSTGRES_DB"); ok { + pgdbname = pdb + } + + sqlitedbname := "test.db" + if sdb, ok := os.LookupEnv("SQLITE3_DB"); ok { + sqlitedbname = sdb + } + wrappers := []testSetup{ { name: Postgres,