From a1e85ffa690c47bbb322fe8b5655b1ecadd156e0 Mon Sep 17 00:00:00 2001 From: Ryan Morton Date: Thu, 3 Aug 2023 16:26:26 -0400 Subject: [PATCH 1/5] feat: add indexes to assets and relations tables - Index on `$.content.Name` for FQDN assets - index on assets.type - index on assets.last_seen - index on relations.last_seen Closes #19 --- migrations/postgres/005_assets_indexes.sql | 23 +++++++++++++++++++ migrations/postgres/006_relations_indexes.sql | 9 ++++++++ migrations/sqlite3/004_assets_indexes.sql | 23 +++++++++++++++++++ migrations/sqlite3/005_relations_indexes.sql | 9 ++++++++ 4 files changed, 64 insertions(+) create mode 100644 migrations/postgres/005_assets_indexes.sql create mode 100644 migrations/postgres/006_relations_indexes.sql create mode 100644 migrations/sqlite3/004_assets_indexes.sql create mode 100644 migrations/sqlite3/005_relations_indexes.sql diff --git a/migrations/postgres/005_assets_indexes.sql b/migrations/postgres/005_assets_indexes.sql new file mode 100644 index 0000000..4bee932 --- /dev/null +++ b/migrations/postgres/005_assets_indexes.sql @@ -0,0 +1,23 @@ +-- +migrate Up + +-- Index the `name` field of the `content` jsonb when type is `FQDN` +-- Assumes the pg_trgm extension is created in the database +CREATE INDEX idx_fqdn_content_name ON assets USING gin ((content->>'name') gin_trgm_ops) WHERE type = 'FQDN'; + +-- Index assets.type +CREATE INDEX idx_assets_type_hash ON assets USING hash (type); + +-- Index created_at +CREATE INDEX idx_as_created_at ON assets (created_at); + +-- Index last_seen +CREATE INDEX idx_as_last_seen ON assets (last_seen); + + +-- +migrate Down + +-- drop all the indexes we just created +DROP INDEX idx_fqdn_content_name; +DROP INDEX idx_assets_type_hash; +DROP INDEX idx_as_created_at; +DROP INDEX idx_as_last_seen; \ No newline at end of file diff --git a/migrations/postgres/006_relations_indexes.sql b/migrations/postgres/006_relations_indexes.sql new file mode 100644 index 0000000..fc70ad7 --- /dev/null +++ b/migrations/postgres/006_relations_indexes.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +CREATE INDEX idx_rel_created_at ON relations (created_at); +CREATE INDEX idx_rel_last_seen ON assets (last_seen); + +-- +migrate Down + +DROP INDEX idx_rel_created_at; +DROP INDEX idx_rel_last_seen; \ No newline at end of file diff --git a/migrations/sqlite3/004_assets_indexes.sql b/migrations/sqlite3/004_assets_indexes.sql new file mode 100644 index 0000000..5266384 --- /dev/null +++ b/migrations/sqlite3/004_assets_indexes.sql @@ -0,0 +1,23 @@ +-- +migrate Up + +-- Index the `name` field of the `content` jsonb when type is `FQDN` +-- Assumes the pg_trgm extension is created in the database +CREATE INDEX idx_fqdn_content_name ON assets (content->>'name' COLLATE NOCASE) WHERE type = 'FQDN'; + +-- Index assets.type +CREATE INDEX idx_assets_type ON assets (type); + +-- Index created_at +CREATE INDEX idx_as_created_at ON assets (created_at); + +-- Index last_seen +CREATE INDEX idx_as_last_seen ON assets (last_seen); + + +-- +migrate Down + +-- drop all the indexes we just created +DROP INDEX idx_fqdn_content_name; +DROP INDEX idx_assets_type; +DROP INDEX idx_as_created_at; +DROP INDEX idx_as_last_seen; \ No newline at end of file diff --git a/migrations/sqlite3/005_relations_indexes.sql b/migrations/sqlite3/005_relations_indexes.sql new file mode 100644 index 0000000..480e69a --- /dev/null +++ b/migrations/sqlite3/005_relations_indexes.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +CREATE INDEX idx_rel_created_at ON relations (created_at); +CREATE INDEX idx_rel_last_seen ON relations (last_seen); + +-- +migrate Down + +DROP INDEX idx_rel_created_at; +DROP INDEX idx_rel_last_seen; \ No newline at end of file From 6a91d98bba14dc22a2a82c10f37ade6b91a5c914 Mon Sep 17 00:00:00 2001 From: Ryan Morton Date: Thu, 3 Aug 2023 16:32:08 -0400 Subject: [PATCH 2/5] chore: move README/CONTRIBUTING to docs/ folder --- CONTRIBUTING.md => docs/CONTRIBUTING.md | 0 README.md => docs/README.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename CONTRIBUTING.md => docs/CONTRIBUTING.md (100%) rename README.md => docs/README.md (100%) diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/README.md b/docs/README.md similarity index 100% rename from README.md rename to docs/README.md From b6a193bf052fe23a87ec685f8750874f5c70d478 Mon Sep 17 00:00:00 2001 From: Ryan Morton Date: Thu, 3 Aug 2023 16:32:32 -0400 Subject: [PATCH 3/5] docs: add user guide includes important details related to postgres --- docs/README.md | 4 ++++ docs/USER_GUIDE.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 docs/USER_GUIDE.md diff --git a/docs/README.md b/docs/README.md index 39331c1..76ace6d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,3 +12,7 @@ This repository offers services and repositories to interact with an asset datab ## Contributing If you have interest in contributing, please refer to the [contributing doc](CONTRIBUTING.md) + +## User Guide + +For important information on how to get started with this project, please refer to the [user guide](USER_GUIDE.md). \ No newline at end of file diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md new file mode 100644 index 0000000..e2b5d9b --- /dev/null +++ b/docs/USER_GUIDE.md @@ -0,0 +1,49 @@ +# User Guide + +## Postgres + +This project relies upon several Postgres features that require elevated privileges. +If you plan to use a _superuser_ when running migrations and working with the database, +you can skip this section. + +It is best practice to _not_ use a superuser in most enterprise environments. +For this reason, if you are configuring this in an environment +where you do not possess these privileges, +please work with your DBA to configure a new database in the following way: + +```sql +-- Create a new database to store assets and relations. +CREATE DATABASE IF NOT EXISTS assetdb; + +-- set the timezone to UTC +ALTER DATABASE assetdb SET timezone TO 'UTC'; +``` + +Reconnect to the `assetdb` database with the privileged user and run the following: + +```sql + +-- pg_trgm is required for to improve the performance of queries that use the LIKE operator. +-- If you already have pg_trgm installed (extensions are global), you can skip this step +-- If you don't know, you can run the following query to check: +-- SELECT * FROM pg_extension where extname = 'pg_trgm'; + +-- Install the pg_trgm extension on assetdb +CREATE EXTENSION IF NOT EXISTS pg_trgm SCHEMA public; + +-- Create a user +CREATE USER your_username WITH PASSWORD 'your_password'; + +-- on Postgres 15, the public schema is not available except to superusers. +GRANT USAGE ON schema public to your_username; + +-- Grant create permissions to your user on the public schema. +GRANT CREATE ON schema public to your_username; + +-- Grant table modification permissions to your user on the public schema. +GRANT ALL ON ALL TABLES IN SCHEMA public to your_username; + +``` + +If you would like to keep the schema modifications separate from the collection user, +you can create a separate user for this purpose. From 336c4d304555f874679c548e0f2e0f35c4b1deb2 Mon Sep 17 00:00:00 2001 From: Ryan Morton Date: Tue, 8 Aug 2023 10:06:21 -0400 Subject: [PATCH 4/5] ci: restructure workflow --- .github/workflows/ci.yml | 47 ++++++++++++++++++++--------- migrations/postgres/example_test.go | 21 +++++++++++-- migrations/sqlite3/example_test.go | 6 +++- repository/sql_test.go | 25 ++++++++++++--- 4 files changed, 76 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 306df32..1a513cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,11 +7,12 @@ on: branches: develop jobs: + golangci: name: lint runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - uses: actions/setup-go@v4 with: go-version-file: 'go.mod' @@ -20,21 +21,39 @@ jobs: with: args: --timeout=5m - unit: - name: unit + integration: + name: integration runs-on: ubuntu-20.04 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - SQLITE3_DB: test.db + + services: + postgres: + image: postgres:latest + env: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - name: Add database extensions + run: | + psql -h localhost -U postgres -tc "CREATE EXTENSION pg_trgm;" + env: + PGPASSWORD: postgres + + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Run Tests + uses: actions/setup-go@v4 with: go-version-file: 'go.mod' - - name: Setup PostgreSQL - uses: ikalnytskyi/action-setup-postgres@v4 - id: postgres - run: | - go test -race -timeout 240s ./... + go test -v ./... diff --git a/migrations/postgres/example_test.go b/migrations/postgres/example_test.go index 0805bbc..1a821fe 100644 --- a/migrations/postgres/example_test.go +++ b/migrations/postgres/example_test.go @@ -2,6 +2,7 @@ package postgres_test import ( "fmt" + "log" "os" "github.com/owasp-amass/asset-db/migrations/postgres" @@ -11,9 +12,23 @@ import ( ) func ExampleMigrations() { - user := os.Getenv("POSTGRES_USER") - password := os.Getenv("POSTGRES_PASSWORD") - dbname := os.Getenv("POSTGRES_DB") + + user := "postgres" + if u, ok := os.LookupEnv("POSTGRES_USER"); ok { + user = u + } + + password := "postgres" + if p, ok := os.LookupEnv("POSTGRES_PASSWORD"); ok { + password = p + } + + dbname := "postgres" + if db, ok := os.LookupEnv("POSTGRES_DB"); ok { + dbname = db + } + + log.Printf("DSN: %s", fmt.Sprintf("host=localhost port=5432 user=%s password=%s dbname=%s", user, password, dbname)) dsn := fmt.Sprintf("host=localhost port=5432 user=%s password=%s dbname=%s", user, password, dbname) db, err := gorm.Open(pg.Open(dsn), &gorm.Config{}) diff --git a/migrations/sqlite3/example_test.go b/migrations/sqlite3/example_test.go index 7914c9e..bad1ac7 100644 --- a/migrations/sqlite3/example_test.go +++ b/migrations/sqlite3/example_test.go @@ -11,7 +11,11 @@ import ( ) func ExampleMigrations() { - dsn := os.Getenv("SQLITE3_DB") + + dsn := "test.db" + if v, ok := os.LookupEnv("SQLITE3_DB"); ok { + dsn = v + } db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) if err != nil { diff --git a/repository/sql_test.go b/repository/sql_test.go index 0ac265b..a0f570e 100644 --- a/repository/sql_test.go +++ b/repository/sql_test.go @@ -20,11 +20,6 @@ import ( "gorm.io/gorm" ) -var user = os.Getenv("POSTGRES_USER") -var password = os.Getenv("POSTGRES_PASSWORD") -var pgdbname = os.Getenv("POSTGRES_DB") -var sqlitedbname = os.Getenv("SQLITE3_DB") - var store *sqlRepository type testSetup struct { @@ -112,6 +107,26 @@ func teardownPostgres(dsn string) { } func TestMain(m *testing.M) { + user := "postgres" + if u, ok := os.LookupEnv("POSTGRES_USER"); ok { + user = u + } + + password := "postgres" + if p, ok := os.LookupEnv("POSTGRES_PASSWORD"); ok { + password = p + } + + pgdbname := "postgres" + if pdb, ok := os.LookupEnv("POSTGRES_DB"); ok { + pgdbname = pdb + } + + sqlitedbname := "test.db" + if sdb, ok := os.LookupEnv("SQLITE3_DB"); ok { + sqlitedbname = sdb + } + wrappers := []testSetup{ { name: Postgres, From c4009a105ac0cb171464b301ddd9a30f67f10719 Mon Sep 17 00:00:00 2001 From: Ryan Morton Date: Thu, 17 Aug 2023 16:08:48 -0400 Subject: [PATCH 5/5] fix: switch actions/checkout back to v3 actions were temporarily changed to v2 in an attempt to run github actions locally with nektos/act. Found it was better to run it on my fork of the repo --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a513cf..1c88bef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: name: lint runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: go-version-file: 'go.mod' @@ -48,7 +48,7 @@ jobs: PGPASSWORD: postgres - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Run Tests @@ -56,4 +56,4 @@ jobs: with: go-version-file: 'go.mod' - run: | - go test -v ./... + go test -v -cover ./...