diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 52d99b0..0fb53c7 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -19,34 +19,15 @@ on: - master pull_request: jobs: - e2e: + unit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - - name: Create k8s Kind Cluster - uses: helm/kind-action@v1 - - name: Install Skaffold - run: | - curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.3.1/skaffold-linux-amd64 && \ - sudo install skaffold /usr/local/bin/ - - name: Run e2e tests - run: | - make test-e2e \ - ATLAS_TOKEN=${{ secrets.ATLAS_TOKEN }} \ - KIND_CLUSTER=chart-testing - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version-file: 'go.mod' - name: Install Atlas CLI - run: | - curl -sSf https://atlasgo.sh | ATLAS_DEBUG=true sh + uses: ariga/setup-atlas@master - name: Generate run: | make cli-gen @@ -63,9 +44,37 @@ jobs: - name: Run tests run: | make test - - uses: azure/setup-kubectl@v3 + e2e: + runs-on: ubuntu-latest + needs: [unit] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1 + - name: Install Skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.3.1/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + - name: Run e2e tests + run: | + make test-e2e \ + ATLAS_TOKEN=${{ secrets.ATLAS_TOKEN }} \ + KIND_CLUSTER=chart-testing + e2e-shell: + runs-on: ubuntu-latest + needs: [unit] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: Install Atlas CLI + uses: ariga/setup-atlas@master + - uses: azure/setup-kubectl@v4 - name: Start minikube - id: minikube uses: medyagh/setup-minikube@master - name: Install Skaffold run: | diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 0d479c0..4492060 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -58,34 +58,11 @@ cd ./config/integration # Bring up the database resources kubectl apply -f ./databases -mysql_reset "Test the atlas schema controller" -# Create a table not described in the desired schema but excluded from it. -mysql_exec "create table myapp.ignore_me (c int);" -# Apply the desired schema and wait for it to be ready. -kubectl apply -f ./schema -kubectl wait --for=condition=ready --timeout=360s atlasschemas --all -# Expect the excluded table to be present. -mysql_exec "describe myapp.ignore_me" -# Update sql schema configmap -kubectl create configmap mysql-schema \ - --from-file=./schema/mysql \ - --dry-run=client -o yaml | kubectl apply -f - -sleep 1 # Wait for the migration to be applied. -kubectl wait --for=condition=ready --timeout=120s atlasschemas --all -echo "" -echo "Expect the new column to be present" -mysql_exec "SHOW COLUMNS FROM myapp.users LIKE 'phone';" | grep -q 'phone' -echo "" -echo "Expect the devdb deployment is scaled to 1" -kubectl get deployment atlasschema-mysql-atlas-dev-db \ - -o=jsonpath='{.spec.replicas}' | grep -q '1' - mysql_reset "Test prewarm_devdb flag" # SET PREWARM_DEVDB to false kubectl set env -n atlas-operator-system deployment/atlas-operator-controller-manager \ PREWARM_DEVDB=false # Apply the desired schema and wait for it to be ready. -kubectl delete -f ./schema kubectl apply -f ./schema kubectl wait --for=condition=ready --timeout=120s atlasschemas --all echo "" diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index fd5c0c7..ff409c6 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -76,7 +76,7 @@ func TestOperator(t *testing.T) { "MSSQL_ACCEPT_EULA=Y", "MSSQL_PID=Developer") require.NoError(t, err) var controllerPod string - for range 5 { + for range 10 { // Getting the controller-manager pod name output, err := kind("kubectl", "get", "pod", "-n", nsController, @@ -96,7 +96,7 @@ func TestOperator(t *testing.T) { require.NotEmpty(t, controllerPod, "controller-manager pod not found") // Running the test script testscript.Run(t, testscript.Params{ - Dir: filepath.Join("testdata", "atlas-schema"), + Dir: filepath.Join("testscript"), Setup: func(e *testscript.Env) (err error) { e.Setenv("CONTROLLER_NS", nsController) e.Setenv("CONTROLLER", controllerPod) diff --git a/test/e2e/testscript/schema-mysql.txtar b/test/e2e/testscript/schema-mysql.txtar new file mode 100644 index 0000000..35e244d --- /dev/null +++ b/test/e2e/testscript/schema-mysql.txtar @@ -0,0 +1,227 @@ +env DB_URL=mysql://root:pass@mysql.${NAMESPACE}:3306/myapp +kubectl apply -f database.yaml +kubectl create secret generic mysql-credentials --from-literal=url=${DB_URL} +# Wait for the DB ready before creating the schema +kubectl wait --for=condition=ready --timeout=60s -l app=mysql pods + +# Sync the $WORK directory to the controller pod +kubectl cp -n ${CONTROLLER_NS} ${WORK} ${CONTROLLER}:/tmp/${NAMESPACE}/ +env DEV_URL=mysql://root:pass@mysql.${NAMESPACE}:3307/myapp +# Create a table not described in the desired schema but excluded from it. +atlas schema apply --auto-approve --dev-url=${DEV_URL} --url=${DB_URL} --to=file:///tmp/${NAMESPACE}/ignore.sql + +# Create the configmap to store the schema.sql +kubectl create configmap mysql-schema --from-file=./schema-v1 --dry-run=client -o yaml +stdin stdout +kubectl apply -f - + +# Create the schema +kubectl apply -f schema.yaml +kubectl wait --for=condition=ready --timeout=120s AtlasSchema/atlasschema-mysql + +# Inspect the schema to ensure it's correct +atlas schema inspect -u ${DB_URL} +cmp stdout schema-v1.hcl + +# Update the configmap with the new schema +kubectl create configmap mysql-schema --from-file=./schema-v2 --dry-run=client -o yaml +stdin stdout +kubectl apply -f - + +# Ensure the controller is aware of the change +kubectl wait --for=condition=ready=false --timeout=60s AtlasSchema/atlasschema-mysql +kubectl wait --for=condition=ready --timeout=120s AtlasSchema/atlasschema-mysql + +# Inspect the schema to ensure it's correct +atlas schema inspect -u ${DB_URL} +cmp stdout schema-v2.hcl +-- schema-v1.hcl -- +table "ignore_me" { + schema = schema.myapp + column "c" { + null = true + type = int + } +} +table "users" { + schema = schema.myapp + column "id" { + null = false + type = int + auto_increment = true + } + column "name" { + null = false + type = varchar(255) + } + column "email" { + null = false + type = varchar(255) + } + column "short_bio" { + null = false + type = varchar(255) + } + primary_key { + columns = [column.id] + } + index "email" { + unique = true + columns = [column.email] + } +} +schema "myapp" { + charset = "utf8mb4" + collate = "utf8mb4_0900_ai_ci" +} +-- schema-v2.hcl -- +table "ignore_me" { + schema = schema.myapp + column "c" { + null = true + type = int + } +} +table "users" { + schema = schema.myapp + column "id" { + null = false + type = int + auto_increment = true + } + column "name" { + null = false + type = varchar(255) + } + column "email" { + null = false + type = varchar(255) + } + column "short_bio" { + null = false + type = varchar(255) + } + column "phone" { + null = false + type = varchar(255) + } + primary_key { + columns = [column.id] + } + index "email" { + unique = true + columns = [column.email] + } +} +schema "myapp" { + charset = "utf8mb4" + collate = "utf8mb4_0900_ai_ci" +} +-- schema-v1/schema.sql -- +create table users ( + id int not null auto_increment, + name varchar(255) not null, + email varchar(255) unique not null, + short_bio varchar(255) not null, + primary key (id) +); +-- schema-v2/schema.sql -- +create table users ( + id int not null auto_increment, + name varchar(255) not null, + email varchar(255) unique not null, + short_bio varchar(255) not null, + phone varchar(255) not null, + primary key (id) +); +-- ignore.sql -- +create table myapp.ignore_me (c int); +-- schema.yaml -- +apiVersion: db.atlasgo.io/v1alpha1 +kind: AtlasSchema +metadata: + name: atlasschema-mysql +spec: + urlFrom: + secretKeyRef: + key: url + name: mysql-credentials + policy: + lint: + destructive: + error: true + diff: + skip: + drop_column: true + schema: + configMapKeyRef: + key: schema.sql + name: mysql-schema + exclude: + - ignore_me +-- database.yaml -- +apiVersion: v1 +kind: Service +metadata: + name: mysql +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + targetPort: mysql + - name: mysql-dev + port: 3307 + targetPort: mysql-dev + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql +spec: + selector: + matchLabels: + app: mysql + replicas: 1 + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:latest + env: + - name: MYSQL_ROOT_PASSWORD + value: pass + - name: MYSQL_DATABASE + value: myapp + ports: + - containerPort: 3306 + name: mysql + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + exec: + command: [ "mysql", "-ppass", "-h", "127.0.0.1", "-e", "SELECT 1" ] + - name: mysql-dev + image: mysql:latest + env: + - name: MYSQL_ROOT_PASSWORD + value: pass + - name: MYSQL_DATABASE + value: myapp + - name: MYSQL_TCP_PORT + value: "3307" + ports: + - containerPort: 3307 + name: mysql-dev + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + exec: + command: [ "mysql", "-ppass", "-h", "127.0.0.1", "-e", "SELECT 1" ] diff --git a/test/e2e/testdata/atlas-schema/postgres.txtar b/test/e2e/testscript/schema-postgres.txtar similarity index 100% rename from test/e2e/testdata/atlas-schema/postgres.txtar rename to test/e2e/testscript/schema-postgres.txtar diff --git a/test/e2e/testdata/atlas-schema/registry.txtar b/test/e2e/testscript/schema-registry.txtar similarity index 100% rename from test/e2e/testdata/atlas-schema/registry.txtar rename to test/e2e/testscript/schema-registry.txtar diff --git a/test/e2e/testdata/atlas-schema/sqlserver.txtar b/test/e2e/testscript/schema-sqlserver.txtar similarity index 100% rename from test/e2e/testdata/atlas-schema/sqlserver.txtar rename to test/e2e/testscript/schema-sqlserver.txtar