chore: compress directory

controllers/atlasmigration_controller.go

@@ -30,6 +30,8 @@ limitations under the License.
 package controllers
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -197,6 +199,19 @@ func (r *AtlasMigrationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	return ctrl.Result{}, nil
 }
 
+func (r *AtlasMigrationReconciler) readMigrations(ctx context.Context, res *dbv1alpha1.AtlasMigration) (migrate.Dir, error) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      makeKeyLatest(res.Name),
+			Namespace: res.Namespace,
+		},
+	}
+	if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
+		return nil, err
+	}
+	return extractDirFromSecret(secret)
+}
+
 func (r *AtlasMigrationReconciler) storeMigrations(ctx context.Context, res *dbv1alpha1.AtlasMigration, dir migrate.Dir) error {
 	var labels = map[string]string{}
 	for k, v := range res.Labels {
@@ -448,14 +463,19 @@ func makeKeyLatest(resName string) string {
 
 func newSecretObject(key string, dir migrate.Dir, labels map[string]string) (*corev1.Secret, error) {
 	const owner = "atlasgo.io"
-	tar, err := migrate.ArchiveDir(dir)
-	if err != nil {
-		return nil, err
-	}
 	if labels == nil {
 		labels = map[string]string{}
 	}
 	labels["owner"] = owner
+	var buf bytes.Buffer
+	w := gzip.NewWriter(&buf)
+	if err := migrate.ArchiveDirTo(w, dir); err != nil {
+		return nil, err
+	}
+	// Close the gzip writer to flush the buffer
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
 	return &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   key,
@@ -465,7 +485,22 @@ func newSecretObject(key string, dir migrate.Dir, labels map[string]string) (*co
 		Data: map[string][]byte{
 			// k8s already encodes the tarball in base64
 			// so we don't need to encode it again.
-			"migrations.tar": tar,
+			"migrations.tar.gz": buf.Bytes(),
 		},
 	}, nil
 }
+
+func extractDirFromSecret(sec *corev1.Secret) (migrate.Dir, error) {
+	if sec.Type != "atlasgo.io/db.v1" {
+		return nil, fmt.Errorf("invalid secret type, got %q", sec.Type)
+	}
+	tarball, ok := sec.Data["migrations.tar.gz"]
+	if !ok {
+		return nil, errors.New("migrations.tar.gz not found")
+	}
+	r, err := gzip.NewReader(bytes.NewReader(tarball))
+	if err != nil {
+		return nil, err
+	}
+	return migrate.UnarchiveDirFrom(r)
+}

controllers/atlasmigration_controller_test.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -184,7 +185,7 @@ func TestMigration_Local(t *testing.T) {
 			require.Equal(t, version, res.Status.LastAppliedVersion)
 		})
 	}
-	newDir := func(dir map[string]string) {
+	updateDir := func(dir map[string]string) {
 		t.Helper()
 		h.patch(t, &dbv1alpha1.AtlasMigration{
 			ObjectMeta: meta,
@@ -193,26 +194,49 @@ func TestMigration_Local(t *testing.T) {
 			},
 		})
 	}
+	assertDir := func(dirMap map[string]string) {
+		t.Helper()
+		// Check the content of the tarball
+		h.get(t, &dbv1alpha1.AtlasMigration{ObjectMeta: meta})
+		secret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      makeKeyLatest("atlas-migration"),
+				Namespace: "default",
+			},
+		}
+		h.get(t, secret)
+		dir, err := extractDirFromSecret(secret)
+		require.NoError(t, err)
+		require.NotNil(t, dir)
+		// It should contain the same files as the local directory
+		testContent(t, dirMap, dir)
+	}
 	// First reconcile
 	assert(ctrl.Result{Requeue: true}, false, "Reconciling", "Reconciling", "")
 	// Second reconcile
 	assert(ctrl.Result{}, true, "Applied", "", "20230412003626")
+	assertDir(obj.Spec.Dir.Local)
 	// Third reconcile, should not change the status
 	assert(ctrl.Result{}, true, "Applied", "", "20230412003626")
 	// Update the migration script
-	newDir(map[string]string{
+	newDir := map[string]string{
 		"20230412003626_create_foo.sql": "CREATE TABLE foo (id INT PRIMARY KEY);",
 		"20230808132722_add-boo.sql":    "CREATE TABLE boo (id INT PRIMARY KEY);",
 		"atlas.sum": `h1:zgFwhjzwhLZr82YtR4+PijDiVYNxwr18C3EqZtG4wyE=
 		20230412003626_create_foo.sql h1:8C7Hz48VGKB0trI2BsK5FWpizG6ttcm9ep+tX32y0Tw=
 		20230808132722_add-boo.sql h1:tD/Qak7Q4n0bp9wO8bjWYhRRcgp+oYcUDQIumztpYpg=`,
-	})
+	}
+	updateDir(newDir)
 	// Fourth reconcile, should change the status to Reconciling
 	assert(ctrl.Result{Requeue: true}, false, "Reconciling", "Current migration data has changed", "20230412003626")
+	// The content should not change during the migration
+	assertDir(obj.Spec.Dir.Local)
 	// Fifth reconcile, should change the status to Applied
 	assert(ctrl.Result{}, true, "Applied", "", "20230808132722")
+	// The content should change to the new directory
+	assertDir(newDir)
 	// Update the migration script with bad SQL
-	newDir(map[string]string{
+	updateDir(map[string]string{
 		"20230412003626_create_foo.sql": "CREATE TABLE foo (id INT PRIMARY KEY);",
 		"20230808132722_add-boo.sql":    "CREATE TABLE boo (id INT PRIMARY KEY);",
 		"20230808140359_bad-sql.sql":    "SYNTAX ERROR",
@@ -225,6 +249,8 @@ func TestMigration_Local(t *testing.T) {
 	assert(ctrl.Result{Requeue: true}, false, "Reconciling", "Current migration data has changed", "20230808132722")
 	// Seventh reconcile, should change the status to Failed
 	assert(ctrl.Result{}, false, "Migrating", `"SYNTAX ERROR" from version "20230808140359"`, "20230808132722")
+	// The content should not change when the migration fails
+	assertDir(newDir)
 	// Check the events generated by the controller
 	require.Equal(t, []string{
 		"Normal Applied Version 20230412003626 applied",
@@ -315,6 +341,31 @@ func TestReconcile_LocalMigrationDir(t *testing.T) {
 
 	status := tt.status()
 	require.EqualValues(tt, "20230412003626", status.LastAppliedVersion)
+
+	fsDir, err := getSecretValue(context.Background(), tt.r, "default", &corev1.SecretKeySelector{
+		LocalObjectReference: corev1.LocalObjectReference{
+			Name: makeKeyLatest("atlas-migration"),
+		},
+		Key: "migrations.tar.gz",
+	})
+	require.NoError(t, err)
+	require.NotNil(t, fsDir)
+
+	// Check the content of the tarball
+	dir, err := tt.r.readMigrations(context.Background(), am)
+	require.NoError(t, err)
+	require.NotNil(t, dir)
+	// It should contain the same files as the local directory
+	testContent(t, am.Spec.Dir.Local, dir)
+}
+
+func testContent(t *testing.T, files map[string]string, dir fs.FS) {
+	t.Helper()
+	for f, c := range files {
+		foo, err := fs.ReadFile(dir, f)
+		require.NoError(t, err)
+		require.EqualValues(t, c, string(foo))
+	}
 }
 
 func TestReconcile_LocalMigrationDir_ConfigMap(t *testing.T) {

go.mod

@@ -5,7 +5,7 @@ go 1.22
 toolchain go1.22.0
 
 require (
-	ariga.io/atlas v0.21.2-0.20240418081819-02b3f6239b04
+	ariga.io/atlas v0.21.2-0.20240424054305-cf871e4a7c1e
 	ariga.io/atlas-go-sdk v0.5.4-0.20240419053913-865efa3e6f8f
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/mod v0.17.0

go.sum

@@ -1,5 +1,5 @@
-ariga.io/atlas v0.21.2-0.20240418081819-02b3f6239b04 h1:YF3qiqtnhn+y4tfhZKTfZKfizpjqHYt7rWPUb+eA4ZA=
-ariga.io/atlas v0.21.2-0.20240418081819-02b3f6239b04/go.mod h1:VPlcXdd4w2KqKnH54yEZcry79UAhpaWaxEsmn5JRNoE=
+ariga.io/atlas v0.21.2-0.20240424054305-cf871e4a7c1e h1:3iOt1DD4c6IGRjxcm+C5IeI+pHo7HnqMoN14xjVjxsE=
+ariga.io/atlas v0.21.2-0.20240424054305-cf871e4a7c1e/go.mod h1:VPlcXdd4w2KqKnH54yEZcry79UAhpaWaxEsmn5JRNoE=
 ariga.io/atlas-go-sdk v0.5.4-0.20240419053913-865efa3e6f8f h1:KlImyCEkKSsoFVOzxOnsraDzdt95g0Sqewt8sjJcbrY=
 ariga.io/atlas-go-sdk v0.5.4-0.20240419053913-865efa3e6f8f/go.mod h1:9Q+/04PVyJHUse1lEE9Kp6E18xj/6mIzaUTcWYSjSnQ=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=