Skip to content

Commit

Permalink
Add graceful shutdown when interrupted
Browse files Browse the repository at this point in the history
If implemented this permit restoring a clean state in case of signal
interruption.

Signed-off-by: Soule BA <soule@weave.works>
  • Loading branch information
souleb committed Jan 20, 2022
1 parent f7d9ee9 commit 35a0af5
Show file tree
Hide file tree
Showing 11 changed files with 122 additions and 788 deletions.
35 changes: 27 additions & 8 deletions cmd/flux/build_kustomization.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ package main
import (
"fmt"
"os"
"os/signal"

"github.com/spf13/cobra"

"github.com/fluxcd/flux2/internal/kustomization"
"github.com/fluxcd/flux2/internal/build"
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
)

Expand All @@ -33,7 +34,7 @@ var buildKsCmd = &cobra.Command{
Long: `The build command queries the Kubernetes API and fetches the specified Flux Kustomization.
It then uses the fetched in cluster flux kustomization to perform needed transformation on the local kustomization.yaml
pointed at by --path. The local kustomization.yaml is generated if it does not exist. Finally it builds the overlays using the local kustomization.yaml, and write the resulting multi-doc YAML to stdout.`,
Example: `# Create a new overlay.
Example: `# Build the local manifests as they were built on the cluster
flux build kustomization my-app --path ./path/to/local/manifests`,
ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)),
RunE: buildKsCmdRun,
Expand Down Expand Up @@ -64,18 +65,36 @@ func buildKsCmdRun(cmd *cobra.Command, args []string) error {
return fmt.Errorf("invalid resource path %q", buildKsArgs.path)
}

builder, err := kustomization.NewBuilder(kubeconfigArgs, name, buildKsArgs.path, kustomization.WithTimeout(rootArgs.timeout))
builder, err := build.NewBuilder(kubeconfigArgs, name, buildKsArgs.path, build.WithTimeout(rootArgs.timeout))
if err != nil {
return err
}

manifests, err := builder.Build()
if err != nil {
return err
// create a signal channel
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt)

errChan := make(chan error)
go func() {
manifests, err := builder.Build()
if err != nil {
errChan <- err
}

cmd.Print(string(manifests))
errChan <- nil
}()

select {
case <-sigc:
fmt.Println("Build cancelled... exiting.")
return builder.Cancel()
case err := <-errChan:
if err != nil {
return err
}
}

cmd.Print(string(manifests))

return nil

}
37 changes: 28 additions & 9 deletions cmd/flux/diff_kustomization.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ package main
import (
"fmt"
"os"
"os/signal"

"github.com/spf13/cobra"

"github.com/fluxcd/flux2/internal/kustomization"
"github.com/fluxcd/flux2/internal/build"
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
)

Expand All @@ -31,7 +32,7 @@ var diffKsCmd = &cobra.Command{
Aliases: []string{"ks"},
Short: "Diff Kustomization",
Long: `The diff command does a build, then it performs a server-side dry-run and output the diff.`,
Example: `# Create a new overlay.
Example: `# Preview changes local changes as they were applied on the cluster
flux diff kustomization my-app --path ./path/to/local/manifests`,
ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)),
RunE: diffKsCmdRun,
Expand All @@ -44,7 +45,7 @@ type diffKsFlags struct {
var diffKsArgs diffKsFlags

func init() {
diffKsCmd.Flags().StringVar(&diffKsArgs.path, "path", "", "Name of a file containing a file to add to the kustomization file.)")
diffKsCmd.Flags().StringVar(&diffKsArgs.path, "path", "", "Path to a local directory that matches the specified Kustomization.spec.path.)")
diffCmd.AddCommand(diffKsCmd)
}

Expand All @@ -62,18 +63,36 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
return fmt.Errorf("invalid resource path %q", diffKsArgs.path)
}

builder, err := kustomization.NewBuilder(kubeconfigArgs, name, diffKsArgs.path, kustomization.WithTimeout(rootArgs.timeout))
builder, err := build.NewBuilder(kubeconfigArgs, name, diffKsArgs.path, build.WithTimeout(rootArgs.timeout))
if err != nil {
return err
}

output, err := builder.Diff()
if err != nil {
return err
// create a signal channel
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt)

errChan := make(chan error)
go func() {
output, err := builder.Diff()
if err != nil {
errChan <- err
}

cmd.Print(output)
errChan <- nil
}()

select {
case <-sigc:
fmt.Println("Build cancelled... exiting.")
return builder.Cancel()
case err := <-errChan:
if err != nil {
return err
}
}

cmd.Print(output)

return nil

}
4 changes: 2 additions & 2 deletions cmd/flux/diff_kustomization_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"strings"
"testing"

"github.com/fluxcd/flux2/internal/kustomization"
"github.com/fluxcd/flux2/internal/build"
"github.com/fluxcd/pkg/ssa"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
Expand Down Expand Up @@ -85,7 +85,7 @@ func TestDiffKustomization(t *testing.T) {
"fluxns": allocateNamespace("flux-system"),
}

b, _ := kustomization.NewBuilder(kubeconfigArgs, "podinfo", "")
b, _ := build.NewBuilder(kubeconfigArgs, "podinfo", "")

resourceManager, err := b.Manager()
if err != nil {
Expand Down
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ require (
github.com/fluxcd/image-reflector-controller/api v0.15.0
github.com/fluxcd/kustomize-controller/api v0.19.1
github.com/fluxcd/notification-controller/api v0.20.1
github.com/fluxcd/pkg/apis/kustomize v0.3.1
github.com/fluxcd/pkg/apis/kustomize v0.3.1 // indirect
github.com/fluxcd/pkg/apis/meta v0.10.2
github.com/fluxcd/pkg/runtime v0.12.3
github.com/fluxcd/pkg/ssa v0.10.0
github.com/fluxcd/pkg/ssa v0.11.0
github.com/fluxcd/pkg/ssh v0.3.1
github.com/fluxcd/pkg/untar v0.0.5
github.com/fluxcd/pkg/version v0.0.1
Expand All @@ -25,7 +25,7 @@ require (
github.com/gonvenience/ytbx v1.4.2
github.com/google/go-cmp v0.5.6
github.com/google/go-containerregistry v0.2.0
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
github.com/homeport/dyff v1.4.6
github.com/lucasb-eyer/go-colorful v1.2.0
Expand All @@ -49,7 +49,7 @@ require (
)

require (
github.com/drone/envsubst v1.0.3
github.com/fluxcd/pkg/kustomize v0.0.2
sigs.k8s.io/kustomize/kyaml v0.13.0
)

Expand All @@ -72,6 +72,7 @@ require (
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
Expand Down
10 changes: 6 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,8 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g=
github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g=
github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0=
github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
Expand Down Expand Up @@ -240,10 +240,12 @@ github.com/fluxcd/pkg/apis/kustomize v0.3.1 h1:wmb5D9e1+Rr3/5O3235ERuj+h2VKUArVf
github.com/fluxcd/pkg/apis/kustomize v0.3.1/go.mod h1:k2HSRd68UwgNmOYBPOd6WbX6a2MH2X/Jeh7e3s3PFPc=
github.com/fluxcd/pkg/apis/meta v0.10.2 h1:pnDBBEvfs4HaKiVAYgz+e/AQ8dLvcgmVfSeBroZ/KKI=
github.com/fluxcd/pkg/apis/meta v0.10.2/go.mod h1:KQ2er9xa6koy7uoPMZjIjNudB5p4tXs+w0GO6fRcy7I=
github.com/fluxcd/pkg/kustomize v0.0.2 h1:ipvQrxSeuGZDsPZrVUL6tYMlTR5xqYTZp6G0Tdy2hVs=
github.com/fluxcd/pkg/kustomize v0.0.2/go.mod h1:AFwnf3OqQmpTCuwCARTGpPRMBf0ZFJNGCvW63KbgK04=
github.com/fluxcd/pkg/runtime v0.12.3 h1:h21AZ3YG5MAP7DxFF9hfKrP+vFzys2L7CkUbPFjbP/0=
github.com/fluxcd/pkg/runtime v0.12.3/go.mod h1:imJ2xYy/d4PbSinX2IefmZk+iS2c1P5fY0js8mCE4SM=
github.com/fluxcd/pkg/ssa v0.10.0 h1:dhgWDeqz0/zAs5guzmPx/DMPCkzZdlEiPvCs1NChAQM=
github.com/fluxcd/pkg/ssa v0.10.0/go.mod h1:S+qig7BTOxop0c134y8Yv8/iQST4Kt7S2xXiFkP4VMA=
github.com/fluxcd/pkg/ssa v0.11.0 h1:ejEMlHPsbXMP8BJQx3+0PwoBgJur0mHiPcMNKcFwoOE=
github.com/fluxcd/pkg/ssa v0.11.0/go.mod h1:S+qig7BTOxop0c134y8Yv8/iQST4Kt7S2xXiFkP4VMA=
github.com/fluxcd/pkg/ssh v0.3.1 h1:iQw07bkX2rScactX8WYv+uMDsufFOlg8M3fV2TNs244=
github.com/fluxcd/pkg/ssh v0.3.1/go.mod h1:Sebfv4Um51PvomuYdMvKRncQW5dtKhZ5J5TA+wiHNSQ=
github.com/fluxcd/pkg/untar v0.0.5 h1:UGI3Ch1UIEIaqQvMicmImL1s9npQa64DJ/ozqHKB7gk=
Expand Down
54 changes: 43 additions & 11 deletions internal/kustomization/build.go → internal/build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,19 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package kustomization
package build

import (
"bytes"
"context"
"encoding/base64"
"fmt"
"sync"
"time"

"github.com/fluxcd/flux2/internal/utils"
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
"github.com/fluxcd/pkg/kustomize"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -37,9 +39,9 @@ import (
)

const (
controllerName = "kustomize-controller"
controllerGroup = "kustomize.toolkit.fluxcd.io"
mask string = "**SOPS**"
controllerName = "kustomize-controller"
controllerGroup = "kustomize.toolkit.fluxcd.io"
mask = "**SOPS**"
)

var defaultTimeout = 80 * time.Second
Expand All @@ -53,6 +55,9 @@ type Builder struct {
name string
namespace string
resourcesPath string
// mu is used to synchronize access to the kustomization file
mu sync.Mutex
action kustomize.Action
kustomization *kustomizev1.Kustomization
timeout time.Duration
}
Expand Down Expand Up @@ -149,13 +154,15 @@ func (b *Builder) build() (m resmap.ResMap, err error) {
// generate kustomization.yaml if needed
action, er := b.generate(*k, b.resourcesPath)
if er != nil {
errf := CleanDirectory(b.resourcesPath, action)
errf := kustomize.CleanDirectory(b.resourcesPath, action)
err = fmt.Errorf("failed to generate kustomization.yaml: %w", fmt.Errorf("%v %v", er, errf))
return
}

b.action = action

defer func() {
errf := CleanDirectory(b.resourcesPath, action)
errf := b.Cancel()
if err == nil {
err = errf
}
Expand Down Expand Up @@ -185,18 +192,28 @@ func (b *Builder) build() (m resmap.ResMap, err error) {

}

func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath string) (action, error) {
func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath string) (kustomize.Action, error) {
data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&kustomization)
if err != nil {
return "", err
}
gen := NewGenerator(unstructured.Unstructured{Object: data})
return gen.WriteFile(dirPath, WithSaveOriginalKustomization())
gen := kustomize.NewGenerator(unstructured.Unstructured{Object: data})

// acuire the lock
b.mu.Lock()
defer b.mu.Unlock()

return gen.WriteFile(dirPath, kustomize.WithSaveOriginalKustomization())
}

func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomization, dirPath string) (resmap.ResMap, error) {
fs := filesys.MakeFsOnDisk()
m, err := BuildKustomization(fs, dirPath)

// acuire the lock
b.mu.Lock()
defer b.mu.Unlock()

m, err := kustomize.BuildKustomization(fs, dirPath)
if err != nil {
return nil, fmt.Errorf("kustomize build failed: %w", err)
}
Expand All @@ -208,7 +225,7 @@ func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomizatio
if err != nil {
return nil, err
}
outRes, err := SubstituteVariables(ctx, b.client, unstructured.Unstructured{Object: data}, res)
outRes, err := kustomize.SubstituteVariables(ctx, b.client, unstructured.Unstructured{Object: data}, res)
if err != nil {
return nil, fmt.Errorf("var substitution failed for '%s': %w", res.GetName(), err)
}
Expand Down Expand Up @@ -263,3 +280,18 @@ func trimSopsData(res *resource.Resource) error {

return nil
}

// Cancel cancels the build
// It restores a clean reprository
func (b *Builder) Cancel() error {
// acuire the lock
b.mu.Lock()
defer b.mu.Unlock()

err := kustomize.CleanDirectory(b.resourcesPath, b.action)
if err != nil {
return err
}

return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package kustomization
package build

import (
"testing"
Expand Down
Loading

0 comments on commit 35a0af5

Please sign in to comment.