diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4076546b..335d46ce 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -8,7 +8,7 @@ jobs: name: Formatting runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup black linter run: conda create --quiet --name black pyflakes @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Go uses: actions/setup-go@v3 with: diff --git a/README.md b/README.md index 5d33404b..850e5aac 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ We apologize for bugs you run into, and hope you tell us soon so we can work on ## TODO - add [jar](wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.6.0/openapi-generator-cli-6.6.0.jar -O openapi-generator-cli.jar) to makefile + - Ensure that curve.cert is a variable (path) in case we create a custom one. + - When JobSet is available we need it to say job is successful when main application container is done. ## License diff --git a/api/v1alpha2/minicluster_types.go b/api/v1alpha2/minicluster_types.go index 87a814f6..52cd104e 100644 --- a/api/v1alpha2/minicluster_types.go +++ b/api/v1alpha2/minicluster_types.go @@ -19,7 +19,6 @@ import ( const ( entrypointSuffix = "-entrypoint" - fluxConfigSuffix = "-flux-config" curveVolumeSuffix = "-curve-mount" ) @@ -754,6 +753,9 @@ func uniqueExistingVolumes(containers []MiniClusterContainer) map[string]MiniClu func (f *MiniCluster) EntrypointConfigMapName() string { return f.Name + entrypointSuffix } +func (f *MiniCluster) CurveConfigMapName() string { + return f.Name + curveVolumeSuffix +} // Validate ensures we have data that is needed, and sets defaults if needed func (f *MiniCluster) Validate() bool { diff --git a/controllers/flux/configmap.go b/controllers/flux/configmap.go index 716acac2..2982167f 100644 --- a/controllers/flux/configmap.go +++ b/controllers/flux/configmap.go @@ -34,6 +34,7 @@ func (r *MiniClusterReconciler) getConfigMap( ) (*corev1.ConfigMap, ctrl.Result, error) { // Look for the config map by name + r.log.Info("👀️ Looking for ConfigMap 👀️", "Type", configName) existing := &corev1.ConfigMap{} err := r.Get( ctx, @@ -78,7 +79,10 @@ func (r *MiniClusterReconciler) getConfigMap( return dep, ctrl.Result{Requeue: true}, nil } else if err != nil { - r.log.Error(err, "Failed to get MiniCluster ConfigMap") + r.log.Error( + err, "Failed to get MiniCluster ConfigMap", + "Type", configName, + ) return existing, ctrl.Result{}, err } diff --git a/controllers/flux/minicluster.go b/controllers/flux/minicluster.go index 95c0b333..a4b0331e 100644 --- a/controllers/flux/minicluster.go +++ b/controllers/flux/minicluster.go @@ -29,9 +29,7 @@ import ( ) var ( - HostfileName = "hostfile" - CurveCertKey = "curve.cert" - mungeMountName = "munge-key" + CurveCertKey = "curve.cert" ) // This is a MiniCluster! A MiniCluster is associated with a running MiniCluster and include: diff --git a/controllers/flux/volumes.go b/controllers/flux/volumes.go index 9f7eaf67..be67523c 100644 --- a/controllers/flux/volumes.go +++ b/controllers/flux/volumes.go @@ -28,7 +28,6 @@ import ( // Shared function to return consistent set of volume mounts func getVolumeMounts(cluster *api.MiniCluster) []corev1.VolumeMount { mounts := []corev1.VolumeMount{ - // The empty volume for Flux will go here { Name: cluster.Spec.Flux.Container.Name, @@ -43,15 +42,6 @@ func getVolumeMounts(cluster *api.MiniCluster) []corev1.VolumeMount { ReadOnly: true, }, } - // Are we expecting a munge secret? - if cluster.Spec.Flux.MungeSecret != "" { - mungeMount := corev1.VolumeMount{ - Name: mungeMountName, - MountPath: "/etc/munge", - ReadOnly: true, - } - mounts = append(mounts, mungeMount) - } return mounts } @@ -86,6 +76,12 @@ func getVolumes(cluster *api.MiniCluster) []corev1.Volume { } } + // /flux_operator/curve.cert + curveKey := corev1.KeyToPath{ + Key: CurveCertKey, + Path: "curve.cert", + } + // Add the flux init script fluxScript := corev1.KeyToPath{ Key: cluster.Spec.Flux.Container.Name, @@ -93,12 +89,7 @@ func getVolumes(cluster *api.MiniCluster) []corev1.Volume { Mode: &makeExecutable, } runnerStartScripts = append(runnerStartScripts, fluxScript) - - // Only used if we need to provide the munge.key - mungeKey := corev1.KeyToPath{ - Key: cluster.Spec.Flux.MungeSecret, - Path: "munge.key", - } + runnerStartScripts = append(runnerStartScripts, curveKey) // Defaults volumes we always write - entrypoint and empty volume volumes := []corev1.Volume{ @@ -124,19 +115,37 @@ func getVolumes(cluster *api.MiniCluster) []corev1.Volume { }, } - // Are we expecting a munge config map? - if cluster.Spec.Flux.MungeSecret != "" { - mungeVolume := corev1.Volume{ - Name: mungeMountName, + // We either generate a curve.cert config map, or get it from secret + curveVolumeName := cluster.CurveConfigMapName() + if cluster.Spec.Flux.CurveCertSecret != "" { + curveVolume := corev1.Volume{ + Name: curveVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Spec.Flux.MungeSecret, - Items: []corev1.KeyToPath{mungeKey}, + SecretName: cluster.Spec.Flux.CurveCertSecret, + Items: []corev1.KeyToPath{curveKey}, }, }, } - volumes = append(volumes, mungeVolume) + + volumes = append(volumes, curveVolume) + + } else { + curveVolume := corev1.Volume{ + Name: curveVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + + // Namespace based on the cluster + LocalObjectReference: corev1.LocalObjectReference{ + Name: cluster.CurveConfigMapName(), + }, + }, + }, + } + volumes = append(volumes, curveVolume) } + // Add volumes that already exist (not created by the Flux Operator) // These are unique names and path/claim names across containers // This can be a claim, secret, or config map diff --git a/examples/tests/hello-world/test.out.correct b/examples/tests/hello-world/test.out.correct index f35d3e67..759eadb9 100644 --- a/examples/tests/hello-world/test.out.correct +++ b/examples/tests/hello-world/test.out.correct @@ -1 +1,2 @@ -hello world +Defaulted container "flux-sample" out of: flux-sample, flux-view +hello world diff --git a/pkg/flux/entrypoint.go b/pkg/flux/entrypoint.go index e9bdf40d..b28a2a42 100644 --- a/pkg/flux/entrypoint.go +++ b/pkg/flux/entrypoint.go @@ -51,7 +51,11 @@ func GenerateEntrypoints(cluster *api.MiniCluster) (map[string]string, error) { return data, err } data[cluster.Spec.Flux.Container.Name] = script - return data, nil + + // Add the curve.cert + curveCert, err := GetCurveCert(cluster) + data["curve.cert"] = curveCert + return data, err } // generateServiceEntrypoint generates an entrypoint for a service container diff --git a/pkg/flux/templates/wait.sh b/pkg/flux/templates/wait.sh index 99eac658..1c51a3ca 100644 --- a/pkg/flux/templates/wait.sh +++ b/pkg/flux/templates/wait.sh @@ -59,25 +59,30 @@ if [[ "${fluxuser}" != "root" ]]; then fi # Ensure the flux user owns the curve.cert -curvepath=${viewroot}/etc/curve/curve.cert +# We need to move the curve.cert because config map volume is read only +curvesrc=/flux_operator/curve.cert +curvepath=$viewroot/curve/curve.cert + +mkdir -p $viewroot/curve +cp $curvesrc $curvepath {{ if not .Spec.Logging.Quiet }} echo echo "🌟️ Curve Certificate" -ls ${viewroot}/etc/curve/ +ls $viewroot/curve cat ${curvepath} {{ end }} # Remove group and other read -# chmod o-r ${curvepath} -# chmod g-r ${curvepath} +chmod o-r ${curvepath} +chmod g-r ${curvepath} chown -R ${fluxuid} ${curvepath} foundroot=$(find $viewroot -maxdepth 2 -type d -path $viewroot/lib/python3\*) # Ensure we use flux's python (TODO update this to use variable) export PYTHONPATH={{ if .Spec.Flux.Container.PythonPath }}{{ .Spec.Flux.Container.PythonPath }}{{ else }}${foundroot}/site-packages{{ end }} -echo "PYTHONPATH is ${PYTHONPATH}" -echo "PATH is $PATH" +echo "PYTHONPATH is ${PYTHONPATH}" {{ if .Spec.Logging.Quiet }}> /dev/null 2>&1{{ end }} +echo "PATH is $PATH" {{ if .Spec.Logging.Quiet }}> /dev/null 2>&1{{ end }} # Put the state directory in /var/lib on shared view export STATE_DIR=${viewroot}/var/lib/flux @@ -184,7 +189,7 @@ flux jobs -a{{ end }} # Commands only run by the broker {{ .Container.Commands.BrokerPre}} {{ if .Spec.Logging.Quiet }}> /dev/null 2>&1{{ end }} - echo "Command provided is: ${command}" + echo "Command provided is: ${command}" {{ if .Spec.Logging.Quiet }}> /dev/null 2>&1{{ end }} if [ "${command}" == "" ]; then # An interactive job also doesn't require a command diff --git a/pkg/flux/view.go b/pkg/flux/view.go index 1a779b87..d5aa9f1e 100644 --- a/pkg/flux/view.go +++ b/pkg/flux/view.go @@ -65,7 +65,7 @@ allow-root-owner = true path = "%s/view/etc/flux/system/R" [bootstrap] -curve_cert = "%s/view/etc/curve/curve.cert" +curve_cert = "%s/view/curve/curve.cert" default_port = 8050 default_bind = "%s" default_connect = "%s" @@ -103,12 +103,6 @@ func GenerateFluxEntrypoint(cluster *api.MiniCluster) (string, error) { mainHost := fmt.Sprintf("%s-0", cluster.Name) - // Generate the curve certificate - curveCert, err := GetCurveCert(cluster) - if err != nil { - return "", err - } - // Generate hostlists, this is the lead broker hosts := generateHostlist(cluster, cluster.Spec.MaxSize) brokerConfig := generateBrokerConfig(cluster, hosts) @@ -153,16 +147,9 @@ cat ${fluxroot}/etc/flux/config/broker.toml # Along with the state directory and curve certificate mkdir -p ${fluxroot}/run/flux ${fluxroot}/etc/curve -# Generate the certificate (ONLY if the lead broker) -if [[ "$(hostname)" == "${mainHost}" ]]; then -echo "Generating curve certificate at main host..." -cat <> ${fluxroot}/etc/curve/curve.cert -%s -EOT -echo +# View the curve certificate echo "🌟️ Curve Certificate" -cat ${fluxroot}/etc/curve/curve.cert -fi +cat /flux_operator/curve.cert # Now prepare to copy finished spack view over echo "Moving content from /opt/view to be in shared volume at %s" @@ -192,7 +179,6 @@ sleep infinity mainHost, hosts, brokerConfig, - curveCert, cluster.Spec.Flux.Container.MountPath, cluster.Spec.Flux.Container.MountPath, cluster.Spec.Flux.Container.Name,