Skip to content

Commit

Permalink
[TEP-0135] Revert PVC creation
Browse files Browse the repository at this point in the history 
Part of [#6740] and unblocks [#6635]. `PVCs` are created if a user specifies `VolumeClaimTemplate` as the source of a `pipelinerun workspace`.
Prior to this change, such `pvcs` are created via `affinity assistant statefulset` when `affinity assistant` is enabled (in both `workspaces` or `pipelineruns`
coschedule mode).

To delete such pvcs when the owning `pipelinerun` is completed, we must explicitly delete those pvcs in the reconciler since the owner of such pvcs is the `affinity assistant statefulset`
instead of the `pipelinerun`. The problem of such deletion mechanism is that such `pvcs` are left in `terminating` state when the owning `pipelinerun` is `completed` but not `deleted`.
This is because the pvcs are protected by `kubernetes.io/pvc-protection` `finalizer`, which does not allow the `pvcs` to be deleted until the `pipelinerun` consuming the `pvc` is deleted.
Leaving pvcs in `terminating` state is confusing to cluster operators. Instead of changing the pvc deletion behavior in such backward incompatible way,
it is better to make it configurable so that it is backward compatible, as prototyped in [#6635].

This commit reverts the pvc creation behavior `per-workspace` coschedule mode, which changes the owner of the `pvcs` back to the `pipelinerun` instead of the
`affinity assistant statefulset`. After the commit, the pvcs created by specifying `VolumeClaimTemplate` are left in `bounded` state instead of `terminating`.
This commit is the prerequisite of [#6635].

This commit does NOT reverts the pvc creation behavior `per-pipelinerun` coschedule mode as we will enforce the deletion of pvcs when the owning `pipelinerun` is completed.
This is a better practice and there is no backward compatability concern since `per-pipelinerun` coschedule mode is a new feature.

[#6740]: #6740
[#6635]: #6635
  • Loading branch information
@QuanZhang-William
QuanZhang-William committed Jul 11, 2023
1 parent 29ddf85 commit e40338c
Show file tree
Hide file tree
Showing 8 changed files with 285 additions and 129 deletions.
47 changes: 27 additions & 20 deletions pkg/reconciler/pipelinerun/affinity_assistant.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,19 @@ import (

const (
// ReasonCouldntCreateOrUpdateAffinityAssistantStatefulSetPerWorkspace indicates that a PipelineRun uses workspaces with PersistentVolumeClaim
// as a volume source and expect an Assistant StatefulSet, but couldn't create a StatefulSet.
// as a volume source and expect an Assistant StatefulSet in AffinityAssistantPerWorkspace behavior, but couldn't create a StatefulSet.
ReasonCouldntCreateOrUpdateAffinityAssistantStatefulSetPerWorkspace = "ReasonCouldntCreateOrUpdateAffinityAssistantStatefulSetPerWorkspace"

featureFlagDisableAffinityAssistantKey = "disable-affinity-assistant"
)

// createOrUpdateAffinityAssistantsPerAABehavior creates Affinity Assistant StatefulSets based on AffinityAssistantBehavior.
// createOrUpdateAffinityAssistantsAndPVCs creates Affinity Assistant StatefulSets and PVCs based on AffinityAssistantBehavior.
// This is done to achieve Node Affinity for taskruns in a pipelinerun, and make it possible for the taskruns to execute parallel while sharing volume.
// If the AffinityAssitantBehavior is AffinityAssistantPerWorkspace, it creates an Affinity Assistant for
// every taskrun in the pipelinerun that use the same PVC based volume.
// If the AffinityAssitantBehavior is AffinityAssistantPerPipelineRun or AffinityAssistantPerPipelineRunWithIsolation,
// it creates one Affinity Assistant for the pipelinerun.
// Other AffinityAssitantBehaviors are invalid.
func (c *Reconciler) createOrUpdateAffinityAssistantsPerAABehavior(ctx context.Context, pr *v1.PipelineRun, aaBehavior aa.AffinityAssitantBehavior) error {
func (c *Reconciler) createOrUpdateAffinityAssistantsAndPVCs(ctx context.Context, pr *v1.PipelineRun, aaBehavior aa.AffinityAssitantBehavior) error {
var errs []error
var unschedulableNodes sets.Set[string] = nil

Expand All @@ -79,26 +78,40 @@ func (c *Reconciler) createOrUpdateAffinityAssistantsPerAABehavior(ctx context.C
}
}

// create PVCs from PipelineRun's VolumeClaimTemplate when aaBehavior is AffinityAssistantPerWorkspace or AffinityAssistantDisabled before creating
// affinity assistant so that the OwnerReference of the PVCs are the pipelineruns, which is used to achieve PVC auto deletion at PipelineRun deletion time
if (aaBehavior == aa.AffinityAssistantPerWorkspace || aaBehavior == aa.AffinityAssistantDisabled) && pr.HasVolumeClaimTemplate() {
if err := c.pvcHandler.CreatePVCsForWorkspaces(ctx, pr.Spec.Workspaces, *kmeta.NewControllerRef(pr), pr.Namespace); err != nil {
return fmt.Errorf("failed to create PVC for PipelineRun %s: %w", pr.Name, err)
}
}

switch aaBehavior {
case aa.AffinityAssistantPerWorkspace:
for claim, workspaceName := range claimToWorkspace {
aaName := getAffinityAssistantName(workspaceName, pr.Name)
aaName := GetAffinityAssistantName(workspaceName, pr.Name)
err := c.createOrUpdateAffinityAssistant(ctx, aaName, pr, nil, []corev1.PersistentVolumeClaimVolumeSource{*claim}, unschedulableNodes)
errs = append(errs, err...)
}
for claimTemplate, workspaceName := range claimTemplatesToWorkspace {
aaName := getAffinityAssistantName(workspaceName, pr.Name)
err := c.createOrUpdateAffinityAssistant(ctx, aaName, pr, []corev1.PersistentVolumeClaim{*claimTemplate}, nil, unschedulableNodes)
aaName := GetAffinityAssistantName(workspaceName, pr.Name)
// To support PVC auto deletion at pipelinerun deletion time, the OwnerReference of the PVCs should be set to the owning pipelinerun.
// In AffinityAssistantPerWorkspace mode, the reconciler has created PVCs (owned by pipelinerun) from pipelinerun's VolumeClaimTemplate at this point,
// so the VolumeClaimTemplates are pass in as PVCs when creating affinity assistant StatefulSet for volume scheduling.
// If passed in as VolumeClaimTemplates, the PVCs are owned by Affinity Assistant StatefulSet instead of the pipelinerun.
err := c.createOrUpdateAffinityAssistant(ctx, aaName, pr, nil, []corev1.PersistentVolumeClaimVolumeSource{{ClaimName: claimTemplate.Name}}, unschedulableNodes)
errs = append(errs, err...)
}
case aa.AffinityAssistantPerPipelineRun, aa.AffinityAssistantPerPipelineRunWithIsolation:
if claims != nil || claimTemplates != nil {
aaName := getAffinityAssistantName("", pr.Name)
aaName := GetAffinityAssistantName("", pr.Name)
// In AffinityAssistantPerPipelineRun or AffinityAssistantPerPipelineRunWithIsolation modes, the PVCs are created via StatefulSet for volume scheduling.
// PVCs from pipelinerun's VolumeClaimTemplate are enforced to be deleted at pipelinerun completion time,
// so we don't need to worry the OwnerReference of the PVCs
err := c.createOrUpdateAffinityAssistant(ctx, aaName, pr, claimTemplates, claims, unschedulableNodes)
errs = append(errs, err...)
}
case aa.AffinityAssistantDisabled:
return fmt.Errorf("unexpected Affinity Assistant behavior %v", aa.AffinityAssistantDisabled)
}

return errorutils.NewAggregate(errs)
Expand Down Expand Up @@ -175,18 +188,10 @@ func (c *Reconciler) cleanupAffinityAssistants(ctx context.Context, pr *v1.Pipel
var errs []error
for _, w := range pr.Spec.Workspaces {
if w.PersistentVolumeClaim != nil || w.VolumeClaimTemplate != nil {
affinityAssistantStsName := getAffinityAssistantName(w.Name, pr.Name)
affinityAssistantStsName := GetAffinityAssistantName(w.Name, pr.Name)
if err := c.KubeClientSet.AppsV1().StatefulSets(pr.Namespace).Delete(ctx, affinityAssistantStsName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to delete StatefulSet %s: %w", affinityAssistantStsName, err))
}

// cleanup PVCs created by Affinity Assistants
if w.VolumeClaimTemplate != nil {
pvcName := getPersistentVolumeClaimNameWithAffinityAssistant(w.Name, pr.Name, w, *kmeta.NewControllerRef(pr))
if err := c.KubeClientSet.CoreV1().PersistentVolumeClaims(pr.Namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to delete PersistentVolumeClaim %s: %w", pvcName, err))
}
}
}
}
return errorutils.NewAggregate(errs)
Expand All @@ -195,13 +200,15 @@ func (c *Reconciler) cleanupAffinityAssistants(ctx context.Context, pr *v1.Pipel
// getPersistentVolumeClaimNameWithAffinityAssistant returns the PersistentVolumeClaim name that is
// created by the Affinity Assistant StatefulSet VolumeClaimTemplate when Affinity Assistant is enabled.
// The PVCs created by StatefulSet VolumeClaimTemplates follow the format `<pvcName>-<affinityAssistantName>-0`
// TODO(#6740)(WIP): use this function when adding end-to-end support for AffinityAssistantPerPipelineRun mode
func getPersistentVolumeClaimNameWithAffinityAssistant(pipelineWorkspaceName, prName string, wb v1.WorkspaceBinding, owner metav1.OwnerReference) string {
pvcName := volumeclaim.GetPVCNameWithoutAffinityAssistant(wb.VolumeClaimTemplate.Name, wb, owner)
affinityAssistantName := getAffinityAssistantName(pipelineWorkspaceName, prName)
affinityAssistantName := GetAffinityAssistantName(pipelineWorkspaceName, prName)
return fmt.Sprintf("%s-%s-0", pvcName, affinityAssistantName)
}

func getAffinityAssistantName(pipelineWorkspaceName string, pipelineRunName string) string {
// GetAffinityAssistantName returns the Affinity Assistant name based on pipelineWorkspaceName and pipelineRunName
func GetAffinityAssistantName(pipelineWorkspaceName string, pipelineRunName string) string {
hashBytes := sha256.Sum256([]byte(pipelineWorkspaceName + pipelineRunName))
hashString := fmt.Sprintf("%x", hashBytes)
return fmt.Sprintf("%s-%s", workspace.ComponentNameAffinityAssistant, hashString[:10])
Expand Down
122 changes: 68 additions & 54 deletions pkg/reconciler/pipelinerun/affinity_assistant_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package pipelinerun
import (
"context"
"errors"
"fmt"
"testing"

"github.com/google/go-cmp/cmp"
Expand All @@ -29,9 +28,11 @@ import (
"github.com/tektoncd/pipeline/pkg/apis/pipeline/pod"
v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
aa "github.com/tektoncd/pipeline/pkg/internal/affinityassistant"
"github.com/tektoncd/pipeline/pkg/reconciler/volumeclaim"
"github.com/tektoncd/pipeline/pkg/workspace"
"github.com/tektoncd/pipeline/test/diff"
"github.com/tektoncd/pipeline/test/parse"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -168,30 +169,32 @@ func TestCreateAndDeleteOfAffinityAssistantPerPipelineRun(t *testing.T) {
KubeClientSet: fakek8s.NewSimpleClientset(),
}

err := c.createOrUpdateAffinityAssistantsPerAABehavior(ctx, tc.pr, aa.AffinityAssistantPerPipelineRun)
err := c.createOrUpdateAffinityAssistantsAndPVCs(ctx, tc.pr, aa.AffinityAssistantPerPipelineRun)
if err != nil {
t.Errorf("unexpected error from createOrUpdateAffinityAssistantsPerPipelineRun: %v", err)
}

// validate StatefulSets from Affinity Assistant
expectAAName := getAffinityAssistantName("", tc.pr.Name)
expectAAName := GetAffinityAssistantName("", tc.pr.Name)
validateStatefulSetSpec(t, ctx, c, expectAAName, tc.expectStatefulSetSpec)

// TODO(#6740)(WIP): test cleanupAffinityAssistants for coscheduling-pipelinerun mode when fully implemented
})
}
}

// TestCreateAndDeleteOfAffinityAssistantPerWorkspace tests to create and delete an Affinity Assistant
// TestCreateAndDeleteOfAffinityAssistantPerWorkspaceOrDisabled tests to create and delete an Affinity Assistant
// per workspace for a given PipelineRun
func TestCreateAndDeleteOfAffinityAssistantPerWorkspace(t *testing.T) {
func TestCreateAndDeleteOfAffinityAssistantPerWorkspaceOrDisabled(t *testing.T) {
tests := []struct {
name string
name, expectedPVCName string
pr *v1.PipelineRun
expectStatefulSetSpec []*appsv1.StatefulSetSpec
aaBehavior aa.AffinityAssitantBehavior
}{{
name: "PersistentVolumeClaim Workspace type",
pr: testPRWithPVC,
name: "PersistentVolumeClaim Workspace type",
aaBehavior: aa.AffinityAssistantPerWorkspace,
pr: testPRWithPVC,
expectStatefulSetSpec: []*appsv1.StatefulSetSpec{{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Expand All @@ -205,20 +208,43 @@ func TestCreateAndDeleteOfAffinityAssistantPerWorkspace(t *testing.T) {
},
}},
}, {
name: "VolumeClaimTemplate Workspace type",
pr: testPRWithVolumeClaimTemplate,
name: "VolumeClaimTemplate Workspace type",
aaBehavior: aa.AffinityAssistantPerWorkspace,
pr: testPRWithVolumeClaimTemplate,
expectedPVCName: "pvc-b9eea16dce",
expectStatefulSetSpec: []*appsv1.StatefulSetSpec{{
VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{
ObjectMeta: metav1.ObjectMeta{Name: "pvc-b9eea16dce"},
}},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{{
Name: "workspace-0",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc-b9eea16dce"},
},
}},
},
},
}},
}, {
name: "VolumeClaimTemplate and PersistentVolumeClaim Workspaces",
pr: testPRWithVolumeClaimTemplateAndPVC,
name: "VolumeClaimTemplate Workspace type - AA disabled",
aaBehavior: aa.AffinityAssistantDisabled,
pr: testPRWithVolumeClaimTemplate,
expectedPVCName: "pvc-b9eea16dce",
}, {
name: "VolumeClaimTemplate and PersistentVolumeClaim Workspaces",
aaBehavior: aa.AffinityAssistantPerWorkspace,
pr: testPRWithVolumeClaimTemplateAndPVC,
expectedPVCName: "pvc-b9eea16dce",
expectStatefulSetSpec: []*appsv1.StatefulSetSpec{{
VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{
ObjectMeta: metav1.ObjectMeta{Name: "pvc-b9eea16dce"},
}}}, {
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{{
Name: "workspace-0",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc-b9eea16dce"},
},
}},
},
}}, {
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{{
Expand All @@ -232,6 +258,7 @@ func TestCreateAndDeleteOfAffinityAssistantPerWorkspace(t *testing.T) {
}},
}, {
name: "other Workspace type",
aaBehavior: aa.AffinityAssistantPerWorkspace,
pr: testPRWithEmptyDir,
expectStatefulSetSpec: nil,
}}
Expand All @@ -240,23 +267,33 @@ func TestCreateAndDeleteOfAffinityAssistantPerWorkspace(t *testing.T) {
tc := tc
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
kubeClientSet := fakek8s.NewSimpleClientset()
c := Reconciler{
KubeClientSet: fakek8s.NewSimpleClientset(),
KubeClientSet: kubeClientSet,
pvcHandler: volumeclaim.NewPVCHandler(kubeClientSet, zap.NewExample().Sugar()),
}

err := c.createOrUpdateAffinityAssistantsPerAABehavior(ctx, tc.pr, aa.AffinityAssistantPerWorkspace)
err := c.createOrUpdateAffinityAssistantsAndPVCs(ctx, tc.pr, tc.aaBehavior)
if err != nil {
t.Errorf("unexpected error from createOrUpdateAffinityAssistantsPerWorkspace: %v", err)
t.Fatalf("unexpected error from createOrUpdateAffinityAssistantsPerWorkspace: %v", err)
}

// validate StatefulSets from Affinity Assistant
for i, w := range tc.pr.Spec.Workspaces {
if tc.expectStatefulSetSpec != nil {
expectAAName := getAffinityAssistantName(w.Name, tc.pr.Name)
expectAAName := GetAffinityAssistantName(w.Name, tc.pr.Name)
validateStatefulSetSpec(t, ctx, c, expectAAName, tc.expectStatefulSetSpec[i])
}
}

// validate PVCs from VolumeClaimTemplate
if tc.expectedPVCName != "" {
_, err = c.KubeClientSet.CoreV1().PersistentVolumeClaims("").Get(ctx, tc.expectedPVCName, metav1.GetOptions{})
if err != nil {
t.Errorf("unexpected error when retrieving PVC: %v", err)
}
}

// clean up Affinity Assistant
c.cleanupAffinityAssistants(ctx, tc.pr)
if err != nil {
Expand All @@ -267,7 +304,7 @@ func TestCreateAndDeleteOfAffinityAssistantPerWorkspace(t *testing.T) {
continue
}

expectAAName := getAffinityAssistantName(w.Name, tc.pr.Name)
expectAAName := GetAffinityAssistantName(w.Name, tc.pr.Name)
_, err = c.KubeClientSet.AppsV1().StatefulSets(tc.pr.Namespace).Get(ctx, expectAAName, metav1.GetOptions{})
if !apierrors.IsNotFound(err) {
t.Errorf("expected a NotFound response, got: %v", err)
Expand All @@ -277,28 +314,10 @@ func TestCreateAndDeleteOfAffinityAssistantPerWorkspace(t *testing.T) {
}
}

func TestCreateAndDeleteOfAffinityAssistantDisabled_Failure(t *testing.T) {
ctx := context.Background()
c := Reconciler{
KubeClientSet: fakek8s.NewSimpleClientset(),
}

wantErr := fmt.Errorf("unexpected Affinity Assistant behavior %s", aa.AffinityAssistantDisabled)

err := c.createOrUpdateAffinityAssistantsPerAABehavior(ctx, testPRWithPVC, aa.AffinityAssistantDisabled)
if err == nil {
t.Fatalf("expecting error: %v, but got nil", wantErr)
}

if diff := cmp.Diff(wantErr.Error(), err.Error()); diff != "" {
t.Errorf("expected error mismatch: %v", diff)
}
}

// TestCreateAffinityAssistantWhenNodeIsCordoned tests an existing Affinity Assistant can identify the node failure and
// can migrate the affinity assistant pod to a healthy node so that the existing pipelineRun runs to competition
func TestCreateOrUpdateAffinityAssistantWhenNodeIsCordoned(t *testing.T) {
expectedAffinityAssistantName := getAffinityAssistantName(workspacePVCName, testPRWithPVC.Name)
expectedAffinityAssistantName := GetAffinityAssistantName(workspacePVCName, testPRWithPVC.Name)

ss := []*appsv1.StatefulSet{{
TypeMeta: metav1.TypeMeta{
Expand Down Expand Up @@ -398,7 +417,7 @@ func TestCreateOrUpdateAffinityAssistantWhenNodeIsCordoned(t *testing.T) {
return true, &corev1.Pod{}, errors.New("error listing/deleting pod")
})
}
err := c.createOrUpdateAffinityAssistantsPerAABehavior(ctx, testPRWithPVC, aa.AffinityAssistantPerWorkspace)
err := c.createOrUpdateAffinityAssistantsAndPVCs(ctx, testPRWithPVC, aa.AffinityAssistantPerWorkspace)
if !tt.expectedError && err != nil {
t.Errorf("expected no error from createOrUpdateAffinityAssistantsPerWorkspace for the test \"%s\", but got: %v", tt.name, err)
}
Expand Down Expand Up @@ -603,7 +622,7 @@ func TestThatTheAffinityAssistantIsWithoutNodeSelectorAndTolerations(t *testing.
// plus 10 chars for a hash. Labels in Kubernetes can not be longer than 63 chars.
// Typical output from the example below is affinity-assistant-0384086f62
func TestThatAffinityAssistantNameIsNoLongerThan53(t *testing.T) {
affinityAssistantName := getAffinityAssistantName(
affinityAssistantName := GetAffinityAssistantName(
"pipeline-workspace-name-that-is-quite-long",
"pipelinerun-with-a-long-custom-name")

Expand All @@ -628,7 +647,7 @@ func TestCleanupAffinityAssistants_Success(t *testing.T) {
}

// seed data to create StatefulSets and PVCs
expectedAffinityAssistantName := getAffinityAssistantName(workspace.Name, pr.Name)
expectedAffinityAssistantName := GetAffinityAssistantName(workspace.Name, pr.Name)
aa := []*appsv1.StatefulSet{{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
Expand All @@ -642,14 +661,12 @@ func TestCleanupAffinityAssistants_Success(t *testing.T) {
ReadyReplicas: 1,
},
}}

expectedPVCName := getPersistentVolumeClaimNameWithAffinityAssistant(workspace.Name, pr.Name, workspace, *kmeta.NewControllerRef(pr))
pvc := []*corev1.PersistentVolumeClaim{{
ObjectMeta: metav1.ObjectMeta{
Name: expectedPVCName,
}},
}

data := Data{
StatefulSets: aa,
PVCs: pvc,
Expand All @@ -667,9 +684,11 @@ func TestCleanupAffinityAssistants_Success(t *testing.T) {
if !apierrors.IsNotFound(err) {
t.Errorf("expected a NotFound response of StatefulSet, got: %v", err)
}

// the PVCs are NOT expected to be deleted at Affinity Assistant cleanup time
_, err = c.KubeClientSet.CoreV1().PersistentVolumeClaims(pr.Namespace).Get(ctx, expectedPVCName, metav1.GetOptions{})
if !apierrors.IsNotFound(err) {
t.Errorf("expected a NotFound response of PersistentVolumeClaims, got: %v", err)
if err != nil {
t.Errorf("unexpected err when getting PersistentVolumeClaims, err: %v", err)
}
}

Expand All @@ -692,14 +711,9 @@ func TestCleanupAffinityAssistants_Failure(t *testing.T) {
func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
return true, &corev1.NodeList{}, errors.New("error deleting statefulsets")
})
c.KubeClientSet.CoreV1().(*fake.FakeCoreV1).PrependReactor("delete", "persistentvolumeclaims",
func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
return true, &corev1.Pod{}, errors.New("error deleting persistentvolumeclaims")
})

expectedErrs := errorutils.NewAggregate([]error{
errors.New("failed to delete StatefulSet affinity-assistant-e3b0c44298: error deleting statefulsets"),
errors.New("failed to delete PersistentVolumeClaim pvc-e3b0c44298-affinity-assistant-e3b0c44298-0: error deleting persistentvolumeclaims"),
})

errs := c.cleanupAffinityAssistants(ctx, pr)
Expand Down
Loading

0 comments on commit e40338c

Please sign in to comment.