From 79221d8daef789d3982cb7c5bdf8142ef7a0ac35 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 12 Oct 2023 15:26:34 -0400 Subject: [PATCH 001/369] Update go-toolset version for CVEs Signed-off-by: Anish Asthana --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b949b1bd5..df2376625 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-10 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-16 as builder WORKDIR /workspace # Copy the Go Modules manifests @@ -15,7 +15,7 @@ COPY pkg/ pkg/ USER root RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / COPY --from=builder /workspace/manager . From 7c5efc022a43210020f534cb6d38d3c3bfd86c22 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 13 Oct 2023 08:49:41 +0200 Subject: [PATCH 002/369] Adjust the SDK PR link as a reason to skip SDK e2e test --- test/e2e/mnist_raycluster_sdk_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 8a3bc3904..e96110651 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -41,8 +41,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() - // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/271 , remove the skip once SDK with the PR is released - test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/271") + // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251") // Create a namespace namespace := test.NewTestNamespace() From b6c158237ddfd295fd7f6b57bfa1641e579ae63d Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 16 Oct 2023 15:46:43 +0100 Subject: [PATCH 003/369] add ClusterRole and ClusterRoleBinding for KubeRay --- config/rbac/kustomization.yaml | 2 ++ .../rbac/mcad-controller-ray-clusterrole.yaml | 31 +++++++++++++++++++ ...cad-controller-ray-clusterrolebinding.yaml | 12 +++++++ 3 files changed, 45 insertions(+) create mode 100644 config/rbac/mcad-controller-ray-clusterrole.yaml create mode 100644 config/rbac/mcad-controller-ray-clusterrolebinding.yaml diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 05bf4c6cf..cdb307c82 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -11,3 +11,5 @@ resources: - instascale_role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +- mcad-controller-ray-clusterrolebinding.yaml +- mcad-controller-ray-clusterrole.yaml diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml new file mode 100644 index 000000000..f95624208 --- /dev/null +++ b/config/rbac/mcad-controller-ray-clusterrole.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mcad-controller-ray-clusterrole +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + - rayclusters/finalizers + - rayclusters/status + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml new file mode 100644 index 000000000..a3931da07 --- /dev/null +++ b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mcad-controller-ray-clusterrolebinding +subjects: + - kind: ServiceAccount + name: codeflare-operator-controller-manager + namespace: openshift-operators +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mcad-controller-ray-clusterrole From 4f21b11efe291699cea6ef1091b88b6695b4ab57 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 6 Oct 2023 13:44:01 +0100 Subject: [PATCH 004/369] Test Instascale machine set functionality Co-authored-by: Karel Suta ksuta@redhat.com Co-authored-by: Fiona Waters fwaters@redhat.com --- test/e2e/instascale_machineset_test.go | 54 ++++++++++++++++++++++++++ test/support/client.go | 13 +++++++ test/support/machinesets.go | 36 +++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 test/e2e/instascale_machineset_test.go create mode 100644 test/support/machinesets.go diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go new file mode 100644 index 000000000..08805555f --- /dev/null +++ b/test/e2e/instascale_machineset_test.go @@ -0,0 +1,54 @@ +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +func TestInstascaleMachineSet(t *testing.T) { + test := With(t) + test.T().Parallel() + + // skip test if not using machine sets + if !MachineSetsExist(test) { + test.T().Skip("Skipping test as machine sets don't exist") + } + + namespace := test.NewTestNamespace() + + // Test configuration + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + // look for machine set with aw name - expect to find it + test.Expect(MachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + // look for machine set replica - expect not to find it + test.Expect(MachineExists(test)).Should(BeFalse()) + + // // Setup batch job and AppWrapper + _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + test.Expect(err).NotTo(HaveOccurred()) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + //look for machine set replica - expect to find it + test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeTrue()) + + // assert that the AppWrapper goes to "Completed" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // look for machine set replica - expect not to find it + test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeFalse()) + +} diff --git a/test/support/client.go b/test/support/client.go index f5696bde8..d911c7c2c 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/clientcmd" imagev1 "github.com/openshift/client-go/image/clientset/versioned" + machinev1 "github.com/openshift/client-go/machine/clientset/versioned" routev1 "github.com/openshift/client-go/route/clientset/versioned" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -33,6 +34,7 @@ import ( type Client interface { Core() kubernetes.Interface + Machine() machinev1.Interface Route() routev1.Interface Image() imagev1.Interface MCAD() mcadclient.Interface @@ -42,6 +44,7 @@ type Client interface { type testClient struct { core kubernetes.Interface + machine machinev1.Interface route routev1.Interface image imagev1.Interface mcad mcadclient.Interface @@ -55,6 +58,10 @@ func (t *testClient) Core() kubernetes.Interface { return t.core } +func (t *testClient) Machine() machinev1.Interface { + return t.machine +} + func (t *testClient) Route() routev1.Interface { return t.route } @@ -88,6 +95,11 @@ func newTestClient() (Client, error) { return nil, err } + machineClient, err := machinev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + routeClient, err := routev1.NewForConfig(cfg) if err != nil { return nil, err @@ -115,6 +127,7 @@ func newTestClient() (Client, error) { return &testClient{ core: kubeClient, + machine: machineClient, route: routeClient, image: imageClient, mcad: mcadClient, diff --git a/test/support/machinesets.go b/test/support/machinesets.go new file mode 100644 index 000000000..b45455360 --- /dev/null +++ b/test/support/machinesets.go @@ -0,0 +1,36 @@ +package support + +import ( + "github.com/onsi/gomega" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1beta1 "github.com/openshift/api/machine/v1beta1" +) + +func MachineSets(t Test) ([]v1beta1.MachineSet, error) { + ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), v1.ListOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms.Items, err +} + +func MachineExists(t Test) (foundReplica bool) { + machineSetList, err := MachineSets(t) + t.Expect(err).NotTo(gomega.HaveOccurred()) + for _, ms := range machineSetList { + if ms.Name == "test-instascale" && &ms.Status.AvailableReplicas == Ptr(int32(1)) { + foundReplica = true + } + } + return foundReplica +} + +func MachineSetId(machineSet v1beta1.MachineSet) string { + return machineSet.Name +} + +func MachineSetsExist(t Test) bool { + ms, err := MachineSets(t) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms != nil +} From 55be710877db3862f29bd11e0d64f93a0311fa9e Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 11 Oct 2023 10:17:53 +0100 Subject: [PATCH 005/369] catching 404 error when machine sets not in use --- test/e2e/instascale_machineset_test.go | 5 ++++- test/support/machinesets.go | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 08805555f..0a1975a48 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -14,7 +16,8 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - if !MachineSetsExist(test) { + ms, err := MachineSetsExist(test) + if !ms || err != nil && errors.IsNotFound(err) { test.T().Skip("Skipping test as machine sets don't exist") } diff --git a/test/support/machinesets.go b/test/support/machinesets.go index b45455360..85259536c 100644 --- a/test/support/machinesets.go +++ b/test/support/machinesets.go @@ -3,6 +3,7 @@ package support import ( "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1beta1 "github.com/openshift/api/machine/v1beta1" @@ -29,8 +30,11 @@ func MachineSetId(machineSet v1beta1.MachineSet) string { return machineSet.Name } -func MachineSetsExist(t Test) bool { +func MachineSetsExist(t Test) (bool, error) { ms, err := MachineSets(t) + if err != nil && errors.IsNotFound(err) { + return false, err + } t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil + return ms != nil, err } From 4d1f8ec8d44b5dd81c79d910823b61986f637334 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 11 Oct 2023 17:52:30 +0200 Subject: [PATCH 006/369] Use generic function to retrieve machines --- test/e2e/instascale_machineset_test.go | 16 +++++----- test/support/machine.go | 42 ++++++++++++++++++++++++++ test/support/machinesets.go | 40 ------------------------ 3 files changed, 50 insertions(+), 48 deletions(-) create mode 100644 test/support/machine.go delete mode 100644 test/support/machinesets.go diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 0a1975a48..df79aec2f 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -32,9 +32,9 @@ func TestInstascaleMachineSet(t *testing.T) { }) // look for machine set with aw name - expect to find it - test.Expect(MachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) - // look for machine set replica - expect not to find it - test.Expect(MachineExists(test)).Should(BeFalse()) + test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + // look for machine belonging to the machine set, there should be none + test.Expect(GetMachines(test, "test-instascale")).Should(BeEmpty()) // // Setup batch job and AppWrapper _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) @@ -44,14 +44,14 @@ func TestInstascaleMachineSet(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - //look for machine set replica - expect to find it - test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeTrue()) + // look for machine belonging to the machine set - expect to find it + test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(HaveLen(1)) // assert that the AppWrapper goes to "Completed" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for machine set replica - expect not to find it - test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeFalse()) + // look for machine belonging to the machine set - there should be none + test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(BeEmpty()) } diff --git a/test/support/machine.go b/test/support/machine.go new file mode 100644 index 000000000..f6be3750d --- /dev/null +++ b/test/support/machine.go @@ -0,0 +1,42 @@ +package support + +import ( + "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + machinev1beta1 "github.com/openshift/api/machine/v1beta1" +) + +func GetMachineSets(t Test) ([]machinev1beta1.MachineSet, error) { + ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms.Items, err +} + +func Machines(t Test, machineSetName string) func(g gomega.Gomega) []machinev1beta1.Machine { + return func(g gomega.Gomega) []machinev1beta1.Machine { + machine, err := t.Client().Machine().MachineV1beta1().Machines("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machineset=" + machineSetName}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return machine.Items + } +} + +func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { + t.T().Helper() + return Machines(t, machineSetName)(t) +} + +func MachineSetId(machineSet machinev1beta1.MachineSet) string { + return machineSet.Name +} + +func MachineSetsExist(t Test) (bool, error) { + ms, err := GetMachineSets(t) + if err != nil && errors.IsNotFound(err) { + return false, err + } + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms != nil, err +} diff --git a/test/support/machinesets.go b/test/support/machinesets.go deleted file mode 100644 index 85259536c..000000000 --- a/test/support/machinesets.go +++ /dev/null @@ -1,40 +0,0 @@ -package support - -import ( - "github.com/onsi/gomega" - - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - v1beta1 "github.com/openshift/api/machine/v1beta1" -) - -func MachineSets(t Test) ([]v1beta1.MachineSet, error) { - ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), v1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms.Items, err -} - -func MachineExists(t Test) (foundReplica bool) { - machineSetList, err := MachineSets(t) - t.Expect(err).NotTo(gomega.HaveOccurred()) - for _, ms := range machineSetList { - if ms.Name == "test-instascale" && &ms.Status.AvailableReplicas == Ptr(int32(1)) { - foundReplica = true - } - } - return foundReplica -} - -func MachineSetId(machineSet v1beta1.MachineSet) string { - return machineSet.Name -} - -func MachineSetsExist(t Test) (bool, error) { - ms, err := MachineSets(t) - if err != nil && errors.IsNotFound(err) { - return false, err - } - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil, err -} From 24a400f82e28210e4dbdba3a3490f5947f9415b0 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 11:41:42 +0100 Subject: [PATCH 007/369] determining cluster type --- test/e2e/instascale_machineset_test.go | 8 +++---- test/support/environment.go | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index df79aec2f..aef1dfe7a 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,8 +6,6 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" - . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -16,9 +14,9 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - ms, err := MachineSetsExist(test) - if !ms || err != nil && errors.IsNotFound(err) { - test.T().Skip("Skipping test as machine sets don't exist") + clusterType := DetermineClusterType() + if clusterType != OcpCluster { + test.T().Skip("Skipping test as not running on an OCP cluster") } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index 540749823..2455603af 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -38,6 +38,15 @@ const ( // Cluster ID for OSD cluster used in tests, used for testing InstaScale OsdClusterID = "CLUSTERID" + + // Type of cluster test is run on + ClusterType = "CLUSTER_TYPE" + // OpenShift Dedicated Cluster + OsdCluster = "OSD" + // OpenShift Container Platform Cluster + OcpCluster = "OCP" + // ROSA Hosted Hypershift Cluster + HypershiftCluster = "HYPERSHIFT" ) func GetCodeFlareSDKVersion() string { @@ -65,6 +74,10 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } +func GetClusterType() (string, bool) { + return os.LookupEnv(ClusterType) +} + func IsOsd() bool { osdClusterId, found := GetOsdClusterId() if found && osdClusterId != "" { @@ -79,3 +92,20 @@ func lookupEnvOrDefault(key, value string) string { } return value } + +func DetermineClusterType() string { + clusterType, ok := GetClusterType() + if !ok { + return "" + } + switch clusterType { + case "OSD": + return OsdCluster + case "OCP": + return OcpCluster + case "HYPERSHIFT": + return HypershiftCluster + default: + return "" + } +} From b5f8b04fc864758af8be55aea255d80fc977170d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 12 Oct 2023 16:00:50 +0200 Subject: [PATCH 008/369] Use enum for cluster type in e2e tests --- test/e2e/instascale_machineset_test.go | 4 +- test/support/environment.go | 53 +++++++++++++------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index aef1dfe7a..8e23d4e7c 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -14,9 +14,9 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - clusterType := DetermineClusterType() + clusterType := GetClusterType(test) if clusterType != OcpCluster { - test.T().Skip("Skipping test as not running on an OCP cluster") + test.T().Skipf("Skipping test as not running on an OCP cluster, resolved cluster type: %s", clusterType) } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index 2455603af..cdafca6e9 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -40,13 +40,16 @@ const ( OsdClusterID = "CLUSTERID" // Type of cluster test is run on - ClusterType = "CLUSTER_TYPE" - // OpenShift Dedicated Cluster - OsdCluster = "OSD" - // OpenShift Container Platform Cluster - OcpCluster = "OCP" - // ROSA Hosted Hypershift Cluster - HypershiftCluster = "HYPERSHIFT" + ClusterTypeEnvVar = "CLUSTER_TYPE" +) + +type ClusterType string + +const ( + OsdCluster ClusterType = "OSD" + OcpCluster ClusterType = "OCP" + HypershiftCluster ClusterType = "HYPERSHIFT" + UndefinedCluster ClusterType = "UNDEFINED" ) func GetCodeFlareSDKVersion() string { @@ -74,8 +77,23 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } -func GetClusterType() (string, bool) { - return os.LookupEnv(ClusterType) +func GetClusterType(t Test) ClusterType { + clusterType, ok := os.LookupEnv(ClusterTypeEnvVar) + if !ok { + t.T().Logf("Expected environment variable %s not found, cluster type is not defined.", ClusterTypeEnvVar) + return UndefinedCluster + } + switch clusterType { + case "OSD": + return OsdCluster + case "OCP": + return OcpCluster + case "HYPERSHIFT": + return HypershiftCluster + default: + t.T().Logf("Expected environment variable %s contains unexpected value: '%s'", ClusterTypeEnvVar, clusterType) + return UndefinedCluster + } } func IsOsd() bool { @@ -92,20 +110,3 @@ func lookupEnvOrDefault(key, value string) string { } return value } - -func DetermineClusterType() string { - clusterType, ok := GetClusterType() - if !ok { - return "" - } - switch clusterType { - case "OSD": - return OsdCluster - case "OCP": - return OcpCluster - case "HYPERSHIFT": - return HypershiftCluster - default: - return "" - } -} From 90a0bbf16f33e041a400bd49469208bfd595a48f Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 15:20:14 +0100 Subject: [PATCH 009/369] updating machinepool test skip --- test/e2e/instascale_machinepool_test.go | 5 +++-- test/support/environment.go | 8 -------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index b6ad0782c..4c2cbab58 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -29,8 +29,9 @@ func TestInstascaleMachinePool(t *testing.T) { test := With(t) test.T().Parallel() - if !IsOsd() { - test.T().Skip("Skipping test as not running on an OSD cluster") + clusterType := GetClusterType(test) + if clusterType != OsdCluster { + test.T().Skipf("Skipping test as not running on an OSD cluster, resolved cluster type: %s", clusterType) } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index cdafca6e9..7d8e6bc80 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -96,14 +96,6 @@ func GetClusterType(t Test) ClusterType { } } -func IsOsd() bool { - osdClusterId, found := GetOsdClusterId() - if found && osdClusterId != "" { - return true - } - return false -} - func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v From da0cf86e8847037746d06d810e0c5cf153889e08 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 15:35:01 +0100 Subject: [PATCH 010/369] removing unused function --- test/support/machine.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/support/machine.go b/test/support/machine.go index f6be3750d..26acb4bce 100644 --- a/test/support/machine.go +++ b/test/support/machine.go @@ -3,7 +3,6 @@ package support import ( "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" machinev1beta1 "github.com/openshift/api/machine/v1beta1" @@ -31,12 +30,3 @@ func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { func MachineSetId(machineSet machinev1beta1.MachineSet) string { return machineSet.Name } - -func MachineSetsExist(t Test) (bool, error) { - ms, err := GetMachineSets(t) - if err != nil && errors.IsNotFound(err) { - return false, err - } - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil, err -} From ea777c6416da985ddab3fcf6467d6d8ca806d850 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 16 Oct 2023 12:53:18 +0100 Subject: [PATCH 011/369] refactor to use aw.Name --- test/e2e/instascale_app_wrapper.go | 8 +++----- test/e2e/instascale_machinepool_test.go | 7 ++++++- test/e2e/instascale_machineset_test.go | 17 +++++++++++------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index f9ec3bd0c..805df1bc3 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -27,7 +27,7 @@ import ( . "github.com/project-codeflare/codeflare-operator/test/support" ) -func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) { +func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { // Batch Job job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -89,7 +89,7 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - // create an appwrapper + // AppWrapper aw := &mcadv1beta1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ Name: "test-instascale", @@ -136,7 +136,5 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - - return job, aw, err + return aw } diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 4c2cbab58..386b224db 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -22,6 +22,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -54,7 +56,10 @@ func TestInstascaleMachinePool(t *testing.T) { ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) // Setup batch job and AppWrapper - _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 8e23d4e7c..fad7717b2 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -29,13 +31,16 @@ func TestInstascaleMachineSet(t *testing.T) { "mnist.py": ReadFile(test, "mnist.py"), }) + // // Setup batch job and AppWrapper + aw := instaScaleJobAppWrapper(test, namespace, cm) + // look for machine set with aw name - expect to find it - test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal(aw.Name)))) // look for machine belonging to the machine set, there should be none - test.Expect(GetMachines(test, "test-instascale")).Should(BeEmpty()) + test.Expect(GetMachines(test, aw.Name)).Should(BeEmpty()) - // // Setup batch job and AppWrapper - _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) // assert that AppWrapper goes to "Running" state @@ -43,13 +48,13 @@ func TestInstascaleMachineSet(t *testing.T) { Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) // look for machine belonging to the machine set - expect to find it - test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(HaveLen(1)) + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(HaveLen(1)) // assert that the AppWrapper goes to "Completed" state test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) // look for machine belonging to the machine set - there should be none - test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(BeEmpty()) + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(BeEmpty()) } From 5a0bb174966ac2530ce44f4dd54a8ea371c3bd88 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 17 Oct 2023 15:50:34 +0100 Subject: [PATCH 012/369] Replaced rayv1alpha1 with rayv1 --- test/e2e/mnist_raycluster_sdk_test.go | 4 ++-- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 24 +++++++++---------- test/support/ray.go | 22 ++++++++--------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index e96110651..29643fe67 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -102,7 +102,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, { Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + APIGroups: []string{rayv1.GroupVersion.Group}, Resources: []string{"rayclusters", "rayclusters/status"}, }, { diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 7752d2eab..a7956f66e 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -22,7 +22,7 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -60,18 +60,18 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // RayCluster - rayCluster := &rayv1alpha1.RayCluster{ + rayCluster := &rayv1.RayCluster{ TypeMeta: metav1.TypeMeta{ - APIVersion: rayv1alpha1.GroupVersion.String(), + APIVersion: rayv1.GroupVersion.String(), Kind: "RayCluster", }, ObjectMeta: metav1.ObjectMeta{ Name: "raycluster", Namespace: namespace.Name, }, - Spec: rayv1alpha1.RayClusterSpec{ + Spec: rayv1.RayClusterSpec{ RayVersion: GetRayVersion(), - HeadGroupSpec: rayv1alpha1.HeadGroupSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ RayStartParams: map[string]string{ "dashboard-host": "0.0.0.0", }, @@ -135,7 +135,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, }, }, - WorkerGroupSpecs: []rayv1alpha1.WorkerGroupSpec{ + WorkerGroupSpecs: []rayv1.WorkerGroupSpec{ { Replicas: Ptr(int32(1)), MinReplicas: Ptr(int32(1)), @@ -220,16 +220,16 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - rayJob := &rayv1alpha1.RayJob{ + rayJob := &rayv1.RayJob{ TypeMeta: metav1.TypeMeta{ - APIVersion: rayv1alpha1.GroupVersion.String(), + APIVersion: rayv1.GroupVersion.String(), Kind: "RayJob", }, ObjectMeta: metav1.ObjectMeta{ Name: "mnist", Namespace: namespace.Name, }, - Spec: rayv1alpha1.RayJobSpec{ + Spec: rayv1.RayJobSpec{ Entrypoint: "python /home/ray/jobs/mnist.py", RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` { @@ -248,7 +248,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { ShutdownAfterJobFinishes: false, }, } - rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) @@ -262,9 +262,9 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). - Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) + Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). - To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) + To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) } diff --git a/test/support/ray.go b/test/support/ray.go index b69609aef..f18edaa03 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -18,27 +18,27 @@ package support import ( "github.com/onsi/gomega" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const RayJobDefaultClusterSelectorKey = "ray.io/cluster" -func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { - return func(g gomega.Gomega) *rayv1alpha1.RayJob { - job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) +func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayJob { + return func(g gomega.Gomega) *rayv1.RayJob { + job, err := t.Client().Ray().RayV1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return job } } -func GetRayJob(t Test, namespace, name string) *rayv1alpha1.RayJob { +func GetRayJob(t Test, namespace, name string) *rayv1.RayJob { t.T().Helper() return RayJob(t, namespace, name)(t) } -func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { +func RayJobStatus(job *rayv1.RayJob) rayv1.JobStatus { return job.Status.JobStatus } @@ -48,20 +48,20 @@ func GetRayJobId(t Test, namespace, name string) string { return job.Status.JobId } -func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayCluster { - return func(g gomega.Gomega) *rayv1alpha1.RayCluster { - cluster, err := t.Client().Ray().RayV1alpha1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) +func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayCluster { + return func(g gomega.Gomega) *rayv1.RayCluster { + cluster, err := t.Client().Ray().RayV1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return cluster } } -func GetRayCluster(t Test, namespace, name string) *rayv1alpha1.RayCluster { +func GetRayCluster(t Test, namespace, name string) *rayv1.RayCluster { t.T().Helper() return RayCluster(t, namespace, name)(t) } -func RayClusterState(cluster *rayv1alpha1.RayCluster) rayv1alpha1.ClusterState { +func RayClusterState(cluster *rayv1.RayCluster) rayv1.ClusterState { return cluster.Status.State } From 523d8e9fff5f89555061e91bd5addb69157371b3 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 17 Oct 2023 16:02:01 +0100 Subject: [PATCH 013/369] Added missing go files --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9166e25a2..08f3cb317 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/instascale v0.1.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36 + github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index f5d91a72c..2c53de1ce 100644 --- a/go.sum +++ b/go.sum @@ -420,8 +420,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36 h1:IzXHzI4OjL1ySpvmPHTWjFjmgBBjAZXBa3cQwejLPWY= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 h1:s+/WzpqoFot/fgo1uq7BLgeiOy3HZuT+UVtCb3h7R+I= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= From 06d724f053ba38b3c4e7d1f3e59868a293c9b17f Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 17 Oct 2023 16:08:16 +0100 Subject: [PATCH 014/369] Updated KubeRay version in MakeFile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 725b661aa..4dcd3618f 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v0.6.0 +KUBERAY_VERSION ?= v1.0.0-rc.1 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 From 86c12525b331eacf1d967aabb5bbb49f190e775c Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Tue, 17 Oct 2023 17:12:46 +0100 Subject: [PATCH 015/369] fix: update ubi image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index df2376625..5c2df0129 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-16 as builder +FROM registry.access.redhat.io/ubi8/go-toolset:1.19 as builder WORKDIR /workspace # Copy the Go Modules manifests From 1268065a9b2f005f610ad1981ad4b4f5689e1f84 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Wed, 18 Oct 2023 10:18:40 +0100 Subject: [PATCH 016/369] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c2df0129..72dfe6e7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.io/ubi8/go-toolset:1.19 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.19 as builder WORKDIR /workspace # Copy the Go Modules manifests From 4b65c07d79f586557aac317f706fcc560b8fbeb7 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 19 Oct 2023 16:19:56 +0100 Subject: [PATCH 017/369] Updated MCAD controller cluster role for ingresses --- config/rbac/mcad-controller-ray-clusterrole.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml index f95624208..39ae487d4 100644 --- a/config/rbac/mcad-controller-ray-clusterrole.yaml +++ b/config/rbac/mcad-controller-ray-clusterrole.yaml @@ -18,9 +18,9 @@ rules: - patch - delete - apiGroups: - - route.openshift.io + - networking.k8s.io resources: - - routes + - ingresses verbs: - create - delete From e10fae85668fd73991cd09ce39f636d80f18f637 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 19 Oct 2023 17:00:17 +0100 Subject: [PATCH 018/369] Re added routes to cluster role --- config/rbac/mcad-controller-ray-clusterrole.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml index 39ae487d4..18e3d98f3 100644 --- a/config/rbac/mcad-controller-ray-clusterrole.yaml +++ b/config/rbac/mcad-controller-ray-clusterrole.yaml @@ -29,3 +29,15 @@ rules: - patch - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch From 61ebcde8a5e70d0f33d705cbf7e6cb8b1f2a4b58 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Tue, 17 Oct 2023 12:03:37 +0100 Subject: [PATCH 019/369] ci: this calls the sync-fork workflow in the odh-cfo --- .github/workflows/odh-fork-sync.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/odh-fork-sync.yml diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml new file mode 100644 index 000000000..72d4a3661 --- /dev/null +++ b/.github/workflows/odh-fork-sync.yml @@ -0,0 +1,15 @@ +name: Call sync on OpenDataHub CFO fork sync +on: + release: + types: [published] + workflow_dispatch: + +jobs: + sync-fork: + runs-on: ubuntu-latest + steps: + - name: Sync-fork + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + run: | + gh workflow run sync-fork.yaml -R github.com/opendatahub-io/codeflare-operator -r main From 4b863f79d4b704c511ea404e27e3ec65d66b5a0f Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 20 Oct 2023 16:16:24 +0100 Subject: [PATCH 020/369] Copying MCAD CRDs in Kustomize config instead of referencing them (#357) * copying mcad crds in kustomize config instead of referencing them * addressing feedback * build: split crds into their own files * refactor: isntall yq if not exists * updating kustomization.yaml and verify_generated_files workflow * remove restore * adding crds * update check_yq target * Update Makefile * Update Makefile * Update Makefile * commenting out lines --------- Co-authored-by: Dimitri Saridakis --- .github/workflows/verify_generated_files.yml | 2 + Makefile | 39 +- config/crd/kustomization.yaml | 5 +- config/crd/mcad/kustomization.yaml | 2 +- config/crd/quota.codeflare_quotasubtrees.yaml | 122 +++ .../crd/workload.codeflare_appwrappers.yaml | 765 ++++++++++++++++++ .../workload.codeflare_schedulingspecs.yaml | 102 +++ 7 files changed, 1030 insertions(+), 7 deletions(-) create mode 100644 config/crd/quota.codeflare_quotasubtrees.yaml create mode 100644 config/crd/workload.codeflare_appwrappers.yaml create mode 100644 config/crd/workload.codeflare_schedulingspecs.yaml diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index aa85fcff5..b99a9a060 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -7,6 +7,7 @@ on: - '**.go' - '**go.mod' - '**go.sum' + - 'config/**' tags-ignore: - 'v*' pull_request: @@ -14,6 +15,7 @@ on: - '**.go' - '**go.mod' - '**go.sum' + - 'config/**' jobs: verify-imports: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 4dcd3618f..00de9c196 100644 --- a/Makefile +++ b/Makefile @@ -146,9 +146,12 @@ defaults: gofmt -w $(DEFAULTS_TEST_FILE) +# this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests -manifests: controller-gen ## Generate RBAC objects. +manifests: controller-gen kustomize ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." +# $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml +# $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml && make split_yaml FILE=config/crd/mcad.yaml .PHONY: fmt fmt: ## Run go fmt against code. @@ -192,19 +195,16 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl apply -f - git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/${ENV} | kubectl apply -f - git restore config/* @@ -224,6 +224,7 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize +YQ ?= $(LOCALBIN)/yq CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports @@ -284,7 +285,6 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion @@ -388,3 +388,32 @@ verify-imports: openshift-goimports ## Run import verifications. .PHONY: scorecard-bundle scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. $(OPERATOR_SDK) scorecard bundle + + +FILE ?= input.yaml # Default value, it isn't a file, but the make cmds fill hang for longer without it +temp_dir := temp_split +output_dir := 'config/crd/' + +.PHONY: check_yq +check_yq: + @command -v wget >/dev/null 2>&1 || (echo "Installing wget..."; apt-get install -y wget) + @command -v $(YQ) >/dev/null 2>&1 || (echo "Installing yq..."; wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64.tar.gz -O - |\ + tar xz && mv yq_linux_amd64 $(YQ)) + +# this works on a MacOS by replacing awk with gawk +.PHONY: split_yaml +split_yaml: + @$(MAKE) check_yq + @mkdir -p $(temp_dir) + @awk '/apiVersion: /{if (x>0) close("$(temp_dir)/section_" x ".yaml"); x++}{print > "$(temp_dir)/section_"x".yaml"}' $(FILE) + @$(MAKE) process_sections + +.PHONY: process_sections +process_sections: + @mkdir -p $(output_dir) + @for section_file in $(temp_dir)/section_*; do \ + metadata_name=$$(YQ e '.metadata.name' $$section_file); \ + file_name=$$(echo $$metadata_name | awk -F'.' '{print $$2"."$$3"_"$$1".yaml"}'); \ + mv $$section_file $(output_dir)/$$file_name; \ + done + @rm -r $(temp_dir) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 81d71c1e1..73a576238 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,5 +2,8 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- mcad +- quota.codeflare_quotasubtrees.yaml +- workload.codeflare_appwrappers.yaml +- workload.codeflare_schedulingspecs.yaml + #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 8ea96b055..e016d77f8 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v0.0.0 # kpt-set: ${MCAD_CRD} +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.36.0 diff --git a/config/crd/quota.codeflare_quotasubtrees.yaml b/config/crd/quota.codeflare_quotasubtrees.yaml new file mode 100644 index 000000000..44bb5e0a6 --- /dev/null +++ b/config/crd/quota.codeflare_quotasubtrees.yaml @@ -0,0 +1,122 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.quota.codeflare.dev +spec: + group: quota.codeflare.dev + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true +--- diff --git a/config/crd/workload.codeflare_appwrappers.yaml b/config/crd/workload.codeflare_appwrappers.yaml new file mode 100644 index 000000000..e1d260411 --- /dev/null +++ b/config/crd/workload.codeflare_appwrappers.yaml @@ -0,0 +1,765 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: appwrappers.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: AppWrapper + listKind: AppWrapperList + plural: appwrappers + singular: appwrapper + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Definition of AppWrapper class + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppWrapperSpec describes how the App Wrapper will look like. + properties: + priority: + format: int32 + type: integer + priorityslope: + format: float + type: number + resources: + description: a collection of AppWrapperResource + properties: + GenericItems: + items: + description: AppWrapperGenericResource is App Wrapper aggregation + resource + properties: + allocated: + description: The number of allocated replicas from this + resource type + format: int32 + type: integer + completionstatus: + description: 'Optional field that drives completion status + of this AppWrapper. This field within an item of an AppWrapper + determines the full state of the AppWrapper. The completionstatus + field contains a list of conditions that make the associate + item considered completed, for instance: - completion + conditions could be "Complete" or "Failed". The associated + item''s level .status.conditions[].type field is monitored + for any one of these conditions. Once all items with this + option is set and the conditionstatus is met the entire + AppWrapper state will be changed to one of the valid AppWrapper + completion state. Note: - this is an AND operation for + all items where this option is set. See the list of AppWrapper + states for a list of valid complete states.' + type: string + custompodresources: + description: Optional section that specifies resource requirements + for non-standard k8s resources, follows same format as + that of standard k8s resources. + items: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, + quantity) pairs. + type: object + replicas: + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'todo: replace with Containers []Container + Contain v1.ResourceRequirements' + type: object + required: + - replicas + - requests + type: object + type: array + generictemplate: + description: The template for the resource; it is now a + raw text because we don't know for what resource it should + be instantiated + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + minavailable: + description: The minimal available pods to run for this + AppWrapper; the default value is nil + format: int32 + type: integer + priority: + description: The priority of this resource + format: int32 + type: integer + priorityslope: + description: The increasing rate of priority value for this + resource + format: float + type: number + replicas: + description: Replicas is the number of desired replicas + format: int32 + type: integer + type: object + type: array + type: object + schedulingSpec: + description: SchedSpec specifies the parameters used for scheduling + generic items wrapped inside AppWrappers. It defines the policy + for requeuing jobs based on the number of running pods. + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed + state. Requeuing is triggered when the number of running/completed + pods is not equal to this value. When not specified, requeuing + is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on + waiting time. Values in this field control how often the pod + check should happen, and if requeuing has reached its maximum + number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time + between requeuing checks. The values available are 'exponential', + 'linear', or 'none'. For example, 'exponential' growth would + double the 'timeInSeconds' value every time a requeuing + event is triggered. If the string value is misspelled or + not one of the possible options, the growth behavior is + defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. + Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing + event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it + may grow as requeuing events happen. + type: integer + type: object + type: object + selector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + service: + description: AppWrapperService is App Wrapper service definition + properties: + spec: + description: ServiceSpec describes the attributes that a user + creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if NodePorts + will be automatically allocated for services with type LoadBalancer. Default + is "true". It may be set to "false" if the cluster load-balancer + does not rely on NodePorts. If the caller requests specific + NodePorts (by specifying a value), those requests will be + respected, regardless of this field. This field may only + be set for services with type LoadBalancer and will be cleared + if the type is changed to any other type. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service and + is usually assigned randomly. If an address is specified + manually, is in-range (as per system configuration), and + is not in use, it will be allocated to the service; otherwise + creation of the service will fail. This field may not be + changed through updates unless the type field is also being + changed to ExternalName (which requires this field to be + blank) or the type field is being changed from ExternalName + (in which case this field may optionally be specified, as + describe above). Valid values are "None", empty string + (""), or a valid IP address. Setting this to "None" makes + a "headless service" (no virtual IP), which is useful when + direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, + and LoadBalancer. If this field is specified when creating + a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned + to this service, and are usually assigned randomly. If + an address is specified manually, is in-range (as per system + configuration), and is not in use, it will be allocated + to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the + type field is also being changed to ExternalName (which + requires this field to be empty) or the type field is being + changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values + are \"None\", empty string (\"\"), or a valid IP address. + \ Setting this to \"None\" makes a \"headless service\" + (no virtual IP), which is useful when direct endpoint connections + are preferred and proxying is not required. Only applies + to types ClusterIP, NodePort, and LoadBalancer. If this + field is specified when creating a Service of type ExternalName, + creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, + it will be initialized from the clusterIP field. If this + field is specified, clients must ensure that clusterIPs[0] + and clusterIP have the same value. \n This field may hold + a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies + field. Both clusterIPs and ipFamilies are governed by the + ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this service. These + IPs are not managed by Kubernetes. The user is responsible + for ensuring that traffic arrives at a node with this IP. A + common example is external load-balancers that are not part + of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that discovery + mechanisms will return as an alias for this service (e.g. + a DNS CNAME record). No proxying will be involved. Must + be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes distribute + service traffic they receive on one of the Service's "externally-facing" + addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). + If set to "Local", the proxy will configure the service + in a way that assumes that external load balancers will + take care of balancing the service traffic between nodes, + and so each node will deliver traffic only to the node-local + endpoints of the service, without masquerading the client + source IP. (Traffic mistakenly sent to a node with no endpoints + will be dropped.) The default value, "Cluster", uses the + standard behavior of routing to all endpoints evenly (possibly + modified by topology and other features). Note that traffic + sent to an External IP or LoadBalancer IP from within the + cluster will always get "Cluster" semantics, but clients + sending to a NodePort from within the cluster may need to + take traffic policy into account when picking a node. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. This only applies when type is + set to LoadBalancer and externalTrafficPolicy is set to + Local. If a value is specified, is in-range, and is not + in use, it will be used. If not specified, a value will + be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints + for this service or not. If this field is specified when + creating a Service which does not need it, creation will + fail. This field will be wiped when updating a Service to + no longer need it (e.g. changing type). This field cannot + be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes distribute + service traffic they receive on the ClusterIP. If set to + "Local", the proxy will assume that pods only want to talk + to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The + default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology + and other features). + type: string + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. IPv4, + IPv6) assigned to this service. This field is usually assigned + automatically based on cluster configuration and the ipFamilyPolicy + field. If this field is specified manually, the requested + family is available in the cluster, and ipFamilyPolicy allows + it, it will be used; otherwise creation of the service will + fail. This field is conditionally mutable: it allows for + adding or removing a secondary IP family, but it does not + allow changing the primary IP family of the Service. Valid + values are \"IPv4\" and \"IPv6\". This field only applies + to Services of types ClusterIP, NodePort, and LoadBalancer, + and does apply to \"headless\" services. This field will + be wiped when updating a Service to type ExternalName. \n + This field may hold a maximum of two entries (dual-stack + families, in either order). These families must correspond + to the values of the clusterIPs field, if specified. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy + field." + items: + description: IPFamily represents the IP Family (IPv4 or + IPv6). This type is used to express the family of an IP + expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by this Service. If there is no value + provided, then this field will be set to SingleStack. Services + can be "SingleStack" (a single IP family), "PreferDualStack" + (two IP families on dual-stack configured clusters or a + single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise + fail). The ipFamilies and clusterIPs fields depend on the + value of this field. This field will be wiped when updating + a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load balancer + implementation this Service belongs to. If specified, the + value of this field must be a label-style identifier, with + an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". + Unprefixed names are reserved for end-users. This field + can only be set when the Service type is 'LoadBalancer'. + If not set, the default load balancer implementation is + used, today this is typically done through the cloud provider + integration, but should apply for any default implementation. + If set, it is assumed that a load balancer implementation + is watching for Services with a matching class. Any default + load balancer implementation (e.g. cloud providers) should + ignore Services that set this field. This field can only + be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped + when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider + supports specifying the loadBalancerIP when a load balancer + is created. This field will be ignored if the cloud-provider + does not support the feature. Deprecated: This field was + under-specified and its meaning varies across implementations, + and it cannot support dual-stack. As of Kubernetes v1.24, + users are encouraged to use implementation-specific annotations + when available. This field may be removed in a future API + version.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider load-balancer + will be restricted to the specified client IPs. This field + will be ignored if the cloud-provider does not support the + feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + appProtocol: + description: The application protocol for this port. + This field follows standard Kubernetes label syntax. + Un-prefixed names are reserved for IANA standard service + names (as per RFC-6335 and https://www.iana.org/assignments/service-names). + Non-standard protocols should use prefixed names such + as mycompany.com/my-custom-protocol. + type: string + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field in + the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this service + is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, in-range, + and not in use it will be used, otherwise the operation + will fail. If not specified, a port will be allocated + if this Service requires one. If this field is specified + when creating a Service which does not need it, creation + will fail. This field will be wiped when updating + a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access on + the pods targeted by the service. Number must be in + the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored for + services with clusterIP=None, and should be omitted + or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any agent + which deals with endpoints for this Service should disregard + any indications of ready/not-ready. The primary use case + for setting this field is for a StatefulSet's Headless Service + to propagate SRV DNS records for its Pods for the purpose + of peer discovery. The Kubernetes controllers that generate + Endpoints and EndpointSlice resources for Services interpret + this to mean that all endpoints are considered "ready" even + if the Pods themselves are not. Agents which consume only + Kubernetes generated endpoints through the Endpoints or + EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only applies + to types ClusterIP, NodePort, and LoadBalancer. Ignored + if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client + IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, + NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal + IP address for load-balancing to endpoints. Endpoints are + determined by the selector or if that is not specified, + by manual construction of an Endpoints object or EndpointSlice + objects. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints rather + than a virtual IP. "NodePort" builds on ClusterIP and allocates + a port on every node which routes to the same endpoints + as the clusterIP. "LoadBalancer" builds on NodePort and + creates an external load-balancer (if supported in the current + cloud) which routes to the same endpoints as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + required: + - spec + type: object + required: + - resources + type: object + status: + description: AppWrapperStatus represents the current state of a AppWrapper + properties: + Succeeded: + description: The number of resources which reached phase Succeeded. + format: int32 + type: integer + canrun: + description: Can run? + type: boolean + conditions: + description: Represents the latest available observations of the AppWrapper's + current condition. + items: + description: AppWrapperCondition describes the state of an AppWrapper + at a certain point. + properties: + lastTransitionMicroTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateMicroTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of AppWrapper condition. + type: string + required: + - status + - type + type: object + type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches + the AppWrapper + format: date-time + type: string + controllerfirsttimestamp: + description: Microsecond level timestamp when controller first sees + QueueJob (by Informer) + format: date-time + type: string + failed: + description: The number of resources which reached phase Failed. + format: int32 + type: integer + filterignore: + description: Tell Informer to ignore this update message (do not generate + a controller event) + type: boolean + isdispatched: + description: Is Dispatched? + type: boolean + local: + description: Indicate if message is a duplicate (for Informer to recognize + duplicate messages) + type: boolean + message: + type: string + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event + has been triggered + type: integer + pending: + description: The number of pending pods. + format: int32 + type: integer + pendingpodconditions: + description: Represents the latest available observations of pods + belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current + condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can + be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array + queuejobstate: + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, + ... + type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent + in requeueing + type: integer + running: + format: int32 + type: integer + sender: + description: Indicate sender of this message (extremely useful for + debugging) + type: string + state: + description: State - Pending, Running, Failed, Deleted + type: string + systempriority: + description: System defined Priority + format: float + type: number + template: + description: The minimal available resources to run for this AppWrapper + (is this different from the MinAvailable from JobStatus) + format: int32 + type: integer + totalcpu: + description: The number of CPU consumed by all pods belonging to the + AppWrapper. + format: int32 + type: integer + totalgpu: + description: The total number of GPUs consumed by all pods belonging + to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to + the AppWrapper. + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- diff --git a/config/crd/workload.codeflare_schedulingspecs.yaml b/config/crd/workload.codeflare_schedulingspecs.yaml new file mode 100644 index 000000000..fc44a4326 --- /dev/null +++ b/config/crd/workload.codeflare_schedulingspecs.yaml @@ -0,0 +1,102 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: schedulingspecs.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: SchedulingSpec + listKind: SchedulingSpecList + plural: schedulingspecs + singular: schedulingspec + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. + Requeuing is triggered when the number of running/completed pods + is not equal to this value. When not specified, requeuing is disabled + and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting + time. Values in this field control how often the pod check should + happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between + requeuing checks. The values available are 'exponential', 'linear', + or 'none'. For example, 'exponential' growth would double the + 'timeInSeconds' value every time a requeuing event is triggered. + If the string value is misspelled or not one of the possible + options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users + cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing + event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it may + grow as requeuing events happen. + type: integer + type: object + type: object + required: + - metadata + type: object + served: true + storage: true From 9d8297e81e677e7465166791230050b8ae2202bf Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 20 Oct 2023 15:52:28 +0000 Subject: [PATCH 021/369] Update dependency versions for release v1.0.0-rc.3 --- Makefile | 6 +++--- README.md | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- test/support/defaults.go | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 00de9c196..636398eaf 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.1.0 +INSTASCALE_VERSION ?= v0.2.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.36.0 +MCAD_VERSION ?= v1.37.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} @@ -28,7 +28,7 @@ KUBERAY_VERSION ?= v1.0.0-rc.1 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.9.0 +CODEFLARE_SDK_VERSION ?= 0.10.0 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index b2f409319..aff1b9a68 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0-rc.2](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.2) | -| Multi-Cluster App Dispatcher | [v1.36.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.36.0) | -| CodeFlare-SDK | [v0.9.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.9.0) | -| InstaScale | [v0.1.0](https://github.com/project-codeflare/instascale/releases/tag/v0.1.0) | +| CodeFlare Operator | [v1.0.0-rc.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.3) | +| Multi-Cluster App Dispatcher | [v1.37.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.0) | +| CodeFlare-SDK | [v0.10.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.0) | +| InstaScale | [v0.2.0](https://github.com/project-codeflare/instascale/releases/tag/v0.2.0) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | diff --git a/go.mod b/go.mod index 08f3cb317..ffda77633 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.1.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 + github.com/project-codeflare/instascale v0.2.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 2c53de1ce..06269a9fa 100644 --- a/go.sum +++ b/go.sum @@ -389,10 +389,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/instascale v0.1.0 h1:oP5PwUIDvh+D3fbg8sT9dKphl9K/Tb/RraTFQezLOxI= -github.com/project-codeflare/instascale v0.1.0/go.mod h1:c4I8w4R0I8OJdkwA/ht/o/3lxRCBx4YdFJ3D1DulFVA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 h1:QSc16Kd4OBidGY4rQivMrUX9B9cNKttfZIRKsR4BX9E= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.2.0 h1:85Bbz6AsAYIKROqFmDIGS9hWV6pDf1iTizDVipxJwGg= +github.com/project-codeflare/instascale v0.2.0/go.mod h1:6x6WhDZiJRRwIAK7sAND3YYSGU3ry2Wa/RqyaHXKy18= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= diff --git a/test/support/defaults.go b/test/support/defaults.go index 1beb7e896..bf9db0814 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.9.0" + CodeFlareSDKVersion = "0.10.0" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From 9db28cf1d3577a703610e5c065907e529c36abe2 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Sat, 21 Oct 2023 19:15:11 +0100 Subject: [PATCH 022/369] build: pin ubi image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 72dfe6e7d..ab7210993 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.19.13 as builder WORKDIR /workspace # Copy the Go Modules manifests From 6f7865b6dfdc46da3d57dc6c60991790ef274b7a Mon Sep 17 00:00:00 2001 From: James Busche Date: Mon, 16 Oct 2023 15:03:09 -0700 Subject: [PATCH 023/369] fixing golang items in manager Signed-off-by: James Busche --- go.mod | 12 ++++++------ go.sum | 23 ++++++++++++----------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index ffda77633..8c43cbbc3 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.1 // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -92,13 +92,13 @@ require ( go.opentelemetry.io/otel/trace v1.10.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.11.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 06269a9fa..fe0d21e8b 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,9 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -541,8 +542,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -619,8 +620,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -702,13 +703,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -718,8 +719,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From f35ffa2bd85f6fe891dab38ca8616aaf64d91b34 Mon Sep 17 00:00:00 2001 From: James Busche Date: Thu, 19 Oct 2023 15:27:46 -0700 Subject: [PATCH 024/369] Update to opentelemetry Signed-off-by: James Busche --- go.mod | 39 +++++++++++----------- go.sum | 100 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 69 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index 8c43cbbc3..83c2243dc 100644 --- a/go.mod +++ b/go.mod @@ -28,8 +28,8 @@ require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -47,7 +47,7 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect - github.com/golang/glog v1.0.0 // indirect + github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/cel-go v0.12.6 // indirect @@ -57,7 +57,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -81,30 +81,31 @@ require ( go.etcd.io/etcd/api/v3 v3.5.5 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect go.etcd.io/etcd/client/v3 v3.5.5 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect - go.opentelemetry.io/otel v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect - go.opentelemetry.io/otel/metric v0.31.0 // indirect - go.opentelemetry.io/otel/sdk v1.10.0 // indirect - go.opentelemetry.io/otel/trace v1.10.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sync v0.2.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect - google.golang.org/grpc v1.49.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc v1.58.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index fe0d21e8b..5f4d67886 100644 --- a/go.sum +++ b/go.sum @@ -13,13 +13,15 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -57,25 +59,24 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -103,8 +104,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -157,8 +158,9 @@ github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -205,7 +207,6 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -236,8 +237,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -325,8 +326,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -425,6 +426,7 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 h github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -458,7 +460,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= @@ -487,34 +489,32 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 h1:Ajldaqhxqw/gNzQA45IKFWLdG7jZuXX/wBW1d5qvbUI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= -go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= -go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= -go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= -go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= -go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= -go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= -go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= -go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -628,10 +628,9 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -643,8 +642,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -688,7 +687,6 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -841,10 +839,13 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -862,10 +863,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -879,9 +878,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 9912d96ccc33eec50061df248f4cfa4a6b4c934a Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Mon, 23 Oct 2023 20:42:23 +0000 Subject: [PATCH 025/369] Update dependency versions for release v1.0.0-rc.4 --- Makefile | 6 +++--- README.md | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- test/support/defaults.go | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 636398eaf..e7bd3787a 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.2.0 +INSTASCALE_VERSION ?= v0.2.1 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.37.0 +MCAD_VERSION ?= v1.37.1 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} @@ -28,7 +28,7 @@ KUBERAY_VERSION ?= v1.0.0-rc.1 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.10.0 +CODEFLARE_SDK_VERSION ?= 0.10.1 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index aff1b9a68..a21c16475 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0-rc.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.3) | -| Multi-Cluster App Dispatcher | [v1.37.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.0) | -| CodeFlare-SDK | [v0.10.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.0) | -| InstaScale | [v0.2.0](https://github.com/project-codeflare/instascale/releases/tag/v0.2.0) | +| CodeFlare Operator | [v1.0.0-rc.4](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.4) | +| Multi-Cluster App Dispatcher | [v1.37.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.1) | +| CodeFlare-SDK | [v0.10.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.1) | +| InstaScale | [v0.2.1](https://github.com/project-codeflare/instascale/releases/tag/v0.2.1) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | diff --git a/go.mod b/go.mod index 83c2243dc..40254116f 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.2.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 + github.com/project-codeflare/instascale v0.2.1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 5f4d67886..867260a31 100644 --- a/go.sum +++ b/go.sum @@ -391,10 +391,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/instascale v0.2.0 h1:85Bbz6AsAYIKROqFmDIGS9hWV6pDf1iTizDVipxJwGg= -github.com/project-codeflare/instascale v0.2.0/go.mod h1:6x6WhDZiJRRwIAK7sAND3YYSGU3ry2Wa/RqyaHXKy18= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.2.1 h1:t+Ax/sk4yPQznO6N+U8Cq+bk3afCcZDj9wnHfiGSDBg= +github.com/project-codeflare/instascale v0.2.1/go.mod h1:zSzBTP4cFkg+bD4JyYTDmDnGwVKY/+6ACks57NAiscc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 h1:hZhGwKTPeHYYhNbvO27NOjozVpy7m3I3apKf81u9U3A= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= diff --git a/test/support/defaults.go b/test/support/defaults.go index bf9db0814..2e0227cf7 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.10.0" + CodeFlareSDKVersion = "0.10.1" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From c7954a9bd6b19f5978bcf318ebb7d839ad4b1b99 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Sun, 22 Oct 2023 20:22:09 +0100 Subject: [PATCH 026/369] updating make manifests target --- Makefile | 43 +- config/crd/crd-appwrapper.yml | 462 +++++++++++ config/crd/crd-quotasubtree.yml | 115 +++ config/crd/crd-schedulingspec.yml | 80 ++ config/crd/kustomization.yaml | 6 +- config/crd/mcad/kustomization.yaml | 2 +- config/crd/quota.codeflare_quotasubtrees.yaml | 122 --- .../crd/workload.codeflare_appwrappers.yaml | 765 ------------------ .../workload.codeflare_schedulingspecs.yaml | 102 --- 9 files changed, 672 insertions(+), 1025 deletions(-) create mode 100644 config/crd/crd-appwrapper.yml create mode 100644 config/crd/crd-quotasubtree.yml create mode 100644 config/crd/crd-schedulingspec.yml delete mode 100644 config/crd/quota.codeflare_quotasubtrees.yaml delete mode 100644 config/crd/workload.codeflare_appwrappers.yaml delete mode 100644 config/crd/workload.codeflare_schedulingspecs.yaml diff --git a/Makefile b/Makefile index e7bd3787a..d89caf1cd 100644 --- a/Makefile +++ b/Makefile @@ -148,10 +148,12 @@ defaults: # this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests -manifests: controller-gen kustomize ## Generate RBAC objects. +manifests: controller-gen kustomize install-yq ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." -# $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml -# $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml && make split_yaml FILE=config/crd/mcad.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml + $(YQ) -s '"crd-" + .spec.names.singular' config/crd/mcad.yaml --no-doc + mv crd-*.yml config/crd .PHONY: fmt fmt: ## Run go fmt against code. @@ -236,6 +238,7 @@ SED ?= /usr/bin/sed KUSTOMIZE_VERSION ?= v4.5.4 CODEGEN_VERSION ?= v0.27.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 +YQ_VERSION ?= v4.30.8 ## latest version that works with go1.19 OPERATOR_SDK_VERSION ?= v1.27.0 GH_CLI_VERSION ?= 2.30.0 @@ -261,6 +264,11 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +.PHONY: install-yq +install-yq: $(YQ) ## Download yq locally if necessary +$(YQ): $(LOCALBIN) + test -s $(LOCALBIN)/yq || GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@$(YQ_VERSION) + .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) @@ -388,32 +396,3 @@ verify-imports: openshift-goimports ## Run import verifications. .PHONY: scorecard-bundle scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. $(OPERATOR_SDK) scorecard bundle - - -FILE ?= input.yaml # Default value, it isn't a file, but the make cmds fill hang for longer without it -temp_dir := temp_split -output_dir := 'config/crd/' - -.PHONY: check_yq -check_yq: - @command -v wget >/dev/null 2>&1 || (echo "Installing wget..."; apt-get install -y wget) - @command -v $(YQ) >/dev/null 2>&1 || (echo "Installing yq..."; wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64.tar.gz -O - |\ - tar xz && mv yq_linux_amd64 $(YQ)) - -# this works on a MacOS by replacing awk with gawk -.PHONY: split_yaml -split_yaml: - @$(MAKE) check_yq - @mkdir -p $(temp_dir) - @awk '/apiVersion: /{if (x>0) close("$(temp_dir)/section_" x ".yaml"); x++}{print > "$(temp_dir)/section_"x".yaml"}' $(FILE) - @$(MAKE) process_sections - -.PHONY: process_sections -process_sections: - @mkdir -p $(output_dir) - @for section_file in $(temp_dir)/section_*; do \ - metadata_name=$$(YQ e '.metadata.name' $$section_file); \ - file_name=$$(echo $$metadata_name | awk -F'.' '{print $$2"."$$3"_"$$1".yaml"}'); \ - mv $$section_file $(output_dir)/$$file_name; \ - done - @rm -r $(temp_dir) diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml new file mode 100644 index 000000000..9140857a0 --- /dev/null +++ b/config/crd/crd-appwrapper.yml @@ -0,0 +1,462 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: appwrappers.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: AppWrapper + listKind: AppWrapperList + plural: appwrappers + singular: appwrapper + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Definition of AppWrapper class + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppWrapperSpec describes how the App Wrapper will look like. + properties: + priority: + format: int32 + type: integer + priorityslope: + format: float + type: number + resources: + description: a collection of AppWrapperResource + properties: + GenericItems: + items: + description: AppWrapperGenericResource is App Wrapper aggregation resource + properties: + allocated: + description: The number of allocated replicas from this resource type + format: int32 + type: integer + completionstatus: + description: 'Optional field that drives completion status of this AppWrapper. This field within an item of an AppWrapper determines the full state of the AppWrapper. The completionstatus field contains a list of conditions that make the associate item considered completed, for instance: - completion conditions could be "Complete" or "Failed". The associated item''s level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this option is set and the conditionstatus is met the entire AppWrapper state will be changed to one of the valid AppWrapper completion state. Note: - this is an AND operation for all items where this option is set. See the list of AppWrapper states for a list of valid complete states.' + type: string + custompodresources: + description: Optional section that specifies resource requirements for non-standard k8s resources, follows same format as that of standard k8s resources. + items: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) pairs. + type: object + replicas: + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'todo: replace with Containers []Container Contain v1.ResourceRequirements' + type: object + required: + - replicas + - requests + type: object + type: array + generictemplate: + description: The template for the resource; it is now a raw text because we don't know for what resource it should be instantiated + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + minavailable: + description: The minimal available pods to run for this AppWrapper; the default value is nil + format: int32 + type: integer + priority: + description: The priority of this resource + format: int32 + type: integer + priorityslope: + description: The increasing rate of priority value for this resource + format: float + type: number + replicas: + description: Replicas is the number of desired replicas + format: int32 + type: integer + type: object + type: array + type: object + schedulingSpec: + description: SchedSpec specifies the parameters used for scheduling generic items wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number of running pods. + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. + type: integer + type: object + type: object + selector: + description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + service: + description: AppWrapperService is App Wrapper service definition + properties: + spec: + description: ServiceSpec describes the attributes that a user creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer. Default is "true". It may be set to "false" if the cluster load-balancer does not rely on NodePorts. If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service and is usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be blank) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are "None", empty string (""), or a valid IP address. Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned to this service, and are usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be empty) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. If this field is not specified, it will be initialized from the clusterIP field. If this field is specified, clients must ensure that clusterIPs[0] and clusterIP have the same value. \n This field may hold a maximum of two entries (dual-stack IPs, in either order). These IPs must correspond to the values of the ipFamilies field. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure the service in a way that assumes that external load balancers will take care of balancing the service traffic between nodes, and so each node will deliver traffic only to the node-local endpoints of the service, without masquerading the client source IP. (Traffic mistakenly sent to a node with no endpoints will be dropped.) The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). Note that traffic sent to an External IP or LoadBalancer IP from within the cluster will always get "Cluster" semantics, but clients sending to a NodePort from within the cluster may need to take traffic policy into account when picking a node. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck nodePort for the service. This only applies when type is set to LoadBalancer and externalTrafficPolicy is set to Local. If a value is specified, is in-range, and is not in use, it will be used. If not specified, a value will be automatically allocated. External systems (e.g. load-balancers) can use this port to determine if a given node holds endpoints for this service or not. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type). This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes distribute service traffic they receive on the ClusterIP. If set to "Local", the proxy will assume that pods only want to talk to endpoints of the service on the same node as the pod, dropping the traffic if there are no local endpoints. The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). + type: string + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply to \"headless\" services. This field will be wiped when updating a Service to type ExternalName. \n This field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field." + items: + description: IPFamily represents the IP Family (IPv4 or IPv6). This type is used to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this field will be set to SingleStack. Services can be "SingleStack" (a single IP family), "PreferDualStack" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or "RequireDualStack" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature. Deprecated: This field was under-specified and its meaning varies across implementations, and it cannot support dual-stack. As of Kubernetes v1.24, users are encouraged to use implementation-specific annotations when available. This field may be removed in a future API version.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's port. + properties: + appProtocol: + description: The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and https://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. + type: string + name: + description: The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not specified, a port will be allocated if this Service requires one. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod''s container ports. If this is not specified, the value of the ''port'' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any agent which deals with endpoints for this Service should disregard any indications of ready/not-ready. The primary use case for setting this field is for a StatefulSet's Headless Service to propagate SRV DNS records for its Pods for the purpose of peer discovery. The Kubernetes controllers that generate Endpoints and EndpointSlice resources for Services interpret this to mean that all endpoints are considered "ready" even if the Pods themselves are not. Agents which consume only Kubernetes generated endpoints through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object or EndpointSlice objects. If clusterIP is "None", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a virtual IP. "NodePort" builds on ClusterIP and allocates a port on every node which routes to the same endpoints as the clusterIP. "LoadBalancer" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the same endpoints as the clusterIP. "ExternalName" aliases this service to the specified externalName. Several other fields do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + required: + - spec + type: object + required: + - resources + type: object + status: + description: AppWrapperStatus represents the current state of a AppWrapper + properties: + Succeeded: + description: The number of resources which reached phase Succeeded. + format: int32 + type: integer + canrun: + description: Can run? + type: boolean + conditions: + description: Represents the latest available observations of the AppWrapper's current condition. + items: + description: AppWrapperCondition describes the state of an AppWrapper at a certain point. + properties: + lastTransitionMicroTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateMicroTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of AppWrapper condition. + type: string + required: + - status + - type + type: object + type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches the AppWrapper + format: date-time + type: string + controllerfirsttimestamp: + description: Microsecond level timestamp when controller first sees QueueJob (by Informer) + format: date-time + type: string + failed: + description: The number of resources which reached phase Failed. + format: int32 + type: integer + filterignore: + description: Tell Informer to ignore this update message (do not generate a controller event) + type: boolean + isdispatched: + description: Is Dispatched? + type: boolean + local: + description: Indicate if message is a duplicate (for Informer to recognize duplicate messages) + type: boolean + message: + type: string + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event has been triggered + type: integer + pending: + description: The number of pending pods. + format: int32 + type: integer + pendingpodconditions: + description: Represents the latest available observations of pods belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array + queuejobstate: + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, ... + type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent in requeueing + type: integer + running: + format: int32 + type: integer + sender: + description: Indicate sender of this message (extremely useful for debugging) + type: string + state: + description: State - Pending, Running, Failed, Deleted + type: string + systempriority: + description: System defined Priority + format: float + type: number + template: + description: The minimal available resources to run for this AppWrapper (is this different from the MinAvailable from JobStatus) + format: int32 + type: integer + totalcpu: + description: The number of CPU consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + totalgpu: + description: The total number of GPUs consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/crd-quotasubtree.yml b/config/crd/crd-quotasubtree.yml new file mode 100644 index 000000000..924aaacd7 --- /dev/null +++ b/config/crd/crd-quotasubtree.yml @@ -0,0 +1,115 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.quota.codeflare.dev +spec: + group: quota.codeflare.dev + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/config/crd/crd-schedulingspec.yml b/config/crd/crd-schedulingspec.yml new file mode 100644 index 000000000..373dc89d8 --- /dev/null +++ b/config/crd/crd-schedulingspec.yml @@ -0,0 +1,80 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: schedulingspecs.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: SchedulingSpec + listKind: SchedulingSpecList + plural: schedulingspecs + singular: schedulingspec + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. + type: integer + type: object + type: object + required: + - metadata + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 73a576238..faaff445f 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,8 +2,8 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- quota.codeflare_quotasubtrees.yaml -- workload.codeflare_appwrappers.yaml -- workload.codeflare_schedulingspecs.yaml +- crd-appwrapper.yml +- crd-quotasubtree.yml +- crd-schedulingspec.yml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index e016d77f8..e53d2dd94 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.36.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.37.1 diff --git a/config/crd/quota.codeflare_quotasubtrees.yaml b/config/crd/quota.codeflare_quotasubtrees.yaml deleted file mode 100644 index 44bb5e0a6..000000000 --- a/config/crd/quota.codeflare_quotasubtrees.yaml +++ /dev/null @@ -1,122 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: quotasubtrees.quota.codeflare.dev -spec: - group: quota.codeflare.dev - names: - kind: QuotaSubtree - listKind: QuotaSubtreeList - plural: quotasubtrees - singular: quotasubtree - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: QuotaSubtree is a specification for a quota subtree resource - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: QuotaSubtreeSpec is the spec for a resource plan - properties: - children: - items: - description: Child is the spec for a QuotaSubtree resource - properties: - name: - type: string - namespace: - type: string - path: - type: string - quotas: - description: Quota is the spec for a QuotaSubtree resource - properties: - disabled: - type: boolean - hardLimit: - type: boolean - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - type: array - parent: - type: string - parentNamespace: - type: string - type: object - status: - description: QuotaSubtreeStatus is the status for a QuotaSubtree resource - properties: - children: - items: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child - resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - type: array - totalAllocation: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child - resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - required: - - children - - totalAllocation - type: object - required: - - spec - type: object - served: true - storage: true ---- diff --git a/config/crd/workload.codeflare_appwrappers.yaml b/config/crd/workload.codeflare_appwrappers.yaml deleted file mode 100644 index e1d260411..000000000 --- a/config/crd/workload.codeflare_appwrappers.yaml +++ /dev/null @@ -1,765 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: appwrappers.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: AppWrapper - listKind: AppWrapperList - plural: appwrappers - singular: appwrapper - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Definition of AppWrapper class - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AppWrapperSpec describes how the App Wrapper will look like. - properties: - priority: - format: int32 - type: integer - priorityslope: - format: float - type: number - resources: - description: a collection of AppWrapperResource - properties: - GenericItems: - items: - description: AppWrapperGenericResource is App Wrapper aggregation - resource - properties: - allocated: - description: The number of allocated replicas from this - resource type - format: int32 - type: integer - completionstatus: - description: 'Optional field that drives completion status - of this AppWrapper. This field within an item of an AppWrapper - determines the full state of the AppWrapper. The completionstatus - field contains a list of conditions that make the associate - item considered completed, for instance: - completion - conditions could be "Complete" or "Failed". The associated - item''s level .status.conditions[].type field is monitored - for any one of these conditions. Once all items with this - option is set and the conditionstatus is met the entire - AppWrapper state will be changed to one of the valid AppWrapper - completion state. Note: - this is an AND operation for - all items where this option is set. See the list of AppWrapper - states for a list of valid complete states.' - type: string - custompodresources: - description: Optional section that specifies resource requirements - for non-standard k8s resources, follows same format as - that of standard k8s resources. - items: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, - quantity) pairs. - type: object - replicas: - type: integer - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'todo: replace with Containers []Container - Contain v1.ResourceRequirements' - type: object - required: - - replicas - - requests - type: object - type: array - generictemplate: - description: The template for the resource; it is now a - raw text because we don't know for what resource it should - be instantiated - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - minavailable: - description: The minimal available pods to run for this - AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this - resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - type: object - type: array - type: object - schedulingSpec: - description: SchedSpec specifies the parameters used for scheduling - generic items wrapped inside AppWrappers. It defines the policy - for requeuing jobs based on the number of running pods. - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed - state. Requeuing is triggered when the number of running/completed - pods is not equal to this value. When not specified, requeuing - is disabled and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on - waiting time. Values in this field control how often the pod - check should happen, and if requeuing has reached its maximum - number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time - between requeuing checks. The values available are 'exponential', - 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing - event is triggered. If the string value is misspelled or - not one of the possible options, the growth behavior is - defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. - Users cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once - this value is reached (e.g., 'numRequeuings = maxNumRequeuings', - no more requeuing checks are performed and the generic items - are stopped and removed from the cluster (AppWrapper remains - deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing - event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions - are checked. This value is specified by the user, but it - may grow as requeuing events happen. - type: integer - type: object - type: object - selector: - description: A label selector is a label query over a set of resources. - The result of matchLabels and matchExpressions are ANDed. An empty - label selector matches all objects. A null label selector matches - no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - service: - description: AppWrapperService is App Wrapper service definition - properties: - spec: - description: ServiceSpec describes the attributes that a user - creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: allocateLoadBalancerNodePorts defines if NodePorts - will be automatically allocated for services with type LoadBalancer. Default - is "true". It may be set to "false" if the cluster load-balancer - does not rely on NodePorts. If the caller requests specific - NodePorts (by specifying a value), those requests will be - respected, regardless of this field. This field may only - be set for services with type LoadBalancer and will be cleared - if the type is changed to any other type. - type: boolean - clusterIP: - description: 'clusterIP is the IP address of the service and - is usually assigned randomly. If an address is specified - manually, is in-range (as per system configuration), and - is not in use, it will be allocated to the service; otherwise - creation of the service will fail. This field may not be - changed through updates unless the type field is also being - changed to ExternalName (which requires this field to be - blank) or the type field is being changed from ExternalName - (in which case this field may optionally be specified, as - describe above). Valid values are "None", empty string - (""), or a valid IP address. Setting this to "None" makes - a "headless service" (no virtual IP), which is useful when - direct endpoint connections are preferred and proxying is - not required. Only applies to types ClusterIP, NodePort, - and LoadBalancer. If this field is specified when creating - a Service of type ExternalName, creation will fail. This - field will be wiped when updating a Service to type ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - clusterIPs: - description: "ClusterIPs is a list of IP addresses assigned - to this service, and are usually assigned randomly. If - an address is specified manually, is in-range (as per system - configuration), and is not in use, it will be allocated - to the service; otherwise creation of the service will fail. - This field may not be changed through updates unless the - type field is also being changed to ExternalName (which - requires this field to be empty) or the type field is being - changed from ExternalName (in which case this field may - optionally be specified, as describe above). Valid values - are \"None\", empty string (\"\"), or a valid IP address. - \ Setting this to \"None\" makes a \"headless service\" - (no virtual IP), which is useful when direct endpoint connections - are preferred and proxying is not required. Only applies - to types ClusterIP, NodePort, and LoadBalancer. If this - field is specified when creating a Service of type ExternalName, - creation will fail. This field will be wiped when updating - a Service to type ExternalName. If this field is not specified, - it will be initialized from the clusterIP field. If this - field is specified, clients must ensure that clusterIPs[0] - and clusterIP have the same value. \n This field may hold - a maximum of two entries (dual-stack IPs, in either order). - These IPs must correspond to the values of the ipFamilies - field. Both clusterIPs and ipFamilies are governed by the - ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: externalIPs is a list of IP addresses for which - nodes in the cluster will also accept traffic for this service. These - IPs are not managed by Kubernetes. The user is responsible - for ensuring that traffic arrives at a node with this IP. A - common example is external load-balancers that are not part - of the Kubernetes system. - items: - type: string - type: array - externalName: - description: externalName is the external reference that discovery - mechanisms will return as an alias for this service (e.g. - a DNS CNAME record). No proxying will be involved. Must - be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) - and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: externalTrafficPolicy describes how nodes distribute - service traffic they receive on one of the Service's "externally-facing" - addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). - If set to "Local", the proxy will configure the service - in a way that assumes that external load balancers will - take care of balancing the service traffic between nodes, - and so each node will deliver traffic only to the node-local - endpoints of the service, without masquerading the client - source IP. (Traffic mistakenly sent to a node with no endpoints - will be dropped.) The default value, "Cluster", uses the - standard behavior of routing to all endpoints evenly (possibly - modified by topology and other features). Note that traffic - sent to an External IP or LoadBalancer IP from within the - cluster will always get "Cluster" semantics, but clients - sending to a NodePort from within the cluster may need to - take traffic policy into account when picking a node. - type: string - healthCheckNodePort: - description: healthCheckNodePort specifies the healthcheck - nodePort for the service. This only applies when type is - set to LoadBalancer and externalTrafficPolicy is set to - Local. If a value is specified, is in-range, and is not - in use, it will be used. If not specified, a value will - be automatically allocated. External systems (e.g. load-balancers) - can use this port to determine if a given node holds endpoints - for this service or not. If this field is specified when - creating a Service which does not need it, creation will - fail. This field will be wiped when updating a Service to - no longer need it (e.g. changing type). This field cannot - be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: InternalTrafficPolicy describes how nodes distribute - service traffic they receive on the ClusterIP. If set to - "Local", the proxy will assume that pods only want to talk - to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The - default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology - and other features). - type: string - ipFamilies: - description: "IPFamilies is a list of IP families (e.g. IPv4, - IPv6) assigned to this service. This field is usually assigned - automatically based on cluster configuration and the ipFamilyPolicy - field. If this field is specified manually, the requested - family is available in the cluster, and ipFamilyPolicy allows - it, it will be used; otherwise creation of the service will - fail. This field is conditionally mutable: it allows for - adding or removing a secondary IP family, but it does not - allow changing the primary IP family of the Service. Valid - values are \"IPv4\" and \"IPv6\". This field only applies - to Services of types ClusterIP, NodePort, and LoadBalancer, - and does apply to \"headless\" services. This field will - be wiped when updating a Service to type ExternalName. \n - This field may hold a maximum of two entries (dual-stack - families, in either order). These families must correspond - to the values of the clusterIPs field, if specified. Both - clusterIPs and ipFamilies are governed by the ipFamilyPolicy - field." - items: - description: IPFamily represents the IP Family (IPv4 or - IPv6). This type is used to express the family of an IP - expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: IPFamilyPolicy represents the dual-stack-ness - requested or required by this Service. If there is no value - provided, then this field will be set to SingleStack. Services - can be "SingleStack" (a single IP family), "PreferDualStack" - (two IP families on dual-stack configured clusters or a - single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise - fail). The ipFamilies and clusterIPs fields depend on the - value of this field. This field will be wiped when updating - a service to type ExternalName. - type: string - loadBalancerClass: - description: loadBalancerClass is the class of the load balancer - implementation this Service belongs to. If specified, the - value of this field must be a label-style identifier, with - an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". - Unprefixed names are reserved for end-users. This field - can only be set when the Service type is 'LoadBalancer'. - If not set, the default load balancer implementation is - used, today this is typically done through the cloud provider - integration, but should apply for any default implementation. - If set, it is assumed that a load balancer implementation - is watching for Services with a matching class. Any default - load balancer implementation (e.g. cloud providers) should - ignore Services that set this field. This field can only - be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped - when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: 'Only applies to Service Type: LoadBalancer. - This feature depends on whether the underlying cloud-provider - supports specifying the loadBalancerIP when a load balancer - is created. This field will be ignored if the cloud-provider - does not support the feature. Deprecated: This field was - under-specified and its meaning varies across implementations, - and it cannot support dual-stack. As of Kubernetes v1.24, - users are encouraged to use implementation-specific annotations - when available. This field may be removed in a future API - version.' - type: string - loadBalancerSourceRanges: - description: 'If specified and supported by the platform, - this will restrict traffic through the cloud-provider load-balancer - will be restricted to the specified client IPs. This field - will be ignored if the cloud-provider does not support the - feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' - items: - type: string - type: array - ports: - description: 'The list of ports that are exposed by this service. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - items: - description: ServicePort contains information on service's - port. - properties: - appProtocol: - description: The application protocol for this port. - This field follows standard Kubernetes label syntax. - Un-prefixed names are reserved for IANA standard service - names (as per RFC-6335 and https://www.iana.org/assignments/service-names). - Non-standard protocols should use prefixed names such - as mycompany.com/my-custom-protocol. - type: string - name: - description: The name of this port within the service. - This must be a DNS_LABEL. All ports within a ServiceSpec - must have unique names. When considering the endpoints - for a Service, this must match the 'name' field in - the EndpointPort. Optional if only one ServicePort - is defined on this service. - type: string - nodePort: - description: 'The port on each node on which this service - is exposed when type is NodePort or LoadBalancer. Usually - assigned by the system. If a value is specified, in-range, - and not in use it will be used, otherwise the operation - will fail. If not specified, a port will be allocated - if this Service requires one. If this field is specified - when creating a Service which does not need it, creation - will fail. This field will be wiped when updating - a Service to no longer need it (e.g. changing type - from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: The IP protocol for this port. Supports - "TCP", "UDP", and "SCTP". Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: 'Number or name of the port to access on - the pods targeted by the service. Number must be in - the range 1 to 65535. Name must be an IANA_SVC_NAME. - If this is a string, it will be looked up as a named - port in the target Pod''s container ports. If this - is not specified, the value of the ''port'' field - is used (an identity map). This field is ignored for - services with clusterIP=None, and should be omitted - or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: publishNotReadyAddresses indicates that any agent - which deals with endpoints for this Service should disregard - any indications of ready/not-ready. The primary use case - for setting this field is for a StatefulSet's Headless Service - to propagate SRV DNS records for its Pods for the purpose - of peer discovery. The Kubernetes controllers that generate - Endpoints and EndpointSlice resources for Services interpret - this to mean that all endpoints are considered "ready" even - if the Pods themselves are not. Agents which consume only - Kubernetes generated endpoints through the Endpoints or - EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: 'Route service traffic to pods with label keys - and values matching this selector. If empty or not present, - the service is assumed to have an external process managing - its endpoints, which Kubernetes will not modify. Only applies - to types ClusterIP, NodePort, and LoadBalancer. Ignored - if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: 'Supports "ClientIP" and "None". Used to maintain - session affinity. Enable client IP based session affinity. - Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations - of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client - IP based session affinity. - properties: - timeoutSeconds: - description: timeoutSeconds specifies the seconds - of ClientIP type session sticky time. The value - must be >0 && <=86400(for 1 day) if ServiceAffinity - == "ClientIP". Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - type: - description: 'type determines how the Service is exposed. - Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, - NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal - IP address for load-balancing to endpoints. Endpoints are - determined by the selector or if that is not specified, - by manual construction of an Endpoints object or EndpointSlice - objects. If clusterIP is "None", no virtual IP is allocated - and the endpoints are published as a set of endpoints rather - than a virtual IP. "NodePort" builds on ClusterIP and allocates - a port on every node which routes to the same endpoints - as the clusterIP. "LoadBalancer" builds on NodePort and - creates an external load-balancer (if supported in the current - cloud) which routes to the same endpoints as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' - type: string - type: object - required: - - spec - type: object - required: - - resources - type: object - status: - description: AppWrapperStatus represents the current state of a AppWrapper - properties: - Succeeded: - description: The number of resources which reached phase Succeeded. - format: int32 - type: integer - canrun: - description: Can run? - type: boolean - conditions: - description: Represents the latest available observations of the AppWrapper's - current condition. - items: - description: AppWrapperCondition describes the state of an AppWrapper - at a certain point. - properties: - lastTransitionMicroTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateMicroTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human-readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of AppWrapper condition. - type: string - required: - - status - - type - type: object - type: array - controllerfirstdispatchtimestamp: - description: Microsecond level timestamp when controller first dispatches - the AppWrapper - format: date-time - type: string - controllerfirsttimestamp: - description: Microsecond level timestamp when controller first sees - QueueJob (by Informer) - format: date-time - type: string - failed: - description: The number of resources which reached phase Failed. - format: int32 - type: integer - filterignore: - description: Tell Informer to ignore this update message (do not generate - a controller event) - type: boolean - isdispatched: - description: Is Dispatched? - type: boolean - local: - description: Indicate if message is a duplicate (for Informer to recognize - duplicate messages) - type: boolean - message: - type: string - numberOfRequeueings: - description: Field to keep track of how many times a requeuing event - has been triggered - type: integer - pending: - description: The number of pending pods. - format: int32 - type: integer - pendingpodconditions: - description: Represents the latest available observations of pods - belonging to the AppWrapper. - items: - properties: - conditions: - items: - description: PodCondition contains details for the current - condition of this pod. - properties: - lastProbeTime: - description: Last time we probed the condition. - format: date-time - type: string - lastTransitionTime: - description: Last time the condition transitioned from - one status to another. - format: date-time - type: string - message: - description: Human-readable message indicating details - about last transition. - type: string - reason: - description: Unique, one-word, CamelCase reason for the - condition's last transition. - type: string - status: - description: 'Status is the status of the condition. Can - be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - type: - description: 'Type is the type of the condition. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - required: - - status - - type - type: object - type: array - podname: - type: string - type: object - type: array - queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, - ... - type: string - requeueingTimeInSeconds: - description: Field to keep track of total number of seconds spent - in requeueing - type: integer - running: - format: int32 - type: integer - sender: - description: Indicate sender of this message (extremely useful for - debugging) - type: string - state: - description: State - Pending, Running, Failed, Deleted - type: string - systempriority: - description: System defined Priority - format: float - type: number - template: - description: The minimal available resources to run for this AppWrapper - (is this different from the MinAvailable from JobStatus) - format: int32 - type: integer - totalcpu: - description: The number of CPU consumed by all pods belonging to the - AppWrapper. - format: int32 - type: integer - totalgpu: - description: The total number of GPUs consumed by all pods belonging - to the AppWrapper. - format: int32 - type: integer - totalmemory: - description: The amount of memory consumed by all pods belonging to - the AppWrapper. - format: int32 - type: integer - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- diff --git a/config/crd/workload.codeflare_schedulingspecs.yaml b/config/crd/workload.codeflare_schedulingspecs.yaml deleted file mode 100644 index fc44a4326..000000000 --- a/config/crd/workload.codeflare_schedulingspecs.yaml +++ /dev/null @@ -1,102 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: schedulingspecs.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: SchedulingSpec - listKind: SchedulingSpecList - plural: schedulingspecs - singular: schedulingspec - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed state. - Requeuing is triggered when the number of running/completed pods - is not equal to this value. When not specified, requeuing is disabled - and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on waiting - time. Values in this field control how often the pod check should - happen, and if requeuing has reached its maximum number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time between - requeuing checks. The values available are 'exponential', 'linear', - or 'none'. For example, 'exponential' growth would double the - 'timeInSeconds' value every time a requeuing event is triggered. - If the string value is misspelled or not one of the possible - options, the growth behavior is defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users - cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once - this value is reached (e.g., 'numRequeuings = maxNumRequeuings', - no more requeuing checks are performed and the generic items - are stopped and removed from the cluster (AppWrapper remains - deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing - event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions - are checked. This value is specified by the user, but it may - grow as requeuing events happen. - type: integer - type: object - type: object - required: - - metadata - type: object - served: true - storage: true From b051b1a64a0dffcd0c0195758152c5f43a7908ff Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 24 Oct 2023 11:06:50 +0100 Subject: [PATCH 027/369] addressing feedback --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d89caf1cd..2e6ab468d 100644 --- a/Makefile +++ b/Makefile @@ -151,8 +151,7 @@ defaults: manifests: controller-gen kustomize install-yq ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml - $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml - $(YQ) -s '"crd-" + .spec.names.singular' config/crd/mcad.yaml --no-doc + $(KUSTOMIZE) build config/crd/mcad | $(YQ) -s '"crd-" + .spec.names.singular' --no-doc mv crd-*.yml config/crd .PHONY: fmt From 7e6da4cc2f3848546bc7bd1893bc9da6df90feba Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 10:50:27 +0100 Subject: [PATCH 028/369] Updated CFO sync and tag-build workflows --- .github/workflows/odh-fork-sync.yml | 9 ++++++--- .github/workflows/tag-and-build.yml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 72d4a3661..690a5937a 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -1,7 +1,8 @@ name: Call sync on OpenDataHub CFO fork sync on: release: - types: [published] + types: + [published] workflow_dispatch: jobs: @@ -9,7 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Sync-fork - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} run: | gh workflow run sync-fork.yaml -R github.com/opendatahub-io/codeflare-operator -r main + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: + bash diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 3fa7484d9..2ecdfb722 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -171,5 +171,5 @@ jobs: gh release edit ${{ github.event.inputs.version }} --notes-file release-notes.md rm release-notes.md env: - GITHUB_TOKEN: ${{ github.TOKEN }} + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 6d3d1221c2027e58d39c4515327d0b7141f3f756 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 11:00:33 +0100 Subject: [PATCH 029/369] Fixed formatting --- .github/workflows/odh-fork-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 690a5937a..3d8b3949a 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -1,7 +1,7 @@ name: Call sync on OpenDataHub CFO fork sync on: release: - types: + types: [published] workflow_dispatch: From 349d05009ebe9270b7b92c66b8a0d809c8a7e7a6 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 11:26:53 +0100 Subject: [PATCH 030/369] Updated release types --- .github/workflows/odh-fork-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 3d8b3949a..7c5a393d3 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -2,7 +2,7 @@ name: Call sync on OpenDataHub CFO fork sync on: release: types: - [published] + - published workflow_dispatch: jobs: From 47bd7064e90cd387bbb22e34708c20139324b8f9 Mon Sep 17 00:00:00 2001 From: Mark Campbell Date: Tue, 24 Oct 2023 12:15:00 +0100 Subject: [PATCH 031/369] Update .github/workflows/odh-fork-sync.yml Co-authored-by: Karel Suta --- .github/workflows/odh-fork-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 7c5a393d3..22f42f000 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Sync-fork run: | - gh workflow run sync-fork.yaml -R github.com/opendatahub-io/codeflare-operator -r main + gh workflow run sync-fork.yaml --repo github.com/opendatahub-io/codeflare-operator --ref main env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: From 14c6fb4e8608772049ea2f19ce498e9e3abe359c Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 13 Oct 2023 15:48:14 +0100 Subject: [PATCH 032/369] removing support package --- test/support/batch.go | 37 -------- test/support/conditions.go | 85 ----------------- test/support/core.go | 112 ---------------------- test/support/defaults.go | 11 --- test/support/events.go | 148 ----------------------------- test/support/ingress.go | 101 -------------------- test/support/mcad.go | 56 ----------- test/support/namespace.go | 55 ----------- test/support/ocm.go | 74 --------------- test/support/openshift.go | 33 ------- test/support/ray_api.go | 45 --------- test/support/ray_cluster_client.go | 128 ------------------------- test/support/route.go | 91 ------------------ test/support/service.go | 27 ------ test/support/support.go | 76 --------------- test/support/test.go | 139 --------------------------- test/support/utils.go | 41 -------- 17 files changed, 1259 deletions(-) delete mode 100644 test/support/batch.go delete mode 100644 test/support/conditions.go delete mode 100644 test/support/core.go delete mode 100644 test/support/defaults.go delete mode 100644 test/support/events.go delete mode 100644 test/support/ingress.go delete mode 100644 test/support/mcad.go delete mode 100644 test/support/namespace.go delete mode 100644 test/support/ocm.go delete mode 100644 test/support/openshift.go delete mode 100644 test/support/ray_api.go delete mode 100644 test/support/ray_cluster_client.go delete mode 100644 test/support/route.go delete mode 100644 test/support/service.go delete mode 100644 test/support/support.go delete mode 100644 test/support/test.go delete mode 100644 test/support/utils.go diff --git a/test/support/batch.go b/test/support/batch.go deleted file mode 100644 index 2cb2d5432..000000000 --- a/test/support/batch.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - batchv1 "k8s.io/api/batch/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job { - return func(g gomega.Gomega) *batchv1.Job { - job, err := t.Client().Core().BatchV1().Jobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return job - } -} - -func GetJob(t Test, namespace, name string) *batchv1.Job { - t.T().Helper() - return Job(t, namespace, name)(t) -} diff --git a/test/support/conditions.go b/test/support/conditions.go deleted file mode 100644 index 4ef4be1ee..000000000 --- a/test/support/conditions.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - - routev1 "github.com/openshift/api/route/v1" -) - -type conditionType interface { - ~string -} - -func ConditionStatus[T conditionType](conditionType T) func(any) corev1.ConditionStatus { - return func(object any) corev1.ConditionStatus { - switch o := object.(type) { - - case *batchv1.Job: - if c := getJobCondition(o.Status.Conditions, batchv1.JobConditionType(conditionType)); c != nil { - return c.Status - } - case *appsv1.Deployment: - if c := getDeploymentCondition(o.Status.Conditions, appsv1.DeploymentConditionType(conditionType)); c != nil { - return c.Status - } - case *routev1.Route: - if len(o.Status.Ingress) == 0 { - // Route is not initialized yet - break - } - if c := getRouteCondition(o.Status.Ingress[0].Conditions, routev1.RouteIngressConditionType(conditionType)); c != nil { - return c.Status - } - } - - return corev1.ConditionUnknown - } -} - -// TODO: to be replaced with a generic version once common struct fields of a type set can be used. -// See https://github.com/golang/go/issues/48522 -func getJobCondition(conditions []batchv1.JobCondition, conditionType batchv1.JobConditionType) *batchv1.JobCondition { - for _, c := range conditions { - if c.Type == conditionType { - return &c - } - } - return nil -} - -func getDeploymentCondition(conditions []appsv1.DeploymentCondition, conditionType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition { - for _, c := range conditions { - if c.Type == conditionType { - return &c - } - } - return nil -} - -func getRouteCondition(conditions []routev1.RouteIngressCondition, conditionType routev1.RouteIngressConditionType) *routev1.RouteIngressCondition { - for _, c := range conditions { - if c.Type == conditionType { - return &c - } - } - return nil -} diff --git a/test/support/core.go b/test/support/core.go deleted file mode 100644 index 70c48c20e..000000000 --- a/test/support/core.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "encoding/json" - "io" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -func CreateConfigMap(t Test, namespace string, content map[string][]byte) *corev1.ConfigMap { - configMap := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "config-", - Namespace: namespace, - }, - BinaryData: content, - Immutable: Ptr(true), - } - - configMap, err := t.Client().Core().CoreV1().ConfigMaps(namespace).Create(t.Ctx(), configMap, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - t.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) - - return configMap -} - -func Raw(t Test, obj runtime.Object) runtime.RawExtension { - t.T().Helper() - data, err := json.Marshal(obj) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return runtime.RawExtension{ - Raw: data, - } -} - -func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod { - t.T().Helper() - pods, err := t.Client().Core().CoreV1().Pods(namespace).List(t.Ctx(), options) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return pods.Items -} - -func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte { - t.T().Helper() - stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx()) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - t.Expect(stream.Close()).To(gomega.Succeed()) - }() - - bytes, err := io.ReadAll(stream) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - return bytes -} - -func storeAllPodLogs(t Test, namespace *corev1.Namespace) { - t.T().Helper() - - pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - for _, pod := range pods.Items { - for _, container := range pod.Spec.Containers { - t.T().Logf("Retrieving Pod Container %s/%s/%s logs", pod.Namespace, pod.Name, container.Name) - storeContainerLog(t, namespace, pod.Name, container.Name) - } - } -} - -func storeContainerLog(t Test, namespace *corev1.Namespace, podName, containerName string) { - t.T().Helper() - - options := corev1.PodLogOptions{Container: containerName} - stream, err := t.Client().Core().CoreV1().Pods(namespace.Name).GetLogs(podName, &options).Stream(t.Ctx()) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - t.Expect(stream.Close()).To(gomega.Succeed()) - }() - - bytes, err := io.ReadAll(stream) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - containerLogFileName := "pod-" + podName + "-" + containerName - WriteToOutputDir(t, containerLogFileName, Log, bytes) -} diff --git a/test/support/defaults.go b/test/support/defaults.go deleted file mode 100644 index 2e0227cf7..000000000 --- a/test/support/defaults.go +++ /dev/null @@ -1,11 +0,0 @@ -package support - -// *********************** -// DO NOT EDIT THIS FILE -// *********************** - -const ( - CodeFlareSDKVersion = "0.10.1" - RayVersion = "2.5.0" - RayImage = "rayproject/ray:2.5.0" -) diff --git a/test/support/events.go b/test/support/events.go deleted file mode 100644 index beafe558e..000000000 --- a/test/support/events.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "bytes" - "fmt" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - eventsv1 "k8s.io/api/events/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Based on https://github.com/apache/incubator-kie-kogito-operator/blob/28b2d3dc945e48659b199cca33723568b848f72e/test/pkg/framework/logging.go - -const ( - eventLastSeenKey = "LAST_SEEN" - eventFirstSeenKey = "FIRST_SEEN" - eventNameKey = "NAME" - eventSubObjectKey = "SUBOBJECT" - eventTypeKey = "TYPE" - eventReasonKey = "REASON" - eventMessageKey = "MESSAGE" - - eventLogFileName = "events" -) - -var eventKeys = []string{ - eventLastSeenKey, - eventFirstSeenKey, - eventNameKey, - eventSubObjectKey, - eventTypeKey, - eventReasonKey, - eventMessageKey, -} - -func storeEvents(t Test, namespace *corev1.Namespace) { - t.T().Helper() - - events, err := t.Client().Core().EventsV1().Events(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - bytes, err := renderEventContent(eventKeys, mapEventsToKeys(events)) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - WriteToOutputDir(t, eventLogFileName, Log, bytes) -} - -func mapEventsToKeys(eventList *eventsv1.EventList) []map[string]string { - eventMaps := []map[string]string{} - - for _, event := range eventList.Items { - eventMap := make(map[string]string) - eventMap[eventLastSeenKey] = getDefaultEventValueIfNull(event.DeprecatedLastTimestamp.Format("2006-01-02 15:04:05")) - eventMap[eventFirstSeenKey] = getDefaultEventValueIfNull(event.DeprecatedFirstTimestamp.Format("2006-01-02 15:04:05")) - eventMap[eventNameKey] = getDefaultEventValueIfNull(event.GetName()) - eventMap[eventSubObjectKey] = getDefaultEventValueIfNull(event.Regarding.FieldPath) - eventMap[eventTypeKey] = getDefaultEventValueIfNull(event.Type) - eventMap[eventReasonKey] = getDefaultEventValueIfNull(event.Reason) - eventMap[eventMessageKey] = getDefaultEventValueIfNull(event.Note) - - eventMaps = append(eventMaps, eventMap) - } - return eventMaps -} - -func getDefaultEventValueIfNull(value string) string { - if len(value) <= 0 { - return "-" - } - return value -} - -func renderEventContent(keys []string, dataMaps []map[string]string) ([]byte, error) { - var content bytes.Buffer - // Get size of strings to be written, to be able to format correctly - maxStringSizeMap := make(map[string]int) - for _, key := range keys { - maxSize := len(key) - for _, dataMap := range dataMaps { - if len(dataMap[key]) > maxSize { - maxSize = len(dataMap[key]) - } - } - maxStringSizeMap[key] = maxSize - } - - // Write headers - for _, header := range keys { - if _, err := content.WriteString(header); err != nil { - return nil, fmt.Errorf("error in writing the header: %v", err) - } - if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[header] - len(header) + 1)); err != nil { - return nil, fmt.Errorf("error in writing headers: %v", err) - } - if _, err := content.WriteString(" | "); err != nil { - return nil, fmt.Errorf("error in writing headers : %v", err) - } - } - if _, err := content.WriteString("\n"); err != nil { - return nil, fmt.Errorf("error in writing headers '|': %v", err) - - } - - // Write events - for _, dataMap := range dataMaps { - for _, key := range keys { - if _, err := content.WriteString(dataMap[key]); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[key] - len(dataMap[key]) + 1)); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - if _, err := content.WriteString(" | "); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - } - if _, err := content.WriteString("\n"); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - } - return content.Bytes(), nil -} - -func getWhitespaceStr(size int) string { - whiteSpaceStr := "" - for i := 0; i < size; i++ { - whiteSpaceStr += " " - } - return whiteSpaceStr -} diff --git a/test/support/ingress.go b/test/support/ingress.go deleted file mode 100644 index d834f3b15..000000000 --- a/test/support/ingress.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "net/url" - - "github.com/onsi/gomega" - - networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Ingress(t Test, namespace, name string) func(g gomega.Gomega) *networkingv1.Ingress { - return func(g gomega.Gomega) *networkingv1.Ingress { - ingress, err := t.Client().Core().NetworkingV1().Ingresses(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return ingress - } -} - -func GetIngress(t Test, namespace, name string) *networkingv1.Ingress { - t.T().Helper() - return Ingress(t, namespace, name)(t) -} - -func LoadBalancerIngresses(ingress *networkingv1.Ingress) []networkingv1.IngressLoadBalancerIngress { - return ingress.Status.LoadBalancer.Ingress -} - -func ExposeServiceByIngress(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { - ingress := &networkingv1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: networkingv1.SchemeGroupVersion.String(), - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": "true", - "nginx.ingress.kubernetes.io/rewrite-target": "/$2", - }, - }, - Spec: networkingv1.IngressSpec{ - Rules: []networkingv1.IngressRule{ - networkingv1.IngressRule{ - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - networkingv1.HTTPIngressPath{ - Path: "/" + name + "(/|$)(.*)", - PathType: Ptr(networkingv1.PathTypePrefix), - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: serviceName, - Port: networkingv1.ServiceBackendPort{ - Name: servicePort, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - _, err := t.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(t.Ctx(), ingress, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - t.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) - - t.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) - t.Eventually(Ingress(t, ingress.Namespace, ingress.Name), TestTimeoutMedium). - Should(gomega.WithTransform(LoadBalancerIngresses, gomega.HaveLen(1))) - - ingress = GetIngress(t, ingress.Namespace, ingress.Name) - - ingressURL := url.URL{ - Scheme: "http", - Host: ingress.Status.LoadBalancer.Ingress[0].IP, - Path: name, - } - return ingressURL -} diff --git a/test/support/mcad.go b/test/support/mcad.go deleted file mode 100644 index 8fdda1652..000000000 --- a/test/support/mcad.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { - return func(g gomega.Gomega) *mcadv1beta1.AppWrapper { - aw, err := t.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return aw - } -} - -func AppWrappers(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []*mcadv1beta1.AppWrapper { - return func(g gomega.Gomega) []*mcadv1beta1.AppWrapper { - aws, err := t.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - awsp := []*mcadv1beta1.AppWrapper{} - for _, v := range aws.Items { - v := v - awsp = append(awsp, &v) - } - - return awsp - } -} - -func AppWrapperName(aw *mcadv1beta1.AppWrapper) string { - return aw.Name -} - -func AppWrapperState(aw *mcadv1beta1.AppWrapper) mcadv1beta1.AppWrapperState { - return aw.Status.State -} diff --git a/test/support/namespace.go b/test/support/namespace.go deleted file mode 100644 index 145acbb4a..000000000 --- a/test/support/namespace.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func createTestNamespace(t Test, options ...Option[*corev1.Namespace]) *corev1.Namespace { - t.T().Helper() - namespace := &corev1.Namespace{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Namespace", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-ns-", - }, - } - - for _, option := range options { - t.Expect(option.applyTo(namespace)).To(gomega.Succeed()) - } - - namespace, err := t.Client().Core().CoreV1().Namespaces().Create(t.Ctx(), namespace, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - return namespace -} - -func deleteTestNamespace(t Test, namespace *corev1.Namespace) { - t.T().Helper() - propagationPolicy := metav1.DeletePropagationBackground - err := t.Client().Core().CoreV1().Namespaces().Delete(t.Ctx(), namespace.Name, metav1.DeleteOptions{ - PropagationPolicy: &propagationPolicy, - }) - t.Expect(err).NotTo(gomega.HaveOccurred()) -} diff --git a/test/support/ocm.go b/test/support/ocm.go deleted file mode 100644 index 3d6bd7f19..000000000 --- a/test/support/ocm.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "fmt" - "os" - - "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - ocmsdk "github.com/openshift-online/ocm-sdk-go" - cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" -) - -func CreateOCMConnection(test Test) *ocmsdk.Connection { - secretNamespace, secretName := GetInstascaleOcmSecret() - instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets(secretNamespace).Get(test.Ctx(), secretName, metav1.GetOptions{}) - test.Expect(err).NotTo(gomega.HaveOccurred()) - - ocmToken := string(instascaleOCMSecret.Data["token"]) - test.T().Logf("Retrieved Secret %s/%s successfully", instascaleOCMSecret.Namespace, instascaleOCMSecret.Name) - - connection, err := buildOCMConnection(ocmToken) - test.Expect(err).NotTo(gomega.HaveOccurred()) - return connection -} - -func buildOCMConnection(secret string) (*ocmsdk.Connection, error) { - connection, err := ocmsdk.NewConnectionBuilder(). - Tokens(secret). - Build() - if err != nil { - fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) - return nil, err - } - - return connection, nil -} - -func MachinePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.MachinePool { - osdClusterId, found := GetOsdClusterId() - t.Expect(found).To(gomega.BeTrue(), "OSD cluster id not found, please configure environment properly") - - return func(g gomega.Gomega) []*cmv1.MachinePool { - machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(osdClusterId).MachinePools().List().Send() - g.Expect(err).NotTo(gomega.HaveOccurred()) - return machinePoolsListResponse.Items().Slice() - } -} - -func GetMachinePools(t Test, connection *ocmsdk.Connection) []*cmv1.MachinePool { - t.T().Helper() - return MachinePools(t, connection)(t) -} - -func MachinePoolId(machinePool *cmv1.MachinePool) string { - return machinePool.ID() -} diff --git a/test/support/openshift.go b/test/support/openshift.go deleted file mode 100644 index cfe3b5a3f..000000000 --- a/test/support/openshift.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - "k8s.io/apimachinery/pkg/api/errors" -) - -func IsOpenShift(test Test) bool { - test.T().Helper() - _, err := test.Client().Core().Discovery().ServerResourcesForGroupVersion("image.openshift.io/v1") - if err != nil && errors.IsNotFound(err) { - return false - } - test.Expect(err).NotTo(gomega.HaveOccurred()) - return true -} diff --git a/test/support/ray_api.go b/test/support/ray_api.go deleted file mode 100644 index 1f04f6dda..000000000 --- a/test/support/ray_api.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" -) - -func GetRayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) *RayJobDetailsResponse { - t.T().Helper() - return RayJobAPIDetails(t, rayClient, jobID)(t) -} - -func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) { - t.T().Helper() - logs, err := rayClient.GetJobLogs(jobID) - t.Expect(err).NotTo(gomega.HaveOccurred()) - WriteToOutputDir(t, "ray-job-log-"+jobID, Log, []byte(logs)) -} - -func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse { - return func(g gomega.Gomega) *RayJobDetailsResponse { - jobDetails, err := rayClient.GetJobDetails(jobID) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return jobDetails - } -} - -func GetRayJobAPIDetailsStatus(jobDetails *RayJobDetailsResponse) string { - return jobDetails.Status -} diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go deleted file mode 100644 index bbce9af2e..000000000 --- a/test/support/ray_cluster_client.go +++ /dev/null @@ -1,128 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" -) - -type RayJobSetup struct { - EntryPoint string `json:"entrypoint"` - RuntimeEnv map[string]any `json:"runtime_env"` -} - -type RayJobResponse struct { - JobID string `json:"job_id"` - SubmissionID string `json:"submission_id"` -} - -type RayJobDetailsResponse struct { - JobID string `json:"job_id"` - SubmissionID string `json:"submission_id"` - Status string `json:"status"` -} - -type RayJobLogsResponse struct { - Logs string `json:"logs"` -} - -var _ RayClusterClient = (*rayClusterClient)(nil) - -type rayClusterClient struct { - endpoint url.URL -} - -type RayClusterClient interface { - CreateJob(job *RayJobSetup) (*RayJobResponse, error) - GetJobDetails(jobID string) (*RayJobDetailsResponse, error) - GetJobLogs(jobID string) (string, error) -} - -func NewRayClusterClient(dashboardEndpoint url.URL) RayClusterClient { - return &rayClusterClient{endpoint: dashboardEndpoint} -} - -func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobResponse, err error) { - marshalled, err := json.Marshal(job) - if err != nil { - return - } - - createJobURL := client.endpoint.String() + "/api/jobs/" - resp, err := http.Post(createJobURL, "application/json", bytes.NewReader(marshalled)) - if err != nil { - return - } - - respData, err := io.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("incorrect response code: %d for creating Ray Job, response body: %s", resp.StatusCode, respData) - } - - err = json.Unmarshal(respData, &response) - return -} - -func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDetailsResponse, err error) { - getJobDetailsURL := client.endpoint.String() + "/api/jobs/" + jobID - resp, err := http.Get(getJobDetailsURL) - if err != nil { - return - } - - respData, err := io.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("incorrect response code: %d for retrieving Ray Job details, response body: %s", resp.StatusCode, respData) - } - - err = json.Unmarshal(respData, &response) - return -} - -func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error) { - getJobLogsURL := client.endpoint.String() + "/api/jobs/" + jobID + "/logs" - resp, err := http.Get(getJobLogsURL) - if err != nil { - return - } - - respData, err := io.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode != 200 { - return "", fmt.Errorf("incorrect response code: %d for retrieving Ray Job logs, response body: %s", resp.StatusCode, respData) - } - - jobLogs := RayJobLogsResponse{} - err = json.Unmarshal(respData, &jobLogs) - return jobLogs.Logs, err -} diff --git a/test/support/route.go b/test/support/route.go deleted file mode 100644 index df1608909..000000000 --- a/test/support/route.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "net/http" - "net/url" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - routev1 "github.com/openshift/api/route/v1" -) - -func Route(t Test, namespace, name string) func(g gomega.Gomega) *routev1.Route { - return func(g gomega.Gomega) *routev1.Route { - route, err := t.Client().Route().RouteV1().Routes(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return route - } -} - -func GetRoute(t Test, namespace, name string) *routev1.Route { - t.T().Helper() - return Route(t, namespace, name)(t) -} - -func ExposeServiceByRoute(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { - r := &routev1.Route{ - TypeMeta: metav1.TypeMeta{ - APIVersion: routev1.SchemeGroupVersion.String(), - Kind: "Route", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: routev1.RouteSpec{ - To: routev1.RouteTargetReference{ - Name: serviceName, - }, - Port: &routev1.RoutePort{ - TargetPort: intstr.FromString(servicePort), - }, - }, - } - - _, err := t.Client().Route().RouteV1().Routes(r.Namespace).Create(t.Ctx(), r, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - t.T().Logf("Created Route %s/%s successfully", r.Namespace, r.Name) - - t.T().Logf("Waiting for Route %s/%s to be available", r.Namespace, r.Name) - t.Eventually(Route(t, r.Namespace, r.Name), TestTimeoutLong). - Should(gomega.WithTransform(ConditionStatus(routev1.RouteAdmitted), gomega.Equal(corev1.ConditionTrue))) - - // Retrieve hostname - r, err = t.Client().Route().RouteV1().Routes(r.Namespace).Get(t.Ctx(), r.Name, metav1.GetOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - hostname := r.Status.Ingress[0].Host - - // Wait for expected HTTP code - t.Eventually(func() int { - resp, _ := http.Get("http://" + hostname) - return resp.StatusCode - }, TestTimeoutLong).Should(gomega.Not(gomega.Equal(503))) - - r = GetRoute(t, r.Namespace, r.Name) - routeURL := url.URL{ - Scheme: "http", - Host: r.Status.Ingress[0].Host, - } - - return routeURL -} diff --git a/test/support/service.go b/test/support/service.go deleted file mode 100644 index d59ea3ec9..000000000 --- a/test/support/service.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import "net/url" - -func ExposeService(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { - if IsOpenShift(t) { - return ExposeServiceByRoute(t, name, namespace, serviceName, servicePort) - } else { - return ExposeServiceByIngress(t, name, namespace, serviceName, servicePort) - } -} diff --git a/test/support/support.go b/test/support/support.go deleted file mode 100644 index 36c8c9beb..000000000 --- a/test/support/support.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "fmt" - "os" - "time" - - "github.com/onsi/gomega" - "github.com/onsi/gomega/format" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var ( - ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} - - TestTimeoutShort = 1 * time.Minute - TestTimeoutMedium = 2 * time.Minute - TestTimeoutLong = 5 * time.Minute - TestTimeoutGpuProvisioning = 30 * time.Minute -) - -func init() { - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_SHORT"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutShort = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_SHORT. Using default value: %s", TestTimeoutShort) - } - } - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_MEDIUM"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutMedium = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_MEDIUM. Using default value: %s", TestTimeoutMedium) - } - } - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_LONG"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutLong = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_LONG. Using default value: %s", TestTimeoutLong) - } - } - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutGpuProvisioning = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING. Using default value: %s", TestTimeoutGpuProvisioning) - } - } - - // Gomega settings - gomega.SetDefaultEventuallyTimeout(TestTimeoutShort) - gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) - gomega.SetDefaultConsistentlyDuration(30 * time.Second) - gomega.SetDefaultConsistentlyPollingInterval(1 * time.Second) - // Disable object truncation on test results - format.MaxLength = 0 -} diff --git a/test/support/test.go b/test/support/test.go deleted file mode 100644 index 683b0f214..000000000 --- a/test/support/test.go +++ /dev/null @@ -1,139 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "context" - "os" - "path" - "sync" - "testing" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" -) - -type Test interface { - T() *testing.T - Ctx() context.Context - Client() Client - OutputDir() string - - gomega.Gomega - - NewTestNamespace(...Option[*corev1.Namespace]) *corev1.Namespace -} - -type Option[T any] interface { - applyTo(to T) error -} - -type errorOption[T any] func(to T) error - -// nolint: unused -// To be removed when the false-positivity is fixed. -func (o errorOption[T]) applyTo(to T) error { - return o(to) -} - -var _ Option[any] = errorOption[any](nil) - -func With(t *testing.T) Test { - t.Helper() - ctx := context.Background() - if deadline, ok := t.Deadline(); ok { - withDeadline, cancel := context.WithDeadline(ctx, deadline) - t.Cleanup(cancel) - ctx = withDeadline - } - - return &T{ - WithT: gomega.NewWithT(t), - t: t, - ctx: ctx, - } -} - -type T struct { - *gomega.WithT - t *testing.T - // nolint: containedctx - ctx context.Context - client Client - outputDir string - once struct { - client sync.Once - outputDir sync.Once - } -} - -func (t *T) T() *testing.T { - return t.t -} - -func (t *T) Ctx() context.Context { - return t.ctx -} - -func (t *T) Client() Client { - t.T().Helper() - t.once.client.Do(func() { - c, err := newTestClient() - if err != nil { - t.T().Fatalf("Error creating client: %v", err) - } - t.client = c - }) - return t.client -} - -func (t *T) OutputDir() string { - t.T().Helper() - t.once.outputDir.Do(func() { - if parent, ok := os.LookupEnv(CodeFlareTestOutputDir); ok { - if !path.IsAbs(parent) { - if cwd, err := os.Getwd(); err == nil { - // best effort to output the parent absolute path - parent = path.Join(cwd, parent) - } - } - t.T().Logf("Creating output directory in parent directory: %s", parent) - dir, err := os.MkdirTemp(parent, t.T().Name()) - if err != nil { - t.T().Fatalf("Error creating output directory: %v", err) - } - t.outputDir = dir - } else { - t.T().Logf("Creating ephemeral output directory as %s env variable is unset", CodeFlareTestOutputDir) - t.outputDir = t.T().TempDir() - } - t.T().Logf("Output directory has been created at: %s", t.outputDir) - }) - return t.outputDir -} - -func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Namespace { - t.T().Helper() - namespace := createTestNamespace(t, options...) - t.T().Cleanup(func() { - storeAllPodLogs(t, namespace) - storeEvents(t, namespace) - deleteTestNamespace(t, namespace) - }) - return namespace -} diff --git a/test/support/utils.go b/test/support/utils.go deleted file mode 100644 index 595ac4103..000000000 --- a/test/support/utils.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "io/fs" - "os" - "path" - - "github.com/onsi/gomega" -) - -func Ptr[T any](v T) *T { - return &v -} - -type OutputType string - -const ( - Log OutputType = "log" -) - -func WriteToOutputDir(t Test, fileName string, fileType OutputType, data []byte) { - t.T().Helper() - t.Expect(os.WriteFile(path.Join(t.OutputDir(), fileName+"."+string(fileType)), data, fs.ModePerm)). - To(gomega.Succeed()) -} From 484a3fa58bf6eece2441ae2338228171962818e3 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 13 Oct 2023 15:49:04 +0100 Subject: [PATCH 033/369] updating support package import --- test/e2e/instascale_app_wrapper.go | 3 +-- test/e2e/instascale_machinepool_test.go | 2 +- test/e2e/mnist_pytorch_mcad_job_test.go | 3 +-- test/e2e/mnist_raycluster_sdk_test.go | 3 +-- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 3 +-- test/e2e/support.go | 3 +-- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index 805df1bc3..e030817f3 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -17,14 +17,13 @@ limitations under the License. package e2e import ( + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 386b224db..2791cb07e 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -20,11 +20,11 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/project-codeflare/codeflare-operator/test/support" ) func TestInstascaleMachinePool(t *testing.T) { diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 84722152a..7a157c040 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -20,14 +20,13 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) // Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 29643fe67..b8cc5af7e 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -20,6 +20,7 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -27,8 +28,6 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) // Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index a7956f66e..725ced9db 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -21,14 +21,13 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) // Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, diff --git a/test/e2e/support.go b/test/e2e/support.go index 14bf19ce4..e91656880 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -20,8 +20,7 @@ import ( "embed" "github.com/onsi/gomega" - - "github.com/project-codeflare/codeflare-operator/test/support" + "github.com/project-codeflare/codeflare-common/support" ) //go:embed *.py *.txt From f6e75536e1487bc2d757b232340a6ba919561074 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 16 Oct 2023 11:12:25 +0100 Subject: [PATCH 034/369] removing defaults make target --- Makefile | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 2e6ab468d..541fba37b 100644 --- a/Makefile +++ b/Makefile @@ -125,27 +125,6 @@ help: ## Display this help. ##@ Development -DEFAULTS_TEST_FILE := test/support/defaults.go - -.PHONY: defaults -defaults: - $(info Regenerating $(DEFAULTS_TEST_FILE)) - @echo "package support" > $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) - @echo "// DO NOT EDIT THIS FILE" >> $(DEFAULTS_TEST_FILE) - @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - @echo "const (" >> $(DEFAULTS_TEST_FILE) - @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_TEST_FILE) - @echo " RayVersion = \"$(RAY_VERSION)\"" >> $(DEFAULTS_TEST_FILE) - @echo " RayImage = \"$(RAY_IMAGE)\"" >> $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - @echo ")" >> $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - - gofmt -w $(DEFAULTS_TEST_FILE) - # this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests manifests: controller-gen kustomize install-yq ## Generate RBAC objects. @@ -173,11 +152,11 @@ modules: ## Update Go dependencies. go mod tidy .PHONY: build -build: modules defaults fmt vet ## Build manager binary. +build: modules fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: modules defaults manifests fmt vet ## Run a controller from your host. +run: modules manifests fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build @@ -290,7 +269,7 @@ validate-bundle: install-operator-sdk $(OPERATOR_SDK) bundle validate ./bundle --select-optional suite=operatorframework .PHONY: bundle -bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. +bundle: manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion @@ -369,11 +348,11 @@ catalog-push: ## Push a catalog image. podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT) .PHONY: test-unit -test-unit: defaults manifests fmt vet envtest ## Run unit tests. +test-unit: manifests fmt vet envtest ## Run unit tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out .PHONY: test-e2e -test-e2e: defaults manifests fmt vet ## Run e2e tests. +test-e2e: manifests fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e .PHONY: kind-e2e From cad078ee681a07d3d0f08a0e97f1b9606ec394d4 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 23 Oct 2023 14:41:42 +0100 Subject: [PATCH 035/369] updating files --- test/e2e/instascale_machinepool_test.go | 1 - test/e2e/instascale_machineset_test.go | 5 +- test/support/client.go | 137 ------------------------ test/support/environment.go | 104 ------------------ test/support/machine.go | 32 ------ test/support/ray.go | 70 ------------ 6 files changed, 2 insertions(+), 347 deletions(-) delete mode 100644 test/support/client.go delete mode 100644 test/support/environment.go delete mode 100644 test/support/machine.go delete mode 100644 test/support/ray.go diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 2791cb07e..69fa037d3 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -24,7 +24,6 @@ import ( mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ) func TestInstascaleMachinePool(t *testing.T) { diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index fad7717b2..827fd4044 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -4,11 +4,10 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) func TestInstascaleMachineSet(t *testing.T) { @@ -31,7 +30,7 @@ func TestInstascaleMachineSet(t *testing.T) { "mnist.py": ReadFile(test, "mnist.py"), }) - // // Setup batch job and AppWrapper + // Setup batch job and AppWrapper aw := instaScaleJobAppWrapper(test, namespace, cm) // look for machine set with aw name - expect to find it diff --git a/test/support/client.go b/test/support/client.go deleted file mode 100644 index d911c7c2c..000000000 --- a/test/support/client.go +++ /dev/null @@ -1,137 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned" - rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" - - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/clientcmd" - - imagev1 "github.com/openshift/client-go/image/clientset/versioned" - machinev1 "github.com/openshift/client-go/machine/clientset/versioned" - routev1 "github.com/openshift/client-go/route/clientset/versioned" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. -) - -type Client interface { - Core() kubernetes.Interface - Machine() machinev1.Interface - Route() routev1.Interface - Image() imagev1.Interface - MCAD() mcadclient.Interface - Ray() rayclient.Interface - Dynamic() dynamic.Interface -} - -type testClient struct { - core kubernetes.Interface - machine machinev1.Interface - route routev1.Interface - image imagev1.Interface - mcad mcadclient.Interface - ray rayclient.Interface - dynamic dynamic.Interface -} - -var _ Client = (*testClient)(nil) - -func (t *testClient) Core() kubernetes.Interface { - return t.core -} - -func (t *testClient) Machine() machinev1.Interface { - return t.machine -} - -func (t *testClient) Route() routev1.Interface { - return t.route -} - -func (t *testClient) Image() imagev1.Interface { - return t.image -} -func (t *testClient) MCAD() mcadclient.Interface { - return t.mcad -} - -func (t *testClient) Ray() rayclient.Interface { - return t.ray -} - -func (t *testClient) Dynamic() dynamic.Interface { - return t.dynamic -} - -func newTestClient() (Client, error) { - cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), - &clientcmd.ConfigOverrides{}, - ).ClientConfig() - if err != nil { - return nil, err - } - - kubeClient, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, err - } - - machineClient, err := machinev1.NewForConfig(cfg) - if err != nil { - return nil, err - } - - routeClient, err := routev1.NewForConfig(cfg) - if err != nil { - return nil, err - } - - imageClient, err := imagev1.NewForConfig(cfg) - if err != nil { - return nil, err - } - - mcadClient, err := mcadclient.NewForConfig(cfg) - if err != nil { - return nil, err - } - - rayClient, err := rayclient.NewForConfig(cfg) - if err != nil { - return nil, err - } - - dynamicClient, err := dynamic.NewForConfig(cfg) - if err != nil { - return nil, err - } - - return &testClient{ - core: kubeClient, - machine: machineClient, - route: routeClient, - image: imageClient, - mcad: mcadClient, - ray: rayClient, - dynamic: dynamicClient, - }, nil -} diff --git a/test/support/environment.go b/test/support/environment.go deleted file mode 100644 index 7d8e6bc80..000000000 --- a/test/support/environment.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "os" - "strings" -) - -const ( - // The environment variables hereafter can be used to change the components - // used for testing. - - CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" - CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" - CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" - CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE" - - // The testing output directory, to write output files into. - CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR" - - // The namespace where a secret containing InstaScale OCM token is stored and the secret name. - InstaScaleOcmSecret = "INSTASCALE_OCM_SECRET" - - // Cluster ID for OSD cluster used in tests, used for testing InstaScale - OsdClusterID = "CLUSTERID" - - // Type of cluster test is run on - ClusterTypeEnvVar = "CLUSTER_TYPE" -) - -type ClusterType string - -const ( - OsdCluster ClusterType = "OSD" - OcpCluster ClusterType = "OCP" - HypershiftCluster ClusterType = "HYPERSHIFT" - UndefinedCluster ClusterType = "UNDEFINED" -) - -func GetCodeFlareSDKVersion() string { - return lookupEnvOrDefault(CodeFlareTestSdkVersion, CodeFlareSDKVersion) -} - -func GetRayVersion() string { - return lookupEnvOrDefault(CodeFlareTestRayVersion, RayVersion) -} - -func GetRayImage() string { - return lookupEnvOrDefault(CodeFlareTestRayImage, RayImage) -} - -func GetPyTorchImage() string { - return lookupEnvOrDefault(CodeFlareTestPyTorchImage, "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime") -} - -func GetInstascaleOcmSecret() (string, string) { - res := strings.SplitN(lookupEnvOrDefault(InstaScaleOcmSecret, "default/instascale-ocm-secret"), "/", 2) - return res[0], res[1] -} - -func GetOsdClusterId() (string, bool) { - return os.LookupEnv(OsdClusterID) -} - -func GetClusterType(t Test) ClusterType { - clusterType, ok := os.LookupEnv(ClusterTypeEnvVar) - if !ok { - t.T().Logf("Expected environment variable %s not found, cluster type is not defined.", ClusterTypeEnvVar) - return UndefinedCluster - } - switch clusterType { - case "OSD": - return OsdCluster - case "OCP": - return OcpCluster - case "HYPERSHIFT": - return HypershiftCluster - default: - t.T().Logf("Expected environment variable %s contains unexpected value: '%s'", ClusterTypeEnvVar, clusterType) - return UndefinedCluster - } -} - -func lookupEnvOrDefault(key, value string) string { - if v, ok := os.LookupEnv(key); ok { - return v - } - return value -} diff --git a/test/support/machine.go b/test/support/machine.go deleted file mode 100644 index 26acb4bce..000000000 --- a/test/support/machine.go +++ /dev/null @@ -1,32 +0,0 @@ -package support - -import ( - "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - machinev1beta1 "github.com/openshift/api/machine/v1beta1" -) - -func GetMachineSets(t Test) ([]machinev1beta1.MachineSet, error) { - ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms.Items, err -} - -func Machines(t Test, machineSetName string) func(g gomega.Gomega) []machinev1beta1.Machine { - return func(g gomega.Gomega) []machinev1beta1.Machine { - machine, err := t.Client().Machine().MachineV1beta1().Machines("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machineset=" + machineSetName}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return machine.Items - } -} - -func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { - t.T().Helper() - return Machines(t, machineSetName)(t) -} - -func MachineSetId(machineSet machinev1beta1.MachineSet) string { - return machineSet.Name -} diff --git a/test/support/ray.go b/test/support/ray.go deleted file mode 100644 index f18edaa03..000000000 --- a/test/support/ray.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const RayJobDefaultClusterSelectorKey = "ray.io/cluster" - -func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayJob { - return func(g gomega.Gomega) *rayv1.RayJob { - job, err := t.Client().Ray().RayV1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return job - } -} - -func GetRayJob(t Test, namespace, name string) *rayv1.RayJob { - t.T().Helper() - return RayJob(t, namespace, name)(t) -} - -func RayJobStatus(job *rayv1.RayJob) rayv1.JobStatus { - return job.Status.JobStatus -} - -func GetRayJobId(t Test, namespace, name string) string { - t.T().Helper() - job := RayJob(t, namespace, name)(t) - return job.Status.JobId -} - -func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayCluster { - return func(g gomega.Gomega) *rayv1.RayCluster { - cluster, err := t.Client().Ray().RayV1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return cluster - } -} - -func GetRayCluster(t Test, namespace, name string) *rayv1.RayCluster { - t.T().Helper() - return RayCluster(t, namespace, name)(t) -} - -func RayClusterState(cluster *rayv1.RayCluster) rayv1.ClusterState { - return cluster.Status.State -} - -func WriteRayJobLogs(t Test, rayClient RayClusterClient, namespace, name string) { - WriteRayJobAPILogs(t, rayClient, GetRayJobId(t, namespace, name)) -} From 7c8b0fd13f38506285f9dcc454b37be785606a37 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 24 Oct 2023 13:16:25 +0100 Subject: [PATCH 036/369] update go.mod and go.sum --- go.mod | 5 +++-- go.sum | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 40254116f..912d2a1b2 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,8 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 - github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c + github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 github.com/project-codeflare/instascale v0.2.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 @@ -70,6 +69,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect + github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/go.sum b/go.sum index 867260a31..c2efac624 100644 --- a/go.sum +++ b/go.sum @@ -391,6 +391,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k= +github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.2.1 h1:t+Ax/sk4yPQznO6N+U8Cq+bk3afCcZDj9wnHfiGSDBg= github.com/project-codeflare/instascale v0.2.1/go.mod h1:zSzBTP4cFkg+bD4JyYTDmDnGwVKY/+6ACks57NAiscc= github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 h1:hZhGwKTPeHYYhNbvO27NOjozVpy7m3I3apKf81u9U3A= From 1d5c85d366f29a5a898b177765175f3139f8e48e Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 24 Oct 2023 13:16:59 +0100 Subject: [PATCH 037/369] addressing feedback --- .github/workflows/tag-and-build.yml | 2 -- Makefile | 6 ------ 2 files changed, 8 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 2ecdfb722..54f57a64f 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -93,10 +93,8 @@ jobs: - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | # Remove leading 'v' - CODEFLARE_SDK_VERSION=$(cut -c2- <<< ${{ github.event.inputs.codeflare-sdk-version }}) sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile - sed -i -E "s/(.*CODEFLARE_SDK_VERSION \?= ).*/\1$CODEFLARE_SDK_VERSION/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile - name: Login to Quay.io diff --git a/Makefile b/Makefile index 541fba37b..9f8dcbfb3 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,6 @@ KUBERAY_VERSION ?= v1.0.0-rc.1 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 -# CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.10.1 - # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to OPERATORS_REPO_ORG ?= redhat-openshift-ecosystem @@ -64,9 +61,6 @@ IMAGE_ORG_BASE ?= quay.io/project-codeflare # codeflare.dev/codeflare-operator-bundle:$VERSION and codeflare.dev/codeflare-operator-catalog:$VERSION. IMAGE_TAG_BASE ?= $(IMAGE_ORG_BASE)/codeflare-operator -# RAY_IMAGE defines the default container image for Ray (used for testing) -RAY_IMAGE ?= rayproject/ray:$(RAY_VERSION) - # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION) From ff2a9db5621061d9ef5e259bd7ef4100ecb686a8 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Tue, 24 Oct 2023 11:45:45 -0400 Subject: [PATCH 038/369] Update GRPC version for CVE Signed-off-by: Anish Asthana --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 912d2a1b2..51d0a8566 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/grpc v1.58.2 // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index c2efac624..9f8c35a36 100644 --- a/go.sum +++ b/go.sum @@ -865,8 +865,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= -google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From d851f6765a671928ebc80a962aa5220b666ce264 Mon Sep 17 00:00:00 2001 From: Srihari Date: Fri, 20 Oct 2023 17:36:26 +0530 Subject: [PATCH 039/369] Set security context on test pod to avoid pod security warning --- test/e2e/mnist_raycluster_sdk_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index b8cc5af7e..aeede52e8 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -175,6 +175,16 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, WorkingDir: "/workdir", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: Ptr(false), + SeccompProfile: &corev1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: Ptr(true), + }, }, }, Volumes: []corev1.Volume{ From 487244c25f3eab06cee534b12cefe11cb93e976e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 24 Oct 2023 12:26:23 +0200 Subject: [PATCH 040/369] Use kuberay/ray-operator@vX.Y.Z Go module --- Makefile | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9f8dcbfb3..19776cfba 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ vet: ## Run go vet against code. modules: ## Update Go dependencies. go get $(MCAD_REPO)@$(MCAD_VERSION) go get $(INSTASCALE_REPO)@$(INSTASCALE_VERSION) - go get github.com/ray-project/kuberay/ray-operator + go get github.com/ray-project/kuberay/ray-operator@$(KUBERAY_VERSION) go mod tidy .PHONY: build diff --git a/go.mod b/go.mod index 51d0a8566..124c82115 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 github.com/project-codeflare/instascale v0.2.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 - github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 + github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index 9f8c35a36..729858b68 100644 --- a/go.sum +++ b/go.sum @@ -424,8 +424,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 h1:s+/WzpqoFot/fgo1uq7BLgeiOy3HZuT+UVtCb3h7R+I= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 h1:aoOajXNG2QnTPmnSzbCXLFBXrVk7bzQGrwwVzTXSOzo= +github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= From 15340def5ea22c3426378a607b836d6e1f32b812 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 31 Oct 2023 14:24:31 +0100 Subject: [PATCH 041/369] Remove automatic triage/needs-triage label creation --- .github/workflows/auto-add-issues.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/auto-add-issues.yml b/.github/workflows/auto-add-issues.yml index a8be31eb4..c1e940997 100644 --- a/.github/workflows/auto-add-issues.yml +++ b/.github/workflows/auto-add-issues.yml @@ -4,17 +4,6 @@ on: types: - opened jobs: - add_label: - name: Add needs-triage label to new issues - runs-on: ubuntu-latest - permissions: - issues: write - steps: - - uses: actions/checkout@v3 - - run: gh issue edit ${{ github.event.issue.number }} --add-label "triage/needs-triage" - env: - GH_TOKEN: ${{ github.token }} - add-to-project: name: Add issue to project runs-on: ubuntu-latest From 5c99252a263aa46e07aad4cf1d2c22ae73e07ab6 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 16:25:34 +0000 Subject: [PATCH 042/369] Added context to InstaScale Setup --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 6173b8533..5e7065766 100644 --- a/main.go +++ b/main.go @@ -154,7 +154,7 @@ func main() { Scheme: mgr.GetScheme(), Config: cfg.InstaScale.InstaScaleConfiguration, } - exitOnError(instaScaleController.SetupWithManager(mgr), "Error setting up InstaScale controller") + exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") } exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") From b48815e8c66473f834dc121434399455999f7985 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 16:37:41 +0000 Subject: [PATCH 043/369] Update gomod --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 124c82115..932d7614c 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 - github.com/project-codeflare/instascale v0.2.1 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 + github.com/project-codeflare/instascale v0.3.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 729858b68..c5ad83a71 100644 --- a/go.sum +++ b/go.sum @@ -393,10 +393,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k= github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= -github.com/project-codeflare/instascale v0.2.1 h1:t+Ax/sk4yPQznO6N+U8Cq+bk3afCcZDj9wnHfiGSDBg= -github.com/project-codeflare/instascale v0.2.1/go.mod h1:zSzBTP4cFkg+bD4JyYTDmDnGwVKY/+6ACks57NAiscc= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 h1:hZhGwKTPeHYYhNbvO27NOjozVpy7m3I3apKf81u9U3A= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs= +github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 4930dd50efa66e840f4dce437e4f705c16fa3b69 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 17:37:10 +0000 Subject: [PATCH 044/369] Removed defaults.go reference --- .github/workflows/tag-and-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 54f57a64f..a96567c8a 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -138,7 +138,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md */defaults.go *.yaml Makefile go.mod go.sum' + file_pattern: 'README.md *.yaml Makefile go.mod go.sum' create_branch: true branch: ${{ env.PR_BRANCH_NAME }} From 1c7f2290cb6226aa8eda6d8f33ae748d3227a700 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 18:32:35 +0000 Subject: [PATCH 045/369] Update dependency versions for release v1.0.0 --- Makefile | 4 ++-- README.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 19776cfba..f62aa93ac 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.2.1 +INSTASCALE_VERSION ?= v0.3.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.37.1 +MCAD_VERSION ?= v1.38.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index a21c16475..5cac00319 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0-rc.4](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.4) | -| Multi-Cluster App Dispatcher | [v1.37.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.1) | -| CodeFlare-SDK | [v0.10.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.1) | -| InstaScale | [v0.2.1](https://github.com/project-codeflare/instascale/releases/tag/v0.2.1) | +| CodeFlare Operator | [v1.0.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0) | +| Multi-Cluster App Dispatcher | [v1.38.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.0) | +| CodeFlare-SDK | [v0.11.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.11.0) | +| InstaScale | [v0.3.0](https://github.com/project-codeflare/instascale/releases/tag/v0.3.0) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | From fd87dcd8969ffc1936b1af8dcd6e69a368648815 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 1 Nov 2023 10:33:01 +0000 Subject: [PATCH 046/369] Updated KubeRay version in Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cac00319..d8cdd53dc 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ CodeFlare Stack Compatibility Matrix | Multi-Cluster App Dispatcher | [v1.38.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.0) | | CodeFlare-SDK | [v0.11.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.11.0) | | InstaScale | [v0.3.0](https://github.com/project-codeflare/instascale/releases/tag/v0.3.0) | -| KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | +| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | ## Development From 2e3d952a2802d6940394818db7299c3d0519ba13 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 1 Nov 2023 16:35:59 +0100 Subject: [PATCH 047/369] Fix broken catalog-build-from-index command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f62aa93ac..dff50cba0 100644 --- a/Makefile +++ b/Makefile @@ -331,7 +331,7 @@ catalog-build-from-index: opm ## Build a catalog image. mkdir catalog $(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml $(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml - $(SED) -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml + $(SED) -i -E "s/(.*)(- name: codeflare-operator.v0.2.0)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml $(OPM) validate catalog $(OPM) generate dockerfile catalog podman build . -f catalog.Dockerfile -t $(CATALOG_IMG) From 719b6bdb4c868f4ec512373738665a0d187e3b06 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 23 Oct 2023 10:07:03 -0400 Subject: [PATCH 048/369] remove scorecard gh action Signed-off-by: Kevin --- .github/workflows/olm_scorecard.yml | 28 ---------------------------- Makefile | 4 ---- 2 files changed, 32 deletions(-) delete mode 100644 .github/workflows/olm_scorecard.yml diff --git a/.github/workflows/olm_scorecard.yml b/.github/workflows/olm_scorecard.yml deleted file mode 100644 index ba83b801f..000000000 --- a/.github/workflows/olm_scorecard.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Operator Scorecard Test - -on: [push, pull_request] - -jobs: - scorecard-check: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Setup and start KinD cluster - uses: ./.github/actions/kind - - - name: Build bundle - run: make bundle-build - - - name: Install yq - run: | - sudo wget -O /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64 - sudo chmod +x /usr/bin/yq - - - name: Modify scorecard config - run: | - yq e 'del(.stages[].tests[] | select(.labels.test == "olm-crds-have-resources-test"))' -i bundle/tests/scorecard/config.yaml - - - name: Run Operator SDK Scorecard - run: make scorecard-bundle diff --git a/Makefile b/Makefile index dff50cba0..a52a4e798 100644 --- a/Makefile +++ b/Makefile @@ -364,7 +364,3 @@ imports: openshift-goimports ## Organize imports in go files using openshift-goi .PHONY: verify-imports verify-imports: openshift-goimports ## Run import verifications. ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) - -.PHONY: scorecard-bundle -scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. - $(OPERATOR_SDK) scorecard bundle From 544d1e7bafbf709b0cc0bc41e1d60ffce9a0d831 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 1 Nov 2023 13:12:26 +0100 Subject: [PATCH 049/369] Regenerate CFO CRD manifests for release --- .github/workflows/tag-and-build.yml | 5 ++++- config/crd/mcad/kustomization.yaml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index a96567c8a..f98129302 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -92,7 +92,6 @@ jobs: - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | - # Remove leading 'v' sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile @@ -134,6 +133,10 @@ jobs: run: | go mod tidy + - name: Regenerate CFO CRD manifests + run: | + make manifests + - name: Commit changes in the code back to repository uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index e53d2dd94..23063b679 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.37.1 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.0 From bfadb47830846a307be2a3ce7b30eb0726178bee Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 2 Nov 2023 16:23:52 -0400 Subject: [PATCH 050/369] Use CGO_ENABLED=1 for FIPS compliance reasons Signed-off-by: Anish Asthana --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ab7210993..93f3f3327 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ COPY pkg/ pkg/ # Build USER root -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o manager main.go FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / From 57568b53384fc386d9e92119e8f2223425b5f383 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Mon, 6 Nov 2023 16:53:38 -0500 Subject: [PATCH 051/369] Add stricttag to pass fips check warnings Signed-off-by: Anish Asthana --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 93f3f3327..ea02d1b1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ COPY pkg/ pkg/ # Build USER root -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o manager main.go +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -tags strictfipsruntime -a -o manager main.go FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / From 3885aef496d62a4d220ef2105c3a01a044b6042e Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 17:24:27 +0100 Subject: [PATCH 052/369] Added launch.json for local debugging with VSCode --- .gitignore | 3 +++ .vscode/launch.json | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.gitignore b/.gitignore index 898d07796..3126962c1 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ bundle.Dockerfile *~ .DS_STORE + +#VSCode +.vscode diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..7357bfc75 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug CodeFlare Operator", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "env": { + "KUBECONFIG": "", + "NAMESPACE": "", + }, + "showLog": true + }, + ] + } From 25b40b24aa0f465081e22b823180cb0992c20beb Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 17:33:46 +0100 Subject: [PATCH 053/369] Fixed Json --- .vscode/launch.json | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7357bfc75..731487da5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,17 +1,17 @@ { - "version": "0.2.0", - "configurations": [ - { - "name": "Debug CodeFlare Operator", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/main.go", - "env": { - "KUBECONFIG": "", - "NAMESPACE": "", - }, - "showLog": true + "version": "0.2.0", + "configurations": [ + { + "name": "Debug CodeFlare Operator", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "env": { + "KUBECONFIG": "", + "NAMESPACE": "" }, - ] - } + "showLog": true + } + ] +} From cea4ceb4fd3542fc1513f35632f653122ca78674 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 25 Oct 2023 09:30:01 +0100 Subject: [PATCH 054/369] Added instructions to CONTRIBUTING.md --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83edd2f9b..df63e1796 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,3 +56,9 @@ To write and inspect unit tests: - MCAD and InstaScale unit tests under `mcad_controller_test.go` and `instascale_controller_test.go` in the `controllers` dir - Unit test functions are defined in `suite_test.go` (with utils in `util/util.go`) in the `controllers dir` - Test cases defined under `controllers/testdata` + + ## Local debugging with VSCode + Steps outlining how to run the operator locally. + - Populate the `.vscode/launch.json` file with the location of your Kubernetes config file and desired namespace. + - In VSCode on the activity bar click `Run and Debug` or `CTRL + SHIFT + D` to start a local debugging session of the CodeFlare Operator. + The operator should be running as intended. From b0f15b871219902fb80c4575b0a04707a372208c Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 1 Nov 2023 10:56:50 +0000 Subject: [PATCH 055/369] Review changes links, additional info --- .gitignore | 2 +- CONTRIBUTING.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3126962c1..9be9b01b6 100644 --- a/.gitignore +++ b/.gitignore @@ -30,5 +30,5 @@ bundle.Dockerfile .DS_STORE -#VSCode +# VSCode .vscode diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df63e1796..9fd4a3f8b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,6 +59,7 @@ To write and inspect unit tests: ## Local debugging with VSCode Steps outlining how to run the operator locally. - - Populate the `.vscode/launch.json` file with the location of your Kubernetes config file and desired namespace. + - Ensure you are authenticated to your Kubernetes/OpenShift Cluster. + - Populate the [.vscode/launch.json](https://github.com/project-codeflare/codeflare-operator/tree/main/.vscode/launch.json) file with the location of your Kubernetes config file and desired namespace. - In VSCode on the activity bar click `Run and Debug` or `CTRL + SHIFT + D` to start a local debugging session of the CodeFlare Operator. The operator should be running as intended. From 5f826e0003633a8d88558ada2390bc221f8f55f0 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Tue, 7 Nov 2023 13:43:59 +0530 Subject: [PATCH 056/369] Update workflow to add kuberay in compatibility matrix --- .github/workflows/tag-and-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index f98129302..ed7b1769f 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -24,6 +24,10 @@ on: description: 'Published version of InstaScale' required: true default: 'v0.0.0-dev' + kuberay-version: + description: 'Published version of KubeRay' + required: true + default: 'v0.0.0-dev' is-stable: description: 'Select if the built image should be tagged as stable' required: true @@ -89,6 +93,7 @@ jobs: sed -i -E "s|(.*Multi-Cluster App Dispatcher.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.mcad-version }}\2${{ github.event.inputs.mcad-version }}\3|" README.md sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md sed -i -E "s|(.*InstaScale.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.instascale-version }}\2${{ github.event.inputs.instascale-version }}\3|" README.md + sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | From e95759a6fea725760a6a8a9dbfd7331fd0b4bf08 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Tue, 7 Nov 2023 15:56:58 +0530 Subject: [PATCH 057/369] change description --- .github/workflows/tag-and-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index ed7b1769f..827a33c66 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -25,7 +25,7 @@ on: required: true default: 'v0.0.0-dev' kuberay-version: - description: 'Published version of KubeRay' + description: 'Tested version of KubeRay' required: true default: 'v0.0.0-dev' is-stable: From 72cace9ed1adbd91a22675516e237dd94f2d4d79 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 8 Nov 2023 13:47:46 +0100 Subject: [PATCH 058/369] Enable TestMNISTRayClusterSDK test --- .github/actions/kind/action.yml | 16 ++ go.mod | 2 +- go.sum | 4 +- test/e2e/mnist_raycluster_sdk.py | 24 ++- test/e2e/mnist_raycluster_sdk_test.go | 151 ++++++++---------- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 4 +- 6 files changed, 107 insertions(+), 94 deletions(-) diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml index 59dcafef7..f76e60af5 100644 --- a/.github/actions/kind/action.yml +++ b/.github/actions/kind/action.yml @@ -1,6 +1,12 @@ name: "Set up KinD" description: "Step to start and configure KinD cluster" +inputs: + kind-node-hostname: + description: "Hostname of the main kind node" + required: false + default: kind + runs: using: "composite" steps: @@ -56,3 +62,13 @@ runs: curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n - --status-update-interval=10/g" | kubectl apply -f - kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true" kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all + + - name: Add ${{ inputs.kind-node-hostname }} host to machine hosts + shell: bash + run: echo "127.0.0.1 ${{ inputs.kind-node-hostname }}" | sudo tee -a /etc/hosts + + - name: Set env variables for tests to properly leverage KinD cluster + shell: bash + run: | + echo "CLUSTER_TYPE=KIND" >> $GITHUB_ENV + echo "CLUSTER_HOSTNAME=${{ inputs.kind-node-hostname }}" >> $GITHUB_ENV diff --git a/go.mod b/go.mod index 932d7614c..98011af73 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 + github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb github.com/project-codeflare/instascale v0.3.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 diff --git a/go.sum b/go.sum index c5ad83a71..423343034 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k= -github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= +github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb h1:L2Gdr2SlvshDKZY2KK6507AwzQ1NSfRbMQuz5dOsYNM= +github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs= github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI= github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g= diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py index b830a0040..cb3c0af56 100644 --- a/test/e2e/mnist_raycluster_sdk.py +++ b/test/e2e/mnist_raycluster_sdk.py @@ -1,4 +1,5 @@ import sys +import os from time import sleep @@ -8,17 +9,38 @@ from codeflare_sdk.job.jobs import DDPJobDefinition namespace = sys.argv[1] +ray_image = os.getenv('RAY_IMAGE') +host = os.getenv('CLUSTER_HOSTNAME') + +ingress_options = {} +if host is not None: + ingress_options = { + "ingresses": [ + { + "ingressName": "ray-dashboard", + "port": 8265, + "pathType": "Prefix", + "path": "/", + "host": host, + }, + ] + } + cluster = Cluster(ClusterConfiguration( name='mnist', namespace=namespace, num_workers=1, + head_cpus='500m', + head_memory=2, min_cpus='500m', max_cpus=1, min_memory=0.5, - max_memory=1, + max_memory=2, num_gpus=0, instascale=False, + image=ray_image, + ingress_options=ingress_options, )) cluster.up() diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index aeede52e8..015cbfc58 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -40,104 +40,59 @@ func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() - // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released - test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251") - // Create a namespace namespace := test.NewTestNamespace() // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-raycluster-sdk", - Namespace: namespace.Name, + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // SDK script + "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, }, - BinaryData: map[string][]byte{ - // SDK script - "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // SDK client RBAC - serviceAccount := &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ServiceAccount", + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk-user", - Namespace: namespace.Name, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, }, } - serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - role := &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "Role", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Rules: []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, }, } - role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - roleBinding := &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "RoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacv1.SchemeGroupVersion.Group, - Kind: "Role", - Name: role.Name, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - APIGroup: corev1.SchemeGroupVersion.Group, - Name: serviceAccount.Name, - Namespace: serviceAccount.Namespace, - }, - }, - } - _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, sa, clusterRole) job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -161,7 +116,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { // See https://github.com/project-codeflare/codeflare-sdk/pull/146 Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "RAY_IMAGE", Value: GetRayImage()}, }, Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ @@ -206,12 +162,31 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, + ServiceAccountName: sa.Name, }, }, }, } - job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + if GetClusterType(test) == KindCluster { + // Take first KinD node and redirect pod hostname requests there + node := GetNodes(test)[0] + hostname := GetClusterHostname(test) + IP := GetNodeInternalIP(test, node) + + test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) + job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ + { + IP: IP, + Hostnames: []string{hostname}, + }, + } + + // Propagate hostname into Python code as env variable + hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + } + + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 725ced9db..b8d3f4d06 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -108,7 +108,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), + corev1.ResourceMemory: resource.MustParse("2G"), }, }, VolumeMounts: []corev1.VolumeMount{ @@ -168,7 +168,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), + corev1.ResourceMemory: resource.MustParse("2G"), }, }, }, From 4366322e7e89422ca32af488c35cb372c2f93aff Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 30 Oct 2023 15:53:44 +0100 Subject: [PATCH 059/369] Setup existing DW tests to run in CFO --- test/odh/environment.go | 49 +++++++ test/odh/mcad_ray_test.go | 98 ++++++++++++++ test/odh/notebook.go | 99 ++++++++++++++ test/odh/pytorch_mcad_test.go | 77 +++++++++++ test/odh/resources/custom-nb-small.yaml | 165 +++++++++++++++++++++++ test/odh/resources/mnist.py | 160 ++++++++++++++++++++++ test/odh/resources/mnist_mcad_mini.ipynb | 93 +++++++++++++ test/odh/resources/mnist_ray_mini.ipynb | 145 ++++++++++++++++++++ test/odh/resources/requirements.txt | 4 + test/odh/support.go | 34 +++++ 10 files changed, 924 insertions(+) create mode 100644 test/odh/environment.go create mode 100644 test/odh/mcad_ray_test.go create mode 100644 test/odh/notebook.go create mode 100644 test/odh/pytorch_mcad_test.go create mode 100644 test/odh/resources/custom-nb-small.yaml create mode 100644 test/odh/resources/mnist.py create mode 100644 test/odh/resources/mnist_mcad_mini.ipynb create mode 100644 test/odh/resources/mnist_ray_mini.ipynb create mode 100644 test/odh/resources/requirements.txt create mode 100644 test/odh/support.go diff --git a/test/odh/environment.go b/test/odh/environment.go new file mode 100644 index 000000000..0a087c9d5 --- /dev/null +++ b/test/odh/environment.go @@ -0,0 +1,49 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "os" + + . "github.com/project-codeflare/codeflare-common/support" +) + +const ( + // The environment variable for namespace where ODH is installed to. + odhNamespaceEnvVar = "ODH_NAMESPACE" + // The environment variable for ODH Notebook ImageStream name + notebookImageStreamName = "NOTEBOOK_IMAGE_STREAM_NAME" +) + +func GetOpenDataHubNamespace() string { + return lookupEnvOrDefault(odhNamespaceEnvVar, "opendatahub") +} + +func GetNotebookImageStreamName(t Test) string { + isName, ok := os.LookupEnv(notebookImageStreamName) + if !ok { + t.T().Fatalf("Expected environment variable %s not found, please use this environment variable to specify what ImageStream to use for Notebook.", notebookImageStreamName) + } + return isName +} + +func lookupEnvOrDefault(key, value string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return value +} diff --git a/test/odh/mcad_ray_test.go b/test/odh/mcad_ray_test.go new file mode 100644 index 000000000..770b64d9d --- /dev/null +++ b/test/odh/mcad_ray_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + + rbacv1 "k8s.io/api/rbac/v1" +) + +func TestMCADRay(t *testing.T) { + test := With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Test configuration + jupyterNotebookConfigMapFileName := "mnist_ray_mini.ipynb" + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // MNIST Ray Notebook + jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"), + "mnist.py": ReadFile(test, "resources/mnist.py"), + "requirements.txt": ReadFile(test, "resources/requirements.txt"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + } + + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, + }, + } + + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, sa, clusterRole) + token := CreateToken(test, namespace.Name, sa) + + // Create Notebook CR + createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) + + // Make sure the AppWrapper is created and running + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should( + And( + HaveLen(1), + ContainElement(WithTransform(AppWrapperName, HavePrefix("mnisttest"))), + ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ), + ) + + // Make sure the AppWrapper finishes and is deleted + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should(HaveLen(0)) +} diff --git a/test/odh/notebook.go b/test/odh/notebook.go new file mode 100644 index 000000000..8c7b28275 --- /dev/null +++ b/test/odh/notebook.go @@ -0,0 +1,99 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "bytes" + "html/template" + + gomega "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + + imagev1 "github.com/openshift/api/image/v1" +) + +const recommendedTagAnnotation = "opendatahub.io/workbench-image-recommended" + +var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"} + +type NotebookProps struct { + IngressDomain string + OpenShiftApiUrl string + KubernetesBearerToken string + Namespace string + OpenDataHubNamespace string + ImageStreamName string + ImageStreamTag string + NotebookConfigMapName string + NotebookConfigMapFileName string + NotebookPVC string +} + +func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string) { + // Create PVC for Notebook + notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce) + + // Retrieve ImageStream tag for + is := GetImageStream(test, GetOpenDataHubNamespace(), GetNotebookImageStreamName(test)) + recommendedTagName := getRecommendedImageStreamTag(test, is) + + // Read the Notebook CR from resources and perform replacements for custom values using go template + notebookProps := NotebookProps{ + IngressDomain: GetOpenShiftIngressDomain(test), + OpenShiftApiUrl: GetOpenShiftApiUrl(test), + KubernetesBearerToken: notebookToken, + Namespace: namespace.Name, + OpenDataHubNamespace: GetOpenDataHubNamespace(), + ImageStreamName: GetNotebookImageStreamName(test), + ImageStreamTag: recommendedTagName, + NotebookConfigMapName: jupyterNotebookConfigMapName, + NotebookConfigMapFileName: jupyterNotebookConfigMapFileName, + NotebookPVC: notebookPVC.Name, + } + notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml") + test.Expect(err).NotTo(gomega.HaveOccurred()) + parsedNotebookTemplate, err := template.New("notebook").Parse(string(notebookTemplate)) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + // Filter template and store results to the buffer + notebookBuffer := new(bytes.Buffer) + err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create Notebook CR + notebookCR := &unstructured.Unstructured{} + err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR) + test.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func getRecommendedImageStreamTag(test Test, is *imagev1.ImageStream) (tagName string) { + for _, tag := range is.Spec.Tags { + if tag.Annotations[recommendedTagAnnotation] == "true" { + return tag.Name + } + } + test.T().Fatalf("tag with annotation '%s' not found in ImageStream %s", recommendedTagAnnotation, is.Name) + return +} diff --git a/test/odh/pytorch_mcad_test.go b/test/odh/pytorch_mcad_test.go new file mode 100644 index 000000000..0dd33a363 --- /dev/null +++ b/test/odh/pytorch_mcad_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + rbacv1 "k8s.io/api/rbac/v1" +) + +func TestMnistPyTorchMCAD(t *testing.T) { + test := With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Test configuration + jupyterNotebookConfigMapFileName := "mnist_mcad_mini.ipynb" + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // MNIST MCAD Notebook + jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_mcad_mini.ipynb"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + // Needed for job.logs() + { + Verbs: []string{"get"}, + APIGroups: []string{""}, + Resources: []string{"pods/log"}, + }, + } + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + token := CreateToken(test, namespace.Name, sa) + + // Create Notebook CR + createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) + + // Make sure the AppWrapper is created and running + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should( + And( + HaveLen(1), + ContainElement(WithTransform(AppWrapperName, HavePrefix("mnistjob"))), + ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ), + ) + + // Make sure the AppWrapper finishes and is deleted + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should(HaveLen(0)) +} diff --git a/test/odh/resources/custom-nb-small.yaml b/test/odh/resources/custom-nb-small.yaml new file mode 100644 index 000000000..95aaaf106 --- /dev/null +++ b/test/odh/resources/custom-nb-small.yaml @@ -0,0 +1,165 @@ +# This template maybe used to spin up a custom notebook image +# i.e.: sed s/{{.IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f - +# resources generated: +# pod/jupyter-nb-kube-3aadmin-0 +# service/jupyter-nb-kube-3aadmin +# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com) +# service/jupyter-nb-kube-3aadmin-tls +apiVersion: kubeflow.org/v1 +kind: Notebook +metadata: + annotations: + notebooks.opendatahub.io/inject-oauth: "true" + notebooks.opendatahub.io/last-image-selection: codeflare-notebook:latest + notebooks.opendatahub.io/last-size-selection: Small + notebooks.opendatahub.io/oauth-logout-url: https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home + opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{.Namespace}}.{{.IngressDomain}}/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin + opendatahub.io/username: kube:admin + generation: 1 + labels: + app: jupyter-nb-kube-3aadmin + opendatahub.io/dashboard: "true" + opendatahub.io/odh-managed: "true" + opendatahub.io/user: kube-3aadmin + name: jupyter-nb-kube-3aadmin + namespace: {{.Namespace}} +spec: + template: + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: nvidia.com/gpu.present + operator: NotIn + values: + - "true" + weight: 1 + containers: + - env: + - name: NOTEBOOK_ARGS + value: |- + --ServerApp.port=8888 + --ServerApp.token='' + --ServerApp.password='' + --ServerApp.base_url=/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin + --ServerApp.quit_button=False + --ServerApp.tornado_settings={"user":"kube-3aadmin","hub_host":"https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}","hub_prefix":"/notebookController/kube-3aadmin"} + - name: JUPYTER_IMAGE + value: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} + - name: JUPYTER_NOTEBOOK_PORT + value: "8888" + - name: OCP_SERVER + value: {{.OpenShiftApiUrl}} + - name: OCP_TOKEN + value: {{.KubernetesBearerToken}} + image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} + command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} && sleep infinity"] + # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ] + imagePullPolicy: Always + # livenessProbe: + # failureThreshold: 3 + # httpGet: + # path: /notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin/api + # port: notebook-port + # scheme: HTTP + # initialDelaySeconds: 10 + # periodSeconds: 5 + # successThreshold: 1 + # timeoutSeconds: 1 + name: jupyter-nb-kube-3aadmin + ports: + - containerPort: 8888 + name: notebook-port + protocol: TCP + resources: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 3Gi + volumeMounts: + - mountPath: /opt/app-root/src + name: jupyterhub-nb-kube-3aadmin-pvc + - mountPath: /opt/app-root/notebooks + name: {{.NotebookConfigMapName}} + workingDir: /opt/app-root/src + - args: + - --provider=openshift + - --https-address=:8443 + - --http-address= + - --openshift-service-account=jupyter-nb-kube-3aadmin + - --cookie-secret-file=/etc/oauth/config/cookie_secret + - --cookie-expire=24h0m0s + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:8888 + - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - --skip-auth-regex=^(?:/notebook/$(NAMESPACE)/jupyter-nb-kube-3aadmin)?/api$ + - --email-domain=* + - --skip-provider-button + - --openshift-sar={"verb":"get","resource":"notebooks","resourceAPIGroup":"kubeflow.org","resourceName":"jupyter-nb-kube-3aadmin","namespace":"$(NAMESPACE)"} + - --logout-url=https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.redhat.io/openshift4/ose-oauth-proxy:v4.10 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: oauth-proxy + ports: + - containerPort: 8443 + name: oauth-proxy + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 100m + memory: 64Mi + volumeMounts: + - mountPath: /etc/oauth/config + name: oauth-config + - mountPath: /etc/tls/private + name: tls-certificates + enableServiceLinks: false + serviceAccountName: jupyter-nb-kube-3aadmin + volumes: + - name: jupyterhub-nb-kube-3aadmin-pvc + persistentVolumeClaim: + claimName: {{.NotebookPVC}} + - name: oauth-config + secret: + defaultMode: 420 + secretName: jupyter-nb-kube-3aadmin-oauth-config + - name: tls-certificates + secret: + defaultMode: 420 + secretName: jupyter-nb-kube-3aadmin-tls + - name: {{.NotebookConfigMapName}} + configMap: + name: {{.NotebookConfigMapName}} diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py new file mode 100644 index 000000000..d6a211944 --- /dev/null +++ b/test/odh/resources/mnist.py @@ -0,0 +1,160 @@ +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from pytorch_lightning import LightningModule, Trainer +from pytorch_lightning.callbacks.progress import TQDMProgressBar +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader, random_split +from torchmetrics import Accuracy +from torchvision import transforms +from torchvision.datasets import MNIST + +PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") +BATCH_SIZE = 256 if torch.cuda.is_available() else 64 +# %% + +print("prior to running the trainer") +print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) +print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) + + +class LitMNIST(LightningModule): + def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): + + super().__init__() + + # Set our init args as class attributes + self.data_dir = data_dir + self.hidden_size = hidden_size + self.learning_rate = learning_rate + + # Hardcode some dataset specific attributes + self.num_classes = 10 + self.dims = (1, 28, 28) + channels, width, height = self.dims + self.transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + ] + ) + + # Define PyTorch model + self.model = nn.Sequential( + nn.Flatten(), + nn.Linear(channels * width * height, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, self.num_classes), + ) + + self.val_accuracy = Accuracy() + self.test_accuracy = Accuracy() + + def forward(self, x): + x = self.model(x) + return F.log_softmax(x, dim=1) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.val_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("val_loss", loss, prog_bar=True) + self.log("val_acc", self.val_accuracy, prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.test_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("test_loss", loss, prog_bar=True) + self.log("test_acc", self.test_accuracy, prog_bar=True) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + #################### + # DATA RELATED HOOKS + #################### + + def prepare_data(self): + # download + print("Downloading MNIST dataset...") + MNIST(self.data_dir, train=True, download=True) + MNIST(self.data_dir, train=False, download=True) + + def setup(self, stage=None): + + # Assign train/val datasets for use in dataloaders + if stage == "fit" or stage is None: + mnist_full = MNIST(self.data_dir, train=True, transform=self.transform) + self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) + + # Assign test dataset for use in dataloader(s) + if stage == "test" or stage is None: + self.mnist_test = MNIST( + self.data_dir, train=False, transform=self.transform + ) + + def train_dataloader(self): + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + + def val_dataloader(self): + return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) + + def test_dataloader(self): + return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) + + +# Init DataLoader from MNIST Dataset + +model = LitMNIST() + +print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) +print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) + +# Initialize a trainer +trainer = Trainer( + accelerator="auto", + # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs + max_epochs=2, + callbacks=[TQDMProgressBar(refresh_rate=20)], + num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), + devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + strategy="ddp", +) + +# Train the model ⚡ +trainer.fit(model) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb new file mode 100644 index 000000000..0b53324ab --- /dev/null +++ b/test/odh/resources/mnist_mcad_mini.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\n", + "from codeflare_sdk.job.jobs import DDPJobDefinition\n", + "from time import sleep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47ca5c15", + "metadata": { + "tags": ["parameters"] + }, + "outputs": [], + "source": [ + "#parameters\n", + "namespace = \"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26b21373", + "metadata": {}, + "outputs": [], + "source": [ + "job = DDPJobDefinition(name=\"mnistjob\", script=\"mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/mnist-job-test:v0.0.1\").submit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d24e9f95", + "metadata": {}, + "outputs": [], + "source": [ + "finished = False\n", + "while not finished:\n", + " sleep(1)\n", + " try:\n", + " finished = (\"Epoch 4: 100%\" in job.logs())\n", + " except:\n", + " finished = False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f078b7cd", + "metadata": {}, + "outputs": [], + "source": [ + "job.cancel()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/odh/resources/mnist_ray_mini.ipynb b/test/odh/resources/mnist_ray_mini.ipynb new file mode 100644 index 000000000..38992cc7d --- /dev/null +++ b/test/odh/resources/mnist_ray_mini.ipynb @@ -0,0 +1,145 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\n", + "from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration\n", + "from codeflare_sdk.job.jobs import DDPJobDefinition\n", + "from time import sleep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30888aed", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#parameters\n", + "namespace = \"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f4bc870-091f-4e11-9642-cba145710159", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Create our cluster and submit appwrapper\n", + "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Bring up the cluster\n", + "cluster.up()\n", + "cluster.wait_ready()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df71c1ed", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cluster.status()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cluster.details()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47ca5c15", + "metadata": {}, + "outputs": [], + "source": [ + "job = DDPJobDefinition(name=\"mnisttest\", script=\"mnist.py\", workspace=\"file:///opt/app-root/notebooks/..data\", scheduler_args={\"requirements\": \"/opt/app-root/notebooks/requirements.txt\"}).submit(cluster)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f63a178a", + "metadata": {}, + "outputs": [], + "source": [ + "finished = False\n", + "while not finished:\n", + " sleep(1)\n", + " status = job.status()\n", + " finished = (str(status.state) == \"SUCCEEDED\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b099777", + "metadata": {}, + "outputs": [], + "source": [ + "cluster.down()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/odh/resources/requirements.txt b/test/odh/resources/requirements.txt new file mode 100644 index 000000000..7266b064a --- /dev/null +++ b/test/odh/resources/requirements.txt @@ -0,0 +1,4 @@ +pytorch_lightning==1.5.10 +ray_lightning +torchmetrics==0.9.1 +torchvision==0.12.0 diff --git a/test/odh/support.go b/test/odh/support.go new file mode 100644 index 000000000..d828ed950 --- /dev/null +++ b/test/odh/support.go @@ -0,0 +1,34 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "embed" + + "github.com/onsi/gomega" + "github.com/project-codeflare/codeflare-common/support" +) + +//go:embed resources/* +var files embed.FS + +func ReadFile(t support.Test, fileName string) []byte { + t.T().Helper() + file, err := files.ReadFile(fileName) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return file +} From 6d4a5cd4088cb1204ed54f619d978bf497947a0e Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Mon, 13 Nov 2023 16:43:57 +0200 Subject: [PATCH 060/369] Add build info to logs --- Makefile | 14 +++++++++++++- main.go | 7 +++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a52a4e798..564fbb078 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,18 @@ endif SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +BUILD_DATE := $(shell date +%Y-%m-%d\ %H:%M) +BUILD_TAG_SHA := $(shell git rev-list --abbrev-commit --tags --max-count=1) +BUILD_TAG_NAME := $(shell git describe --abbrev=0 --tags ${BUILD_TAG_SHA} 2>/dev/null || true) +BUILD_SHA := $(shell git rev-parse --short HEAD) +BUILD_VERSION := $(BUILD_TAG_NAME:v%=%) +ifneq ($(BUILD_SHA), $(BUILD_TAG_SHA)) + BUILD_VERSION := $(BUILD_VERSION)-$(BUILD_SHA) +endif +ifneq ($(shell git status --porcelain),) + BUILD_VERSION := $(BUILD_VERSION)-dirty +endif + .PHONY: all all: build @@ -147,7 +159,7 @@ modules: ## Update Go dependencies. .PHONY: build build: modules fmt vet ## Build manager binary. - go build -o bin/manager main.go + go build -ldflags "-X 'main.BuildVersion=$(BUILD_VERSION)' -X 'main.BuildDate=$(BUILD_DATE)'" -o bin/manager main.go .PHONY: run run: modules manifests fmt vet ## Run a controller from your host. diff --git a/main.go b/main.go index 5e7065766..6387f18f7 100644 --- a/main.go +++ b/main.go @@ -58,8 +58,10 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + BuildVersion = "UNKNOWN" + BuildDate = "UNKNOWN" ) func init() { @@ -85,6 +87,7 @@ func main() { zapOptions.BindFlags(flag.CommandLine) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) + setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate) ctx := ctrl.SetupSignalHandler() From 32c66716ec87bcfcdf0ddf904febd198253c5499 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 14 Nov 2023 11:53:54 +0200 Subject: [PATCH 061/369] Log MCAD and InstaScale versions --- Makefile | 9 ++++++++- main.go | 17 ++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 564fbb078..7d647462e 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,14 @@ modules: ## Update Go dependencies. .PHONY: build build: modules fmt vet ## Build manager binary. - go build -ldflags "-X 'main.BuildVersion=$(BUILD_VERSION)' -X 'main.BuildDate=$(BUILD_DATE)'" -o bin/manager main.go + go build \ + -ldflags " \ + -X 'main.OperatorVersion=$(BUILD_VERSION)' \ + -X 'main.McadVersion=$(MCAD_VERSION)' \ + -X 'main.InstaScaleVersion=$(INSTASCALE_VERSION)' \ + -X 'main.BuildDate=$(BUILD_DATE)' \ + " \ + -o bin/manager main.go .PHONY: run run: modules manifests fmt vet ## Run a controller from your host. diff --git a/main.go b/main.go index 6387f18f7..84827f43c 100644 --- a/main.go +++ b/main.go @@ -58,10 +58,12 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - BuildVersion = "UNKNOWN" - BuildDate = "UNKNOWN" + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + OperatorVersion = "UNKNOWN" + McadVersion = "UNKNOWN" + InstaScaleVersion = "UNKNOWN" + BuildDate = "UNKNOWN" ) func init() { @@ -87,7 +89,12 @@ func main() { zapOptions.BindFlags(flag.CommandLine) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) - setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate) + setupLog.Info("Build info", + "operatorVersion", OperatorVersion, + "mcadVersion", McadVersion, + "instaScaleVersion", InstaScaleVersion, + "date", BuildDate, + ) ctx := ctrl.SetupSignalHandler() From 881946ee9a701fa58b29c94ccdcb7681ab6c1767 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 14 Nov 2023 14:47:05 +0200 Subject: [PATCH 062/369] Parse command line flags --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 84827f43c..4b2ce4288 100644 --- a/main.go +++ b/main.go @@ -81,6 +81,7 @@ func main() { flag.StringVar(&configMapName, "config", "codeflare-operator-config", "The name of the ConfigMap to load the operator configuration from. "+ "If it does not exist, the operator will create and initialise it.") + flag.Parse() zapOptions := zap.Options{ Development: true, From 6b06d33aa5b29d824769cf76a3495241a6734e08 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 14 Nov 2023 14:48:19 +0200 Subject: [PATCH 063/369] Update github.com/golang/glog v1.1.0 -> v1.1.2 This fixes vmodule panic ref: https://groups.google.com/g/golang-nuts/c/Atlr8uAjn6U/m/iId17Td5BQAJ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 98011af73..e60aaafea 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect - github.com/golang/glog v1.1.0 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/cel-go v0.12.6 // indirect diff --git a/go.sum b/go.sum index 423343034..54fcdff52 100644 --- a/go.sum +++ b/go.sum @@ -159,8 +159,8 @@ github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5 github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= From f818a6b32cfd4536a95ed809809aeb27571fd7d9 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Wed, 15 Nov 2023 09:33:45 +0200 Subject: [PATCH 064/369] Parse Zap flags --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 4b2ce4288..bf7f74545 100644 --- a/main.go +++ b/main.go @@ -81,13 +81,13 @@ func main() { flag.StringVar(&configMapName, "config", "codeflare-operator-config", "The name of the ConfigMap to load the operator configuration from. "+ "If it does not exist, the operator will create and initialise it.") - flag.Parse() zapOptions := zap.Options{ Development: true, TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), } zapOptions.BindFlags(flag.CommandLine) + flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) setupLog.Info("Build info", From 7776a0311a4338d81ba97862eda733b3004f575c Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 8 Nov 2023 15:18:15 +0000 Subject: [PATCH 065/369] Changed namespace in MCAD Ray CRB --- config/rbac/mcad-controller-ray-clusterrolebinding.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml index a3931da07..9ab3599e7 100644 --- a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml +++ b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml @@ -5,7 +5,7 @@ metadata: subjects: - kind: ServiceAccount name: codeflare-operator-controller-manager - namespace: openshift-operators + namespace: system roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole From 5789a4d52b44956ab9c543e6a9b9f329d70a004d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 16 Nov 2023 14:50:40 +0100 Subject: [PATCH 066/369] e2e: Fix deployment namespace --- config/e2e/kustomization.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index faf613e70..e06123b2a 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,3 +1,5 @@ +namespace: openshift-operators + bases: - ../default From 9c0ceccfc960c0acf5595100959b7f209e4b27e3 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Thu, 16 Nov 2023 12:46:18 +0000 Subject: [PATCH 067/369] Update dependency versions for release v1.0.1 --- Makefile | 4 ++-- README.md | 10 +++++----- config/crd/mcad/kustomization.yaml | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 7d647462e..d32fdc01f 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.3.0 +INSTASCALE_VERSION ?= v0.3.1 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.38.0 +MCAD_VERSION ?= v1.38.1 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index d8cdd53dc..7a0280fa1 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0) | -| Multi-Cluster App Dispatcher | [v1.38.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.0) | -| CodeFlare-SDK | [v0.11.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.11.0) | -| InstaScale | [v0.3.0](https://github.com/project-codeflare/instascale/releases/tag/v0.3.0) | -| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | +| CodeFlare Operator | [v1.0.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.1) | +| Multi-Cluster App Dispatcher | [v1.38.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.1) | +| CodeFlare-SDK | [v0.12.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.12.1) | +| InstaScale | [v0.3.1](https://github.com/project-codeflare/instascale/releases/tag/v0.3.1) | +| KubeRay | [v0.0.0-dev](https://github.com/opendatahub-io/kuberay/releases/tag/v0.0.0-dev) | ## Development diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 23063b679..3cd50c483 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.1 diff --git a/go.mod b/go.mod index e60aaafea..fd23755d4 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb - github.com/project-codeflare/instascale v0.3.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 + github.com/project-codeflare/instascale v0.3.1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 54fcdff52..6c33548bf 100644 --- a/go.sum +++ b/go.sum @@ -393,10 +393,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb h1:L2Gdr2SlvshDKZY2KK6507AwzQ1NSfRbMQuz5dOsYNM= github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= -github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs= -github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= +github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 h1:6ILHYAFxDkAnQu3CJebGQPQGcmcy7/E/AhRiea6yOTc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 42cce2d0713e4a22a00d145fbac14f67ab3f3016 Mon Sep 17 00:00:00 2001 From: Srihari Date: Tue, 3 Oct 2023 18:28:35 +0530 Subject: [PATCH 068/369] Update SDK e2e test to reflect poetry changes Add Ingress domain for sdk e2e test Revert "Add Ingress domain for sdk e2e test" This reverts commit ffc200474502e2aeebdce1654c8c1a2bce086c25. --- test/e2e/install-codeflare-sdk.sh | 22 ++++++++++++++++++++++ test/e2e/mnist_raycluster_sdk_test.go | 14 +++++++++++++- test/e2e/support.go | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100755 test/e2e/install-codeflare-sdk.sh diff --git a/test/e2e/install-codeflare-sdk.sh b/test/e2e/install-codeflare-sdk.sh new file mode 100755 index 000000000..e90f4071c --- /dev/null +++ b/test/e2e/install-codeflare-sdk.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# go to codeflare-sdk folder and install codeflare-sdk +cd .. + +# Install Poetry and configure virtualenvs +pip install poetry +poetry config virtualenvs.create false + +cd codeflare-sdk +# Clone the CodeFlare SDK repository +git clone --branch main https://github.com/project-codeflare/codeflare-sdk.git + +cd codeflare-sdk + +# Lock dependencies and install them +poetry lock --no-update +poetry install --with test,docs + +# Return to the workdir +cd ../.. +cd workdir diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 015cbfc58..be6933f67 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -51,6 +51,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), // MNIST training script "mnist.py": ReadFile(test, "mnist.py"), + // codeflare-sdk installation script + "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), }) // Create RBAC, retrieve token for user with limited rights @@ -119,12 +121,16 @@ func TestMNISTRayClusterSDK(t *testing.T) { {Name: "PYTHONUSERBASE", Value: "/workdir"}, {Name: "RAY_IMAGE", Value: GetRayImage()}, }, - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, + Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", MountPath: "/test", }, + { + Name: "codeflare-sdk", + MountPath: "/codeflare-sdk", + }, { Name: "workdir", MountPath: "/workdir", @@ -154,6 +160,12 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, }, + { + Name: "codeflare-sdk", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, { Name: "workdir", VolumeSource: corev1.VolumeSource{ diff --git a/test/e2e/support.go b/test/e2e/support.go index e91656880..8fc862957 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -23,7 +23,7 @@ import ( "github.com/project-codeflare/codeflare-common/support" ) -//go:embed *.py *.txt +//go:embed *.py *.txt *.sh var files embed.FS func ReadFile(t support.Test, fileName string) []byte { From ee0e762cc9f938b4931fc046b8c82e8774e77a1b Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 21 Nov 2023 16:08:58 +0100 Subject: [PATCH 069/369] Add missing KubeRay version parameter for release workflow --- .github/workflows/project-codeflare-release.yml | 5 ++++- .github/workflows/tag-and-build.yml | 1 + README.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 1f8bb0c7d..77d0a0a1f 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -19,6 +19,9 @@ on: instascale-version: description: 'Version of InstaScale to be released (for example: v0.0.0)' required: true + kuberay-version: + description: 'Tested version of KubeRay (for example: v0.0.0)' + required: true is-stable: description: 'Select if the built images should be tagged as stable' required: true @@ -156,7 +159,7 @@ jobs: steps: - name: Release CodeFlare operator run: | - gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 827a33c66..b97deb336 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -100,6 +100,7 @@ jobs: sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile + sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile - name: Login to Quay.io uses: redhat-actions/podman-login@v1 diff --git a/README.md b/README.md index 7a0280fa1..c3048b299 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ CodeFlare Stack Compatibility Matrix | Multi-Cluster App Dispatcher | [v1.38.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.1) | | CodeFlare-SDK | [v0.12.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.12.1) | | InstaScale | [v0.3.1](https://github.com/project-codeflare/instascale/releases/tag/v0.3.1) | -| KubeRay | [v0.0.0-dev](https://github.com/opendatahub-io/kuberay/releases/tag/v0.0.0-dev) | +| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | ## Development From 4f04b9f18f260fc6d59506bcf5c2a244a4b7c029 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 21 Nov 2023 14:17:00 -0500 Subject: [PATCH 070/369] set klog to the controller runtime logger Signed-off-by: Kevin --- go.mod | 2 +- main.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fd23755d4..455963b03 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 k8s.io/component-base v0.26.2 + k8s.io/klog/v2 v2.90.1 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/controller-runtime v0.14.6 sigs.k8s.io/yaml v1.3.0 @@ -114,7 +115,6 @@ require ( k8s.io/apiextensions-apiserver v0.26.1 // indirect k8s.io/apiserver v0.26.2 // indirect k8s.io/klog v1.0.0 // indirect - k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kms v0.26.2 // indirect k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect k8s.io/metrics v0.26.2 // indirect diff --git a/main.go b/main.go index bf7f74545..5e2bb6e37 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" configv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/klog/v2" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -90,6 +91,8 @@ func main() { flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) + klog.SetLogger(ctrl.Log) + setupLog.Info("Build info", "operatorVersion", OperatorVersion, "mcadVersion", McadVersion, From 426a67fa701508c11aabb21dbd41eb3aa0383d75 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 23 Nov 2023 08:31:33 +0100 Subject: [PATCH 071/369] Raise SDK e2e test timeout in Python script --- test/e2e/mnist_raycluster_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py index cb3c0af56..d171c9b08 100644 --- a/test/e2e/mnist_raycluster_sdk.py +++ b/test/e2e/mnist_raycluster_sdk.py @@ -62,7 +62,7 @@ done = False time = 0 -timeout = 300 +timeout = 900 while not done: status = job.status() if is_terminal(status.state): From 4de7141a011a8069e22d083402c2a67d5a49825a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 24 Nov 2023 10:49:05 +0100 Subject: [PATCH 072/369] Adjust release GH workflow to handle error cases --- .github/workflows/tag-and-build.yml | 38 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index b97deb336..28d27c991 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -121,20 +121,6 @@ jobs: podman tag quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:${{ github.event.inputs.version }} quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:stable make image-push -e IMG=quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:stable - - name: Build bundle and create PR in OpenShift community operators repository - run: | - git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" - git config --global user.name "codeflare-machine-account" - make openshift-community-operator-release - env: - VERSION: ${{ github.event.inputs.version }} - PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} - INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} - MCAD_VERSION: ${{ github.event.inputs.mcad-version }} - GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} - OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} - - name: Cleanup the go.mod and go.sum run: | go mod tidy @@ -144,6 +130,7 @@ jobs: make manifests - name: Commit changes in the code back to repository + id: create-pr-branch uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} @@ -159,12 +146,21 @@ jobs: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - name: Wait until PR with code changes is merged + id: wait-pr-merge run: | timeout 3600 bash -c 'until [[ $(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state) == "MERGED" ]]; do sleep 5 && echo "$(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state)"; done' env: GITHUB_TOKEN: ${{ github.TOKEN }} + - name: Close PR if PR merge failed + if: always() && (steps.wait-pr-merge.outcome == 'failure' || steps.wait-pr-merge.outcome == 'cancelled') + run: | + gh pr close ${{ env.PR_BRANCH_NAME }} + env: + GITHUB_TOKEN: ${{ github.TOKEN }} + - name: Delete remote branch + if: always() && steps.create-pr-branch.outcome == 'success' run: | git push origin --delete ${{ env.PR_BRANCH_NAME }} @@ -180,3 +176,17 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + + - name: Build bundle and create PR in OpenShift community operators repository + run: | + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" + make openshift-community-operator-release + env: + VERSION: ${{ github.event.inputs.version }} + PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} + INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} + MCAD_VERSION: ${{ github.event.inputs.mcad-version }} + GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} + OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} From 0acdfde08f9d8d7dc407ac5f329c0a3fe181d1a1 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 24 Nov 2023 17:01:15 +0100 Subject: [PATCH 073/369] Remove accidental prefix for service account for mcad-controller-ray-clusterrolebinding --- config/rbac/mcad-controller-ray-clusterrolebinding.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml index 9ab3599e7..da9e8c023 100644 --- a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml +++ b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml @@ -4,7 +4,7 @@ metadata: name: mcad-controller-ray-clusterrolebinding subjects: - kind: ServiceAccount - name: codeflare-operator-controller-manager + name: controller-manager namespace: system roleRef: apiGroup: rbac.authorization.k8s.io From 39e942246be34ac421cf0e3e9f4e186595216c90 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 24 Nov 2023 15:22:01 +0100 Subject: [PATCH 074/369] Send notification on Slack if e2e fails on push event --- .github/workflows/e2e_tests.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 64d4567e1..b20ff9689 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -93,3 +93,12 @@ jobs: retention-days: 10 path: | ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log + + - name: Post notification about failure to a Slack channel in case of push event + if: failure() && github.event_name == 'push' + uses: slackapi/slack-github-action@v1.24.0 + with: + channel-id: "codeflare-nightlies" + slack-message: "e2e test on push failed, " + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} From 23bffcc83574cd09fcd5e70c0a917de8ab191e3c Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 29 Nov 2023 12:48:00 +0100 Subject: [PATCH 075/369] Add completion status for TestMNISTPyTorchMCAD --- go.mod | 2 +- go.sum | 4 ++-- test/e2e/mnist_pytorch_mcad_job_test.go | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 455963b03..e9f358d48 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb + github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 diff --git a/go.sum b/go.sum index 6c33548bf..9af7ce38d 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb h1:L2Gdr2SlvshDKZY2KK6507AwzQ1NSfRbMQuz5dOsYNM= -github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= +github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 h1:81+ma1mchF/LtAGsf+poAt50kJ/fLYjoTAcZOxci1Yc= +github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 h1:6ILHYAFxDkAnQu3CJebGQPQGcmcy7/E/AhRiea6yOTc= diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 7a157c040..cbe55fed2 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -143,7 +143,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { }, }, }, - GenericTemplate: Raw(test, job), + GenericTemplate: Raw(test, job), + CompletionStatus: "Complete", }, }, }, @@ -159,13 +160,13 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), )) // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.Expect(GetAppWrapper(test, namespace, aw.Name)). + To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) } From 148d43ff98ea9195739f7c5e87e601bca9d28cc3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 29 Nov 2023 16:04:53 +0100 Subject: [PATCH 076/369] Use dedicated KinD composite action from CodeFlare common --- .github/actions/kind/action.yml | 74 -------------------------------- .github/resources-kind/kind.yaml | 31 ------------- .github/workflows/e2e_tests.yaml | 9 +++- .github/workflows/olm_tests.yaml | 9 +++- 4 files changed, 16 insertions(+), 107 deletions(-) delete mode 100644 .github/actions/kind/action.yml delete mode 100644 .github/resources-kind/kind.yaml diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml deleted file mode 100644 index f76e60af5..000000000 --- a/.github/actions/kind/action.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: "Set up KinD" -description: "Step to start and configure KinD cluster" - -inputs: - kind-node-hostname: - description: "Hostname of the main kind node" - required: false - default: kind - -runs: - using: "composite" - steps: - - name: Init directories - shell: bash - run: | - TEMP_DIR="$(pwd)/tmp" - mkdir -p "${TEMP_DIR}" - echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV - - mkdir -p "$(pwd)/bin" - echo "$(pwd)/bin" >> $GITHUB_PATH - - - name: Container image registry - shell: bash - run: | - podman run -d -p 5000:5000 --name registry registry:2.8.1 - - export REGISTRY_ADDRESS=$(hostname -i):5000 - echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV - echo "Container image registry started at ${REGISTRY_ADDRESS}" - - KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml - echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV - envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE} - - sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <> $GITHUB_ENV - echo "CLUSTER_HOSTNAME=${{ inputs.kind-node-hostname }}" >> $GITHUB_ENV diff --git a/.github/resources-kind/kind.yaml b/.github/resources-kind/kind.yaml deleted file mode 100644 index 4546589b8..000000000 --- a/.github/resources-kind/kind.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# --------------------------------------------------------------------------- -# Copyright 2023. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- - -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" -containerdConfigPatches: - - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${REGISTRY_ADDRESS}"] - endpoint = ["http://${REGISTRY_ADDRESS}"] diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index b20ff9689..1962a8b84 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -35,6 +35,13 @@ jobs: with: submodules: recursive + - name: Checkout common repo code + uses: actions/checkout@v3 + with: + repository: 'project-codeflare/codeflare-common' + ref: 'main' + path: 'common' + - name: Set Go uses: actions/setup-go@v3 with: @@ -46,7 +53,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Setup and start KinD cluster - uses: ./.github/actions/kind + uses: ./common/github-actions/kind - name: Deploy CodeFlare stack id: deploy diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 382c95f3c..83175503a 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -34,6 +34,13 @@ jobs: with: fetch-depth: 0 # fetching also previous commits to get tags + - name: Checkout common repo code + uses: actions/checkout@v3 + with: + repository: 'project-codeflare/codeflare-common' + ref: 'main' + path: 'common' + - name: Set Go uses: actions/setup-go@v3 with: @@ -45,7 +52,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Setup and start KinD cluster - uses: ./.github/actions/kind + uses: ./common/github-actions/kind - name: Install OLM run: | From 68c5f67817694252fc76ff68a117b8a01a3f7c3a Mon Sep 17 00:00:00 2001 From: Srihari Date: Thu, 14 Sep 2023 13:22:33 +0530 Subject: [PATCH 077/369] Create OLM upgrade e2e scenario using codeflare SDK --- .github/workflows/olm_tests.yaml | 17 ++ test/e2e/mnist_rayjob.py | 44 +++++ test/e2e/olm_upgrade_test.go | 322 +++++++++++++++++++++++++++++++ test/e2e/raycluster_sdk.py | 29 +++ 4 files changed, 412 insertions(+) create mode 100644 test/e2e/mnist_rayjob.py create mode 100644 test/e2e/olm_upgrade_test.go create mode 100644 test/e2e/raycluster_sdk.py diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 83175503a..de1de85bb 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -121,6 +121,14 @@ jobs: OPM_BUNDLE_OPT: "--use-http" BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" + - name: Run OLM Upgrade e2e Ray cluster Up test + run: | + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + set -euo pipefail + go test -timeout 30m -v ./test/e2e -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + env: + RUN_OLM_TESTS: true - name: Update Operator to the built version run: | @@ -151,6 +159,15 @@ jobs: SUBSCRIPTION_NAME: "codeflare-operator" SUBSCRIPTION_NAMESPACE: "openshift-operators" + - name: Run OLM Upgrade e2e Ray cluster Job submit test + run: | + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + set -euo pipefail + go test -timeout 30m -v ./test/e2e -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + env: + RUN_OLM_TESTS: true + - name: Run e2e tests against built operator run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} diff --git a/test/e2e/mnist_rayjob.py b/test/e2e/mnist_rayjob.py new file mode 100644 index 000000000..99cfcb1a7 --- /dev/null +++ b/test/e2e/mnist_rayjob.py @@ -0,0 +1,44 @@ +import sys + +from time import sleep + +from torchx.specs.api import AppState, is_terminal + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +from codeflare_sdk.job.jobs import DDPJobDefinition + +namespace = sys.argv[1] + +cluster = Cluster(ClusterConfiguration(name='mnist')) + +jobdef = DDPJobDefinition( + name="mnist", + script="mnist.py", + scheduler_args={"requirements": "requirements.txt"}, +) +job = jobdef.submit(cluster) + +done = False +time = 0 +timeout = 300 +while not done: + status = job.status() + if is_terminal(status.state): + break + if not done: + print(status) + if timeout and time >= timeout: + raise TimeoutError(f"job has timed out after waiting {timeout}s") + sleep(5) + time += 5 + +print(f"Job has completed: {status.state}") + +print(job.logs()) + +cluster.down() + +if not status.state == AppState.SUCCEEDED: + exit(1) +else: + exit(0) diff --git a/test/e2e/olm_upgrade_test.go b/test/e2e/olm_upgrade_test.go new file mode 100644 index 000000000..3dde139f9 --- /dev/null +++ b/test/e2e/olm_upgrade_test.go @@ -0,0 +1,322 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "os" + "testing" + + . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +var namespaceName string = "test-ns-olmupgrade" +var namespace *corev1.Namespace + +// Creates a Ray cluster +func TestMNISTRayClusterUp(t *testing.T) { + + test := With(t) + test.T().Parallel() + if os.Getenv("RUN_OLM_TESTS") == "true" { + // Create a namespace + namespace := CreateTestNamespaceWithName(test, namespaceName) + test.T().Logf("Created namespace %s successfully", namespace.Name) + + // Test configuration + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-raycluster-sdk", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + // SDK script + "raycluster_sdk.py": ReadFile(test, "raycluster_sdk.py"), + }, + Immutable: Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // SDK client RBAC + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk-user", + Namespace: namespace.Name, + }, + } + serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + role := &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "Role", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + }, + } + role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + roleBinding := &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: corev1.SchemeGroupVersion.Group, + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + }, + }, + } + _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python raycluster_sdk.py" + " " + namespace.Name}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, + }, + }, + }, + } + job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + + } else { + test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") + } +} + +// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. +func TestMnistJobSubmit(t *testing.T) { + + test := With(t) + test.T().Parallel() + if os.Getenv("RUN_OLM_TESTS") == "true" { + + // Test configuration + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-ray-job", + Namespace: namespaceName, + }, + BinaryData: map[string][]byte{ + // SDK script + "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }, + Immutable: Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespaceName).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // SDK client RBAC + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ray-user", + Namespace: namespaceName, + }, + } + serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespaceName).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rayjob", + Namespace: namespaceName, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_rayjob.py" + " " + namespaceName}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, + }, + }, + }, + } + job, err = test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + } else { + test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") + } + +} diff --git a/test/e2e/raycluster_sdk.py b/test/e2e/raycluster_sdk.py new file mode 100644 index 000000000..5a8abf5cd --- /dev/null +++ b/test/e2e/raycluster_sdk.py @@ -0,0 +1,29 @@ +import sys + +from time import sleep + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration + +namespace = sys.argv[1] + +cluster = Cluster(ClusterConfiguration( + name='mnist', + namespace=namespace, + num_workers=1, + min_cpus='500m', + max_cpus=1, + min_memory=0.5, + max_memory=1, + num_gpus=0, + instascale=False, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() From 2a367fa9457a895bc7de50c54e99e27858ceec13 Mon Sep 17 00:00:00 2001 From: Srihari Date: Thu, 14 Sep 2023 13:32:41 +0530 Subject: [PATCH 078/369] Create OLM upgrade e2e scenario using codeflare SDK --- test/e2e/olm_upgrade_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/e2e/olm_upgrade_test.go b/test/e2e/olm_upgrade_test.go index 3dde139f9..acb25a4b0 100644 --- a/test/e2e/olm_upgrade_test.go +++ b/test/e2e/olm_upgrade_test.go @@ -42,9 +42,16 @@ func TestMNISTRayClusterUp(t *testing.T) { test.T().Parallel() if os.Getenv("RUN_OLM_TESTS") == "true" { // Create a namespace - namespace := CreateTestNamespaceWithName(test, namespaceName) + namespace = CreateTestNamespaceWithName(test, namespaceName) test.T().Logf("Created namespace %s successfully", namespace.Name) + // Delete namespace only if test failed + defer func() { + if t.Failed() { + DeleteTestNamespace(test, namespaceName) + } + }() + // Test configuration config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -212,6 +219,9 @@ func TestMnistJobSubmit(t *testing.T) { test.T().Parallel() if os.Getenv("RUN_OLM_TESTS") == "true" { + //delete the namespace after test complete + defer DeleteTestNamespace(test, namespaceName) + // Test configuration config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -315,6 +325,7 @@ func TestMnistJobSubmit(t *testing.T) { // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + } else { test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") } From 2d99cc93bec02b6999e4a773e5fb9da866ed40ab Mon Sep 17 00:00:00 2001 From: Srihari Date: Thu, 23 Nov 2023 12:15:37 +0530 Subject: [PATCH 079/369] rebase and resolving conflicts --- .github/workflows/olm_tests.yaml | 9 +- go.mod | 1 - test/e2e/mnist_rayjob.py | 2 +- test/e2e/olm_upgrade_test.go | 333 ------------------------- test/e2e/raycluster_sdk.py | 29 --- test/e2e/start_ray_cluster.py | 51 ++++ test/upgrade/olm_upgrade_test.go | 403 +++++++++++++++++++++++++++++++ 7 files changed, 458 insertions(+), 370 deletions(-) delete mode 100644 test/e2e/olm_upgrade_test.go delete mode 100644 test/e2e/raycluster_sdk.py create mode 100644 test/e2e/start_ray_cluster.py create mode 100644 test/upgrade/olm_upgrade_test.go diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index de1de85bb..927dced70 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -121,14 +121,13 @@ jobs: OPM_BUNDLE_OPT: "--use-http" BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" + - name: Run OLM Upgrade e2e Ray cluster Up test run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/e2e -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - env: - RUN_OLM_TESTS: true + go test -timeout 30m -v ./test/upgrade -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Update Operator to the built version run: | @@ -164,9 +163,7 @@ jobs: export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/e2e -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - env: - RUN_OLM_TESTS: true + go test -timeout 30m -v ./test/upgrade -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Run e2e tests against built operator run: | diff --git a/go.mod b/go.mod index e9f358d48..c7405e30e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 - github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 diff --git a/test/e2e/mnist_rayjob.py b/test/e2e/mnist_rayjob.py index 99cfcb1a7..7ffb39271 100644 --- a/test/e2e/mnist_rayjob.py +++ b/test/e2e/mnist_rayjob.py @@ -9,7 +9,7 @@ namespace = sys.argv[1] -cluster = Cluster(ClusterConfiguration(name='mnist')) +cluster = Cluster(ClusterConfiguration(name='mnist',namespace=namespace)) jobdef = DDPJobDefinition( name="mnist", diff --git a/test/e2e/olm_upgrade_test.go b/test/e2e/olm_upgrade_test.go deleted file mode 100644 index acb25a4b0..000000000 --- a/test/e2e/olm_upgrade_test.go +++ /dev/null @@ -1,333 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "os" - "testing" - - . "github.com/onsi/gomega" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" -) - -var namespaceName string = "test-ns-olmupgrade" -var namespace *corev1.Namespace - -// Creates a Ray cluster -func TestMNISTRayClusterUp(t *testing.T) { - - test := With(t) - test.T().Parallel() - if os.Getenv("RUN_OLM_TESTS") == "true" { - // Create a namespace - namespace = CreateTestNamespaceWithName(test, namespaceName) - test.T().Logf("Created namespace %s successfully", namespace.Name) - - // Delete namespace only if test failed - defer func() { - if t.Failed() { - DeleteTestNamespace(test, namespaceName) - } - }() - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-raycluster-sdk", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - // SDK script - "raycluster_sdk.py": ReadFile(test, "raycluster_sdk.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // SDK client RBAC - serviceAccount := &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ServiceAccount", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk-user", - Namespace: namespace.Name, - }, - } - serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - role := &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "Role", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Rules: []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1alpha1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - }, - } - role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - roleBinding := &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "RoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacv1.SchemeGroupVersion.Group, - Kind: "Role", - Name: role.Name, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - APIGroup: corev1.SchemeGroupVersion.Group, - Name: serviceAccount.Name, - Namespace: serviceAccount.Namespace, - }, - }, - } - _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python raycluster_sdk.py" + " " + namespace.Name}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, - }, - }, - }, - } - job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - - // Retrieving the job logs once it has completed or timed out - defer WriteJobLogs(test, job.Namespace, job.Name) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - - } else { - test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") - } -} - -// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. -func TestMnistJobSubmit(t *testing.T) { - - test := With(t) - test.T().Parallel() - if os.Getenv("RUN_OLM_TESTS") == "true" { - - //delete the namespace after test complete - defer DeleteTestNamespace(test, namespaceName) - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-ray-job", - Namespace: namespaceName, - }, - BinaryData: map[string][]byte{ - // SDK script - "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespaceName).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // SDK client RBAC - serviceAccount := &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ServiceAccount", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "ray-user", - Namespace: namespaceName, - }, - } - serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespaceName).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "rayjob", - Namespace: namespaceName, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_rayjob.py" + " " + namespaceName}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, - }, - }, - }, - } - job, err = test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - - // Retrieving the job logs once it has completed or timed out - defer WriteJobLogs(test, job.Namespace, job.Name) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - - } else { - test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") - } - -} diff --git a/test/e2e/raycluster_sdk.py b/test/e2e/raycluster_sdk.py deleted file mode 100644 index 5a8abf5cd..000000000 --- a/test/e2e/raycluster_sdk.py +++ /dev/null @@ -1,29 +0,0 @@ -import sys - -from time import sleep - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration - -namespace = sys.argv[1] - -cluster = Cluster(ClusterConfiguration( - name='mnist', - namespace=namespace, - num_workers=1, - min_cpus='500m', - max_cpus=1, - min_memory=0.5, - max_memory=1, - num_gpus=0, - instascale=False, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() diff --git a/test/e2e/start_ray_cluster.py b/test/e2e/start_ray_cluster.py new file mode 100644 index 000000000..238bccdf2 --- /dev/null +++ b/test/e2e/start_ray_cluster.py @@ -0,0 +1,51 @@ +import sys +import os + +from time import sleep + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration + +namespace = sys.argv[1] +ray_image = os.getenv('RAY_IMAGE') +host = os.getenv('CLUSTER_HOSTNAME') + +ingress_options = {} +if host is not None: + ingress_options = { + "ingresses": [ + { + "ingressName": "ray-dashboard", + "port": 8265, + "pathType": "Prefix", + "path": "/", + "host": host, + }, + ] + } + + +cluster = Cluster(ClusterConfiguration( + name='mnist', + namespace=namespace, + num_workers=1, + head_cpus='500m', + head_memory=2, + min_cpus='500m', + max_cpus=1, + min_memory=0.5, + max_memory=2, + num_gpus=0, + instascale=False, + image=ray_image, + ingress_options=ingress_options, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go new file mode 100644 index 000000000..0d27c0649 --- /dev/null +++ b/test/upgrade/olm_upgrade_test.go @@ -0,0 +1,403 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/e2e" +) + +var ( + namespaceName = "test-ns-olmupgrade" +) + +// Creates a Ray cluster +func TestMNISTRayClusterUp(t *testing.T) { + + test := With(t) + test.T().Parallel() + + // Create a namespace + namespace := CreateTestNamespaceWithName(test, namespaceName) + test.T().Logf("Created namespace %s successfully", namespace.Name) + + // Delete namespace only if test failed + defer func() { + if t.Failed() { + DeleteTestNamespace(test, namespace) + } else { + StoreNamespaceLogs(test, namespace) + } + }() + + // Test configuration + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // SDK script + "start_ray_cluster.py": ReadFile(test, "start_ray_cluster.py"), + // codeflare-sdk installation script + "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, + }, + } + + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, + }, + } + + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, sa, clusterRole) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Env: []corev1.EnvVar{ + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "RAY_IMAGE", Value: GetRayImage()}, + }, + Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python start_ray_cluster.py" + " " + namespace.Name}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + { + Name: "codeflare-sdk", + MountPath: "/codeflare-sdk", + }, + { + Name: "workdir", + MountPath: "/workdir", + }, + }, + WorkingDir: "/workdir", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: Ptr(false), + SeccompProfile: &corev1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: Ptr(true), + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + { + Name: "codeflare-sdk", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: sa.Name, + }, + }, + }, + } + if GetClusterType(test) == KindCluster { + // Take first KinD node and redirect pod hostname requests there + node := GetNodes(test)[0] + hostname := GetClusterHostname(test) + IP := GetNodeInternalIP(test, node) + + test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) + job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ + { + IP: IP, + Hostnames: []string{hostname}, + }, + } + + // Propagate hostname into Python code as env variable + hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + } + + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) +} + +// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. +func TestMnistJobSubmit(t *testing.T) { + + test := With(t) + test.T().Parallel() + + namespace := GetNamespaceWithName(test, namespaceName) + + //delete the namespace after test complete + defer DeleteTestNamespace(test, namespace) + + // Test configuration + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // SDK script + "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + // codeflare-sdk installation script + "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, + }, + } + + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, + }, + } + + serviceAccount := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, serviceAccount, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, serviceAccount, clusterRole) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rayjob", + Namespace: namespaceName, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_rayjob.py" + " " + namespaceName}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + { + Name: "codeflare-sdk", + MountPath: "/codeflare-sdk", + }, + { + Name: "workdir", + MountPath: "/workdir", + }, + }, + WorkingDir: "/workdir", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: Ptr(false), + SeccompProfile: &corev1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: Ptr(true), + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + { + Name: "codeflare-sdk", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, + }, + }, + }, + } + + if GetClusterType(test) == KindCluster { + // Take first KinD node and redirect pod hostname requests there + node := GetNodes(test)[0] + hostname := GetClusterHostname(test) + IP := GetNodeInternalIP(test, node) + + test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) + job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ + { + IP: IP, + Hostnames: []string{hostname}, + }, + } + + // Propagate hostname into Python code as env variable + hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + } + + job, err := test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + +} From 0afa252a9a4178cd9e75f1e87e12c6954f41c105 Mon Sep 17 00:00:00 2001 From: Srihari Date: Mon, 4 Dec 2023 16:42:56 +0530 Subject: [PATCH 080/369] Create e2e scenario covering upgrade during training --- .github/workflows/olm_tests.yaml | 8 +- go.mod | 1 + test/e2e/mnist_rayjob.py | 44 ---- test/e2e/start_ray_cluster.py | 51 ----- test/upgrade/olm_upgrade_test.go | 345 +++++++------------------------ 5 files changed, 79 insertions(+), 370 deletions(-) delete mode 100644 test/e2e/mnist_rayjob.py delete mode 100644 test/e2e/start_ray_cluster.py diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 927dced70..1dcb6f2d8 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -122,12 +122,12 @@ jobs: BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" - - name: Run OLM Upgrade e2e Ray cluster Up test + - name: Run OLM Upgrade e2e AppWrapper creation test run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 30m -v ./test/upgrade -run TestMNISTCreateAppWrapper -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Update Operator to the built version run: | @@ -158,12 +158,12 @@ jobs: SUBSCRIPTION_NAME: "codeflare-operator" SUBSCRIPTION_NAMESPACE: "openshift-operators" - - name: Run OLM Upgrade e2e Ray cluster Job submit test + - name: Run OLM Upgrade e2e Appwrapper Job status test to monitor training run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 30m -v ./test/upgrade -run TestMNISTCheckAppWrapperStatus -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Run e2e tests against built operator run: | diff --git a/go.mod b/go.mod index c7405e30e..e9f358d48 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 + github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 diff --git a/test/e2e/mnist_rayjob.py b/test/e2e/mnist_rayjob.py deleted file mode 100644 index 7ffb39271..000000000 --- a/test/e2e/mnist_rayjob.py +++ /dev/null @@ -1,44 +0,0 @@ -import sys - -from time import sleep - -from torchx.specs.api import AppState, is_terminal - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration -from codeflare_sdk.job.jobs import DDPJobDefinition - -namespace = sys.argv[1] - -cluster = Cluster(ClusterConfiguration(name='mnist',namespace=namespace)) - -jobdef = DDPJobDefinition( - name="mnist", - script="mnist.py", - scheduler_args={"requirements": "requirements.txt"}, -) -job = jobdef.submit(cluster) - -done = False -time = 0 -timeout = 300 -while not done: - status = job.status() - if is_terminal(status.state): - break - if not done: - print(status) - if timeout and time >= timeout: - raise TimeoutError(f"job has timed out after waiting {timeout}s") - sleep(5) - time += 5 - -print(f"Job has completed: {status.state}") - -print(job.logs()) - -cluster.down() - -if not status.state == AppState.SUCCEEDED: - exit(1) -else: - exit(0) diff --git a/test/e2e/start_ray_cluster.py b/test/e2e/start_ray_cluster.py deleted file mode 100644 index 238bccdf2..000000000 --- a/test/e2e/start_ray_cluster.py +++ /dev/null @@ -1,51 +0,0 @@ -import sys -import os - -from time import sleep - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration - -namespace = sys.argv[1] -ray_image = os.getenv('RAY_IMAGE') -host = os.getenv('CLUSTER_HOSTNAME') - -ingress_options = {} -if host is not None: - ingress_options = { - "ingresses": [ - { - "ingressName": "ray-dashboard", - "port": 8265, - "pathType": "Prefix", - "path": "/", - "host": host, - }, - ] - } - - -cluster = Cluster(ClusterConfiguration( - name='mnist', - namespace=namespace, - num_workers=1, - head_cpus='500m', - head_memory=2, - min_cpus='500m', - max_cpus=1, - min_memory=0.5, - max_memory=2, - num_gpus=0, - instascale=False, - image=ray_image, - ingress_options=ingress_options, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go index 0d27c0649..2abd086ea 100644 --- a/test/upgrade/olm_upgrade_test.go +++ b/test/upgrade/olm_upgrade_test.go @@ -22,29 +22,27 @@ import ( . "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/project-codeflare/codeflare-operator/test/e2e" ) var ( - namespaceName = "test-ns-olmupgrade" + namespaceName = "test-ns-olmupgrade" + appwrapperName = "mnist" + jobName = "mnist-job" ) -// Creates a Ray cluster -func TestMNISTRayClusterUp(t *testing.T) { - +func TestMNISTCreateAppWrapper(t *testing.T) { test := With(t) test.T().Parallel() // Create a namespace namespace := CreateTestNamespaceWithName(test, namespaceName) - test.T().Logf("Created namespace %s successfully", namespace.Name) // Delete namespace only if test failed defer func() { @@ -56,105 +54,61 @@ func TestMNISTRayClusterUp(t *testing.T) { }() // Test configuration - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // SDK script - "start_ray_cluster.py": ReadFile(test, "start_ray_cluster.py"), - // codeflare-sdk installation script - "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-mcad", + Namespace: namespace.Name, }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, + BinaryData: map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), }, + Immutable: Ptr(true), } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, sa, clusterRole) - + // Batch Job job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", }, ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", + Name: jobName, Namespace: namespace.Name, }, Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Name: "job", + Image: GetPyTorchImage(), Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "RAY_IMAGE", Value: GetRayImage()}, + corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, }, - Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python start_ray_cluster.py" + " " + namespace.Name}, + Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", MountPath: "/test", }, - { - Name: "codeflare-sdk", - MountPath: "/codeflare-sdk", - }, { Name: "workdir", MountPath: "/workdir", }, }, WorkingDir: "/workdir", - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: Ptr(false), - SeccompProfile: &corev1.SeccompProfile{ - Type: "RuntimeDefault", - }, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - RunAsNonRoot: Ptr(true), - }, }, }, Volumes: []corev1.Volume{ @@ -168,12 +122,6 @@ func TestMNISTRayClusterUp(t *testing.T) { }, }, }, - { - Name: "codeflare-sdk", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, { Name: "workdir", VolumeSource: corev1.VolumeSource{ @@ -181,223 +129,78 @@ func TestMNISTRayClusterUp(t *testing.T) { }, }, }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: sa.Name, + RestartPolicy: corev1.RestartPolicyNever, }, }, }, } - if GetClusterType(test) == KindCluster { - // Take first KinD node and redirect pod hostname requests there - node := GetNodes(test)[0] - hostname := GetClusterHostname(test) - IP := GetNodeInternalIP(test, node) - test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) - job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ - { - IP: IP, - Hostnames: []string{hostname}, + // Create an AppWrapper resource + aw := &mcadv1beta1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: appwrapperName, + Namespace: namespace.Name, + }, + Spec: mcadv1beta1.AppWrapperSpec{ + AggrResources: mcadv1beta1.AppWrapperResourceList{ + GenericItems: []mcadv1beta1.AppWrapperGenericResource{ + { + DesiredAvailable: 1, + CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + }, + GenericTemplate: Raw(test, job), + CompletionStatus: "Complete", + }, + }, }, - } - - // Propagate hostname into Python code as env variable - hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} - job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + }, } - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + test.T().Logf("Created MCAD AppWrapper %s/%s successfully", aw.Namespace, aw.Name) - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) + test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } -// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. -func TestMnistJobSubmit(t *testing.T) { - +func TestMNISTCheckAppWrapperStatus(t *testing.T) { test := With(t) test.T().Parallel() + // get namespace namespace := GetNamespaceWithName(test, namespaceName) //delete the namespace after test complete defer DeleteTestNamespace(test, namespace) - // Test configuration - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // SDK script - "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - // codeflare-sdk installation script - "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, - }, - } - - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - serviceAccount := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, serviceAccount, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, serviceAccount, clusterRole) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "rayjob", - Namespace: namespaceName, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_rayjob.py" + " " + namespaceName}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "codeflare-sdk", - MountPath: "/codeflare-sdk", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: Ptr(false), - SeccompProfile: &corev1.SeccompProfile{ - Type: "RuntimeDefault", - }, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - RunAsNonRoot: Ptr(true), - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "codeflare-sdk", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, - }, - }, - }, - } - - if GetClusterType(test) == KindCluster { - // Take first KinD node and redirect pod hostname requests there - node := GetNodes(test)[0] - hostname := GetClusterHostname(test) - IP := GetNodeInternalIP(test, node) - - test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) - job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ - { - IP: IP, - Hostnames: []string{hostname}, - }, - } - - // Propagate hostname into Python code as env variable - hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} - job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) - } - - job, err := test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) + aw, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Get(test.Ctx(), appwrapperName, metav1.GetOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Get(test.Ctx(), jobName, metav1.GetOptions{}) + test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), )) // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.Expect(GetAppWrapper(test, namespace, aw.Name)). + To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) } From a4590eaa09d90957d1ad26e7e9ef3479ab63a772 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 12 Dec 2023 11:13:18 +0100 Subject: [PATCH 081/369] Adjust upgrade e2e test to pause Job before upgrade --- test/upgrade/olm_upgrade_test.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go index 2abd086ea..ac0783e00 100644 --- a/test/upgrade/olm_upgrade_test.go +++ b/test/upgrade/olm_upgrade_test.go @@ -27,13 +27,14 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" . "github.com/project-codeflare/codeflare-operator/test/e2e" ) var ( namespaceName = "test-ns-olmupgrade" - appwrapperName = "mnist" + appWrapperName = "mnist" jobName = "mnist-job" ) @@ -132,13 +133,14 @@ func TestMNISTCreateAppWrapper(t *testing.T) { RestartPolicy: corev1.RestartPolicyNever, }, }, + Suspend: Ptr(true), }, } // Create an AppWrapper resource aw := &mcadv1beta1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ - Name: appwrapperName, + Name: appWrapperName, Namespace: namespace.Name, }, Spec: mcadv1beta1.AppWrapperSpec{ @@ -187,20 +189,27 @@ func TestMNISTCheckAppWrapperStatus(t *testing.T) { //delete the namespace after test complete defer DeleteTestNamespace(test, namespace) - aw, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Get(test.Ctx(), appwrapperName, metav1.GetOptions{}) + // Patch job to resume execution + patch := []byte(`[{"op":"replace","path":"/spec/suspend","value": false}]`) + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Patch(test.Ctx(), jobName, types.JSONPatchType, patch, metav1.PatchOptions{}) test.Expect(err).NotTo(HaveOccurred()) - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Get(test.Ctx(), jobName, metav1.GetOptions{}) - test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + test.T().Logf("Waiting for AppWrapper %s/%s to complete", namespace.Name, appWrapperName) + test.Eventually(AppWrapper(test, namespace, appWrapperName), TestTimeoutShort).Should( Or( WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), )) - // Assert the job has completed successfully - test.Expect(GetAppWrapper(test, namespace, aw.Name)). + // Assert the AppWrapper has completed successfully + test.Expect(GetAppWrapper(test, namespace, appWrapperName)). To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) } From ce7710d9772b201bb8868c06769ccc21755386c1 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 13 Dec 2023 17:40:21 +0000 Subject: [PATCH 082/369] Upgrade to go 1.20 and update dependencies + point to a compatible MCAD --- .github/workflows/e2e_tests.yaml | 2 +- .github/workflows/olm_tests.yaml | 2 +- .github/workflows/operator-image.yml | 2 +- .github/workflows/precommit.yml | 2 +- .github/workflows/tag-and-build.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- .github/workflows/verify_generated_files.yml | 4 +- CONTRIBUTING.md | 2 +- Dockerfile | 2 +- Makefile | 2 +- config/manager/manager.yaml | 2 +- go.mod | 50 +++---- go.sum | 137 +++++++++---------- 13 files changed, 100 insertions(+), 111 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1962a8b84..180bc23cf 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -45,7 +45,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 1dcb6f2d8..e4716bd03 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -44,7 +44,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index d2df8683c..0c155be1d 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -22,7 +22,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Login to Quay.io uses: redhat-actions/podman-login@v1 diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 7ef3a5e42..13c4c7fca 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -26,7 +26,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Activate cache uses: actions/cache@v3 diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 28d27c991..038b77b0e 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -64,7 +64,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Verify that release doesn't exist yet shell: bash {0} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7615ff77e..39ab0ef0f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -27,7 +27,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Activate cache uses: actions/cache@v3 diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index b99a9a060..e2fa269cc 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -24,7 +24,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Verify that imports are organized run: make verify-imports @@ -35,6 +35,6 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated run: make manifests && git diff --exit-code diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9fd4a3f8b..8613b92fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Here are a few things to go over before getting started with CodeFlare Operator ## Environment setup The following should be installed in your working environment: - - Go 1.19.x + - Go 1.20.x - [Download release](https://go.dev/dl/) - [Install Instructions](https://go.dev/doc/install) - [Operator SDK](https://sdk.operatorframework.io/docs/installation/) diff --git a/Dockerfile b/Dockerfile index ea02d1b1e..f5c51c3f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19.13 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.20.10 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/Makefile b/Makefile index d32fdc01f..3f0d0f52c 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ SED ?= /usr/bin/sed KUSTOMIZE_VERSION ?= v4.5.4 CODEGEN_VERSION ?= v0.27.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 -YQ_VERSION ?= v4.30.8 ## latest version that works with go1.19 +YQ_VERSION ?= v4.35.2 ## latest version that works with go1.20 OPERATOR_SDK_VERSION ?= v1.27.0 GH_CLI_VERSION ?= 2.30.0 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ae13f8af5..8a4f6451b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -28,7 +28,7 @@ spec: # it is recommended to ensure that all your Pods/Containers are restrictive. # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # versions < 1.20 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). # seccompProfile: # type: RuntimeDefault containers: diff --git a/go.mod b/go.mod index e9f358d48..ec0700b43 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/project-codeflare/codeflare-operator -go 1.19 +go 1.20 require ( github.com/onsi/gomega v1.27.10 @@ -10,28 +10,31 @@ require ( github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 - k8s.io/api v0.26.3 - k8s.io/apimachinery v0.26.3 - k8s.io/client-go v0.26.3 - k8s.io/component-base v0.26.2 + k8s.io/api v0.27.8 + k8s.io/apimachinery v0.27.8 + k8s.io/client-go v0.27.8 + k8s.io/component-base v0.27.8 k8s.io/klog/v2 v2.90.1 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 - sigs.k8s.io/controller-runtime v0.14.6 + sigs.k8s.io/controller-runtime v0.15.3 sigs.k8s.io/yaml v1.3.0 ) replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 +replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a + require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.3.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect @@ -41,7 +44,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.2.3 // indirect + github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -50,7 +53,7 @@ require ( github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/cel-go v0.12.6 // indirect + github.com/google/cel-go v0.12.7 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -65,24 +68,25 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/v3 v3.5.5 // indirect + go.etcd.io/etcd/api/v3 v3.5.7 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect + go.etcd.io/etcd/client/v3 v3.5.7 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/otel v1.19.0 // indirect @@ -101,7 +105,7 @@ require ( golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect @@ -112,13 +116,13 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.1 // indirect - k8s.io/apiserver v0.26.2 // indirect + k8s.io/apiextensions-apiserver v0.27.7 // indirect + k8s.io/apiserver v0.27.8 // indirect k8s.io/klog v1.0.0 // indirect - k8s.io/kms v0.26.2 // indirect - k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect + k8s.io/kms v0.27.8 // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/metrics v0.26.2 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect diff --git a/go.sum b/go.sum index 9af7ce38d..81c66b132 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a h1:11bIdGaYS1YOxNYyvR8S9VNNhcMDMvjQxdJ4VMEp810= +github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -49,6 +51,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -75,15 +79,14 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= +github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -92,7 +95,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= @@ -101,12 +103,9 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= @@ -114,7 +113,6 @@ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -126,11 +124,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -138,8 +134,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= @@ -194,8 +190,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/cel-go v0.12.6 h1:kjeKudqV0OygrAqA9fX6J55S8gj+Jre2tckIm5RoG4M= -github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= +github.com/google/cel-go v0.12.7 h1:jM6p55R0MKBg79hZjn1zs2OlrywZ1Vk00rxVvad1/O0= +github.com/google/cel-go v0.12.7/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -348,10 +344,12 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -395,40 +393,37 @@ github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 h1:6ILHYAFxDkAnQu3CJebGQPQGcmcy7/E/AhRiea6yOTc= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 h1:aoOajXNG2QnTPmnSzbCXLFBXrVk7bzQGrwwVzTXSOzo= github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -440,7 +435,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -463,7 +458,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -476,16 +471,16 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= -go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= -go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= -go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= +go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= +go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= +go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= +go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= +go.etcd.io/etcd/client/v2 v2.305.7 h1:AELPkjNR3/igjbO7CjyF1fPuVPjrblliiKj+Y6xSGOU= +go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4= +go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= +go.etcd.io/etcd/pkg/v3 v3.5.7 h1:obOzeVwerFwZ9trMWapU/VjDcYUJb5OfgC1zqEGWO/0= +go.etcd.io/etcd/raft/v3 v3.5.7 h1:aN79qxLmV3SvIq84aNTliYGmjwsW6NqJSnqmI1HLJKc= +go.etcd.io/etcd/server/v3 v3.5.7 h1:BTBD8IJUV7YFgsczZMHhMTS67XuA4KpRquL0MFOJGRk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -515,7 +510,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -527,8 +522,7 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -568,7 +562,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -619,7 +612,6 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= @@ -630,7 +622,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -691,7 +682,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -745,7 +735,6 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -775,17 +764,17 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= +gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -840,7 +829,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= @@ -862,9 +850,7 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -918,26 +904,26 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.3 h1:emf74GIQMTik01Aum9dPP0gAypL8JTLl/lHa4V9RFSU= -k8s.io/api v0.26.3/go.mod h1:PXsqwPMXBSBcL1lJ9CYDKy7kIReUydukS5JiRlxC3qE= -k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= -k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= -k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k= -k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= -k8s.io/apiserver v0.26.2 h1:Pk8lmX4G14hYqJd1poHGC08G03nIHVqdJMR0SD3IH3o= -k8s.io/apiserver v0.26.2/go.mod h1:GHcozwXgXsPuOJ28EnQ/jXEM9QeG6HT22YxSNmpYNh8= -k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s= -k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ= -k8s.io/component-base v0.26.2 h1:IfWgCGUDzrD6wLLgXEstJKYZKAFS2kO+rBRi0p3LqcI= -k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= +k8s.io/api v0.27.8 h1:Ja93gbyII5c3TJzWefEwGhlqC5SZksWhzRS+OYHIocU= +k8s.io/api v0.27.8/go.mod h1:2HuWJC6gpx4UScY+ezziNzv6j6Jqd2q0rGgobYSSjcs= +k8s.io/apiextensions-apiserver v0.27.7 h1:YqIOwZAUokzxJIjunmUd4zS1v3JhK34EPXn+pP0/bsU= +k8s.io/apiextensions-apiserver v0.27.7/go.mod h1:x0p+b5a955lfPz9gaDeBy43obM12s+N9dNHK6+dUL+g= +k8s.io/apimachinery v0.27.8 h1:Xg+ogjDm8s7KmV3vZGf7uOZ0jrC6FPy2Lk/h7BIRmvg= +k8s.io/apimachinery v0.27.8/go.mod h1:EIXLxLt/b1muPITiF5zlrcm7I+YnXsIgM+0GdnPTQvA= +k8s.io/apiserver v0.27.8 h1:fdrEm98wl1lxFoF8tCdQiYQOfmU3TOGkQr4YTLYHwzc= +k8s.io/apiserver v0.27.8/go.mod h1:8yLtDnOdRq9fubaea0zQ3AVheQUinb5Y/RqBUdQMT0Y= +k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= +k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= +k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= +k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.26.2 h1:GM1gg3tFK3OUU/QQFi93yGjG3lJT8s8l3Wkn2+VxBLM= -k8s.io/kms v0.26.2/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= -k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= -k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/kms v0.27.8 h1:D7s2vZAFWKD1+1Yg6Y4C+l7k+ZjEsMc7BjY0QRKAOY8= +k8s.io/kms v0.27.8/go.mod h1:+yIM/3y+UgdaZSQaMJObNONp0i237t/dVrxswECh7a0= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/metrics v0.26.2 h1:2gUvUWWnHPdE2tyA5DvyHC8HGryr+izhY9i5dzLP06s= k8s.io/metrics v0.26.2/go.mod h1:PX1wm9REV9hSGuw9GcXTFNDgab1KRXck3mNeiLYbRho= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= @@ -945,16 +931,15 @@ k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 h1:+xBL5uTc+BkPBwmMi3vYfUJjq+N3K+H6PXeETwf5cPI= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= +sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= +sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 h1:puQ5YlyBjhxg+OQ1YPMJXwtk7WhC4E6AlWIQ9pC8jws= sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3/go.mod h1:9nUXR/EgdYZto1aQ6yhwOksPR7J979jSyOqic1IgaOo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 510a3a82c2cbd3fb4272c797f0aa03354d7aa69e Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 19 Dec 2023 09:33:28 +0000 Subject: [PATCH 083/369] Point the CFO to compatible MCAD --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ec0700b43..a8ab01f28 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 -replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a +replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e require ( github.com/NYTimes/gziphandler v1.1.1 // indirect diff --git a/go.sum b/go.sum index 81c66b132..0082a6625 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a h1:11bIdGaYS1YOxNYyvR8S9VNNhcMDMvjQxdJ4VMEp810= -github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -393,6 +391,8 @@ github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e h1:vhauNtZc/4DWVIP+6NyyDifBW9epSxpTbJzV7UKhG9M= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 944300bd1bfc2bb1ef7420e571e86a51ff7e62ff Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 19 Dec 2023 12:17:03 +0100 Subject: [PATCH 084/369] Remove MCAD replace line in go.mod --- go.mod | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a8ab01f28..0293d7dea 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/instascale v0.3.1 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 @@ -22,8 +22,6 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 -replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e - require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect From 8b6232d44969683821e442b6a9702497196ea85f Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 20 Dec 2023 07:39:07 +0100 Subject: [PATCH 085/369] e2e tests: Export all KinD logs --- .github/workflows/e2e_tests.yaml | 6 ++++++ .github/workflows/olm_tests.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 180bc23cf..0786c290c 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -92,6 +92,12 @@ jobs: echo "Printing KubeRay operator logs" kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + - name: Export all KinD pod logs + uses: ./common/github-actions/kind-export-logs + if: always() && steps.deploy.outcome == 'success' + with: + output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} + - name: Upload logs uses: actions/upload-artifact@v3 if: always() && steps.deploy.outcome == 'success' diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index e4716bd03..6889604bb 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -185,6 +185,12 @@ jobs: echo "Printing KubeRay operator logs" kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + - name: Export all KinD pod logs + uses: ./common/github-actions/kind-export-logs + if: always() && steps.deploy.outcome == 'success' + with: + output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} + - name: Upload logs uses: actions/upload-artifact@v3 if: always() && steps.deploy.outcome == 'success' From 80df38e7cb102fed8cb4d03e61eb85260604523d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 21 Dec 2023 16:31:51 +0100 Subject: [PATCH 086/369] Add ChristianZaccaria as reviewer --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index efdcada15..a3556780f 100644 --- a/OWNERS +++ b/OWNERS @@ -9,6 +9,7 @@ approvers: reviewers: - anishasthana - astefanutti + - ChristianZaccaria - dimakis - jbusche - kpostoffice From 86ca67c95d624a4bd6c1b5ccb4c5db176b2cc2d5 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 21 Dec 2023 16:35:21 +0100 Subject: [PATCH 087/369] Add Fiona-Waters as reviewer --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index a3556780f..c962aaaf2 100644 --- a/OWNERS +++ b/OWNERS @@ -11,6 +11,7 @@ reviewers: - astefanutti - ChristianZaccaria - dimakis + - Fiona-Waters - jbusche - kpostoffice - maxusmusti From 575ac74a98cb0b069e9ffc36df61921cf71c1970 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 22 Dec 2023 10:35:00 +0100 Subject: [PATCH 088/369] Fix MCAD version in Makefile --- .github/workflows/tag-and-build.yml | 4 ++++ Makefile | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 038b77b0e..02c850f79 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -109,6 +109,10 @@ jobs: password: ${{ secrets.QUAY_TOKEN }} registry: quay.io + - name: Align go.mod and go.sum dependencies for released components + run: | + make modules + - name: Image Build and Push run: | make build diff --git a/Makefile b/Makefile index 3f0d0f52c..53c0d9af4 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ modules: ## Update Go dependencies. go mod tidy .PHONY: build -build: modules fmt vet ## Build manager binary. +build: fmt vet ## Build manager binary. go build \ -ldflags " \ -X 'main.OperatorVersion=$(BUILD_VERSION)' \ @@ -169,7 +169,7 @@ build: modules fmt vet ## Build manager binary. -o bin/manager main.go .PHONY: run -run: modules manifests fmt vet ## Run a controller from your host. +run: manifests fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build From c5cc868b0f376cbdf833c532a3d7c94b02c59ee9 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 8 Jan 2024 16:49:31 -0500 Subject: [PATCH 089/369] remove auto add issue workflow Signed-off-by: Kevin --- .github/workflows/auto-add-issues.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/workflows/auto-add-issues.yml diff --git a/.github/workflows/auto-add-issues.yml b/.github/workflows/auto-add-issues.yml deleted file mode 100644 index c1e940997..000000000 --- a/.github/workflows/auto-add-issues.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Label new issues as needs-triage and add to CodeFlare Sprint Board -on: - issues: - types: - - opened -jobs: - add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/project-codeflare/projects/8 - github-token: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} From 35fed6730f280ed7797d1d5df33cd7af04965b70 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 10 Jan 2024 09:34:53 +0100 Subject: [PATCH 090/369] Upgrade Kuberay to version v1.0.0 --- Makefile | 2 +- go.mod | 2 +- go.sum | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 53c0d9af4..8f15e0b45 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.0.0-rc.1 +KUBERAY_VERSION ?= v1.0.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/go.mod b/go.mod index 0293d7dea..29e863d4f 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e - github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 + github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 k8s.io/apimachinery v0.27.8 diff --git a/go.sum b/go.sum index 0082a6625..4fdd10218 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 h1:aoOajXNG2QnTPmnSzbCXLFBXrVk7bzQGrwwVzTXSOzo= -github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= +github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -435,7 +435,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= From 0e7bd88bb15ee2eba29f395e9e7e4c8e1903eaf4 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 11 Jan 2024 12:41:45 +0100 Subject: [PATCH 091/369] Remove SDK e2e test --- test/e2e/install-codeflare-sdk.sh | 22 --- test/e2e/mnist_raycluster_sdk.py | 86 ----------- test/e2e/mnist_raycluster_sdk_test.go | 215 -------------------------- 3 files changed, 323 deletions(-) delete mode 100755 test/e2e/install-codeflare-sdk.sh delete mode 100644 test/e2e/mnist_raycluster_sdk.py delete mode 100644 test/e2e/mnist_raycluster_sdk_test.go diff --git a/test/e2e/install-codeflare-sdk.sh b/test/e2e/install-codeflare-sdk.sh deleted file mode 100755 index e90f4071c..000000000 --- a/test/e2e/install-codeflare-sdk.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# go to codeflare-sdk folder and install codeflare-sdk -cd .. - -# Install Poetry and configure virtualenvs -pip install poetry -poetry config virtualenvs.create false - -cd codeflare-sdk -# Clone the CodeFlare SDK repository -git clone --branch main https://github.com/project-codeflare/codeflare-sdk.git - -cd codeflare-sdk - -# Lock dependencies and install them -poetry lock --no-update -poetry install --with test,docs - -# Return to the workdir -cd ../.. -cd workdir diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py deleted file mode 100644 index d171c9b08..000000000 --- a/test/e2e/mnist_raycluster_sdk.py +++ /dev/null @@ -1,86 +0,0 @@ -import sys -import os - -from time import sleep - -from torchx.specs.api import AppState, is_terminal - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration -from codeflare_sdk.job.jobs import DDPJobDefinition - -namespace = sys.argv[1] -ray_image = os.getenv('RAY_IMAGE') -host = os.getenv('CLUSTER_HOSTNAME') - -ingress_options = {} -if host is not None: - ingress_options = { - "ingresses": [ - { - "ingressName": "ray-dashboard", - "port": 8265, - "pathType": "Prefix", - "path": "/", - "host": host, - }, - ] - } - - -cluster = Cluster(ClusterConfiguration( - name='mnist', - namespace=namespace, - num_workers=1, - head_cpus='500m', - head_memory=2, - min_cpus='500m', - max_cpus=1, - min_memory=0.5, - max_memory=2, - num_gpus=0, - instascale=False, - image=ray_image, - ingress_options=ingress_options, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() - -jobdef = DDPJobDefinition( - name="mnist", - script="mnist.py", - scheduler_args={"requirements": "requirements.txt"}, -) -job = jobdef.submit(cluster) - -done = False -time = 0 -timeout = 900 -while not done: - status = job.status() - if is_terminal(status.state): - break - if not done: - print(status) - if timeout and time >= timeout: - raise TimeoutError(f"job has timed out after waiting {timeout}s") - sleep(5) - time += 5 - -print(f"Job has completed: {status.state}") - -print(job.logs()) - -cluster.down() - -if not status.state == AppState.SUCCEEDED: - exit(1) -else: - exit(0) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go deleted file mode 100644 index be6933f67..000000000 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ /dev/null @@ -1,215 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. -// Asserts successful completion of the training job. -// -// This covers the installation of the CodeFlare SDK, as well as the RBAC required -// for the SDK to successfully perform requests to the cluster, on behalf of the -// impersonated user. -func TestMNISTRayClusterSDK(t *testing.T) { - test := With(t) - test.T().Parallel() - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // SDK script - "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - // codeflare-sdk installation script - "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, - }, - } - - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, sa, clusterRole) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "RAY_IMAGE", Value: GetRayImage()}, - }, - Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_raycluster_sdk.py" + " " + namespace.Name}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "codeflare-sdk", - MountPath: "/codeflare-sdk", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: Ptr(false), - SeccompProfile: &corev1.SeccompProfile{ - Type: "RuntimeDefault", - }, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - RunAsNonRoot: Ptr(true), - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "codeflare-sdk", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: sa.Name, - }, - }, - }, - } - if GetClusterType(test) == KindCluster { - // Take first KinD node and redirect pod hostname requests there - node := GetNodes(test)[0] - hostname := GetClusterHostname(test) - IP := GetNodeInternalIP(test, node) - - test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) - job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ - { - IP: IP, - Hostnames: []string{hostname}, - }, - } - - // Propagate hostname into Python code as env variable - hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} - job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) - } - - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) -} From 233de7ea5f895ef5be319386f437f46d3bdde048 Mon Sep 17 00:00:00 2001 From: Mark Campbell Date: Fri, 19 Jan 2024 12:34:36 +0000 Subject: [PATCH 092/369] Added Workflow job to update CFO image (#398) --- .github/workflows/tag-and-build.yml | 7 ++++++- Makefile | 4 ++-- config/manager/kustomization.yaml | 16 ++++++++++++++++ config/manager/manager.yaml | 2 +- config/manager/params.env | 1 + config/manager/params.yaml | 3 +++ config/manifests/kustomization.yaml | 3 ++- 7 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 config/manager/params.env create mode 100644 config/manager/params.yaml diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 02c850f79..f04d43d67 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -102,6 +102,11 @@ jobs: sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile + - name: Update image version in params.env + run: | + VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env + shell: bash + - name: Login to Quay.io uses: redhat-actions/podman-login@v1 with: @@ -138,7 +143,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md *.yaml Makefile go.mod go.sum' + file_pattern: 'README.md *.yaml Makefile go.mod go.sum *.env' create_branch: true branch: ${{ env.PR_BRANCH_NAME }} diff --git a/Makefile b/Makefile index 8f15e0b45..0e2b24c3d 100644 --- a/Makefile +++ b/Makefile @@ -198,7 +198,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && IMAGE=$(IMG) perl -i -pe 's/codeflare-operator-controller-image=(.*)$$/codeflare-operator-controller-image=$$ENV{"IMAGE"}/' params.env $(KUSTOMIZE) build config/${ENV} | kubectl apply -f - git restore config/* @@ -283,8 +283,8 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. + cd config/manager && IMAGE=$(IMG) perl -i -pe 's/codeflare-operator-controller-image=(.*)$$/codeflare-operator-controller-image=$$ENV{"IMAGE"}/' params.env $(OPERATOR_SDK) generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 4fe5ccaa3..847eca176 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -3,3 +3,19 @@ resources: generatorOptions: disableNameSuffixHash: true + +configMapGenerator: +- name: stack-config + envs: + - params.env +configurations: + - params.yaml + +vars: +- name: codeflare_operator_controller_image + objref: + kind: ConfigMap + name: stack-config + apiVersion: v1 + fieldref: + fieldpath: data.codeflare-operator-controller-image diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 8a4f6451b..20a34fe9d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -34,7 +34,7 @@ spec: containers: - command: - /manager - image: controller:latest + image: $(codeflare_operator_controller_image) imagePullPolicy: Always name: manager securityContext: diff --git a/config/manager/params.env b/config/manager/params.env new file mode 100644 index 000000000..8c9de7847 --- /dev/null +++ b/config/manager/params.env @@ -0,0 +1 @@ +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.0.1 diff --git a/config/manager/params.yaml b/config/manager/params.yaml new file mode 100644 index 000000000..43509ff29 --- /dev/null +++ b/config/manager/params.yaml @@ -0,0 +1,3 @@ +varReference: + - path: spec/template/spec/containers[]/image + kind: Deployment diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index 958a9e247..161a419d3 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -1,8 +1,9 @@ # These resources constitute the fully configured set of manifests # used to generate the 'manifests/' directory in a bundle. resources: -- bases/codeflare-operator.clusterserviceversion.yaml - ../default - ../scorecard +- bases/codeflare-operator.clusterserviceversion.yaml + apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization From 4416347aeed370987e933cd04bc58de555e66a84 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Fri, 19 Jan 2024 13:44:22 +0000 Subject: [PATCH 093/369] Update project-codeflare-release.yml (#445) --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 77d0a0a1f..59f24be07 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -70,7 +70,7 @@ jobs: - name: Release MCAD run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 003d61b3ae4ffe5c31cea8071d8d3614c0bdbcc5 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 19 Jan 2024 14:26:58 +0000 Subject: [PATCH 094/369] Update dependency versions for release v1.1.0 --- Makefile | 4 ++-- README.md | 10 +++++----- config/crd/mcad/kustomization.yaml | 2 +- config/manager/params.env | 2 +- go.mod | 5 ++--- go.sum | 11 ++++------- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 0e2b24c3d..dccaea286 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.3.1 +INSTASCALE_VERSION ?= v0.4.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.38.1 +MCAD_VERSION ?= v1.39.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index c3048b299..9488b3b46 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.1) | -| Multi-Cluster App Dispatcher | [v1.38.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.1) | -| CodeFlare-SDK | [v0.12.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.12.1) | -| InstaScale | [v0.3.1](https://github.com/project-codeflare/instascale/releases/tag/v0.3.1) | -| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | +| CodeFlare Operator | [v1.1.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.1.0) | +| Multi-Cluster App Dispatcher | [v1.39.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.39.0) | +| CodeFlare-SDK | [v0.13.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.13.0) | +| InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | +| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | ## Development diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 3cd50c483..0e4ebfa7a 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.1 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.39.0 diff --git a/config/manager/params.env b/config/manager/params.env index 8c9de7847..ace2f2f29 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.0.1 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.1.0 diff --git a/go.mod b/go.mod index 29e863d4f..149ba7720 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 - github.com/project-codeflare/instascale v0.3.1 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e + github.com/project-codeflare/instascale v0.4.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 @@ -116,7 +116,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.7 // indirect k8s.io/apiserver v0.27.8 // indirect - k8s.io/klog v1.0.0 // indirect k8s.io/kms v0.27.8 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/metrics v0.26.2 // indirect diff --git a/go.sum b/go.sum index 4fdd10218..797c519bd 100644 --- a/go.sum +++ b/go.sum @@ -125,7 +125,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -389,10 +388,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 h1:81+ma1mchF/LtAGsf+poAt50kJ/fLYjoTAcZOxci1Yc= github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= -github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= -github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e h1:vhauNtZc/4DWVIP+6NyyDifBW9epSxpTbJzV7UKhG9M= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= +github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= +github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -916,8 +915,6 @@ k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.27.8 h1:D7s2vZAFWKD1+1Yg6Y4C+l7k+ZjEsMc7BjY0QRKAOY8= From b584b5eeb127b7367dbcc3fdbc119704729bbef6 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Fri, 19 Jan 2024 16:35:01 +0000 Subject: [PATCH 095/369] update: change CSV alm-examples to '' --- .../bases/codeflare-operator.clusterserviceversion.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 77295679c..c3db15ce4 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -2,7 +2,7 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: '[]' + alm-examples: '' capabilities: Basic Install categories: AI/Machine Learning, Big Data operatorframework.io/suggested-namespace: openshift-operators From a040aa637e0b47007ec33ef52028619fd627cb34 Mon Sep 17 00:00:00 2001 From: Christian Zaccaria <73656840+ChristianZaccaria@users.noreply.github.com> Date: Fri, 19 Jan 2024 17:42:35 +0000 Subject: [PATCH 096/369] Revert "update: change CSV alm-examples to ''" (#448) --- .../bases/codeflare-operator.clusterserviceversion.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index c3db15ce4..77295679c 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -2,7 +2,7 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: '' + alm-examples: '[]' capabilities: Basic Install categories: AI/Machine Learning, Big Data operatorframework.io/suggested-namespace: openshift-operators From a4cf5752b1a3ba32a8b14786b876a724b0dd4605 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 22 Jan 2024 09:17:53 +0000 Subject: [PATCH 097/369] Fixed MCAD release --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 59f24be07..77d0a0a1f 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -70,7 +70,7 @@ jobs: - name: Release MCAD run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 89efe670eb274e7a5f6cb91060a3bf90f6df71a5 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Mon, 22 Jan 2024 11:18:27 +0000 Subject: [PATCH 098/369] update: change CSV alm-examples Signed-off-by: Eoin Gallinagh --- ...eflare-operator.clusterserviceversion.yaml | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 77295679c..5531e61a8 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -2,7 +2,45 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: '[]' + alm-examples: |- + [{ + "apiVersion": "workload.codeflare.dev/v1beta1", + "kind": "AppWrapper", + "metadata": {"name": "0001-aw-generic-deployment-1"}, + "spec": { + "resources": { + "GenericItems": [{ + "replicas": 1, + "generictemplate": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "0001-aw-generic-deployment-1", + "labels": {"app": "0001-aw-generic-deployment-1"} + }, + "spec": { + "selector": {"matchLabels": {"app": "0001-aw-generic-deployment-1"}}, + "replicas": 2, + "template": { + "metadata": {"labels": {"app": "0001-aw-generic-deployment-1"}}, + "spec": { + "containers": [{ + "name": "0001-aw-generic-deployment-1", + "image": "kicbase/echo-server:1.0", + "ports": [{"containerPort": 80}], + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "100m", "memory": "256Mi"} + } + }] + } + } + } + } + }] + } + } + }] capabilities: Basic Install categories: AI/Machine Learning, Big Data operatorframework.io/suggested-namespace: openshift-operators From 0cf73e844491fa7f995af611b4b0f2680e0a4ef3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 10 Jan 2024 13:13:47 +0100 Subject: [PATCH 099/369] Download MNIST dataset from specific location --- go.mod | 2 +- go.sum | 4 +- test/e2e/mnist.py | 33 ++++++++++++++- test/e2e/mnist_pytorch_mcad_job_test.go | 3 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 42 ++++++++++--------- test/upgrade/olm_upgrade_test.go | 3 +- 6 files changed, 61 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 149ba7720..fbac9a7ff 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 + github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 github.com/project-codeflare/instascale v0.4.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 github.com/ray-project/kuberay/ray-operator v1.0.0 diff --git a/go.sum b/go.sum index 797c519bd..e501c2351 100644 --- a/go.sum +++ b/go.sum @@ -386,8 +386,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 h1:81+ma1mchF/LtAGsf+poAt50kJ/fLYjoTAcZOxci1Yc= -github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= +github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 h1:knUKEKvfEzVuSwQ4NAe2+I/Oxo4WztU5rYR8d/F66Lw= +github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index 244c84d29..4cfe0b43b 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -15,6 +15,7 @@ import os import torch +import requests from pytorch_lightning import LightningModule, Trainer from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn @@ -32,6 +33,8 @@ print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) +print("MNIST_DATASET_URL: is ", os.getenv("MNIST_DATASET_URL")) +MNIST_DATASET_URL = os.getenv("MNIST_DATASET_URL") class LitMNIST(LightningModule): def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): @@ -110,8 +113,34 @@ def configure_optimizers(self): #################### def prepare_data(self): - # download - print("Downloading MNIST dataset...") + datasetFiles = [ + "t10k-images-idx3-ubyte", + "t10k-labels-idx1-ubyte", + "train-images-idx3-ubyte", + "train-labels-idx1-ubyte" + ] + + # Create required folder structure + downloadLocation = os.path.join(PATH_DATASETS, "MNIST", "raw") + os.makedirs(downloadLocation, exist_ok=True) + print(f"{downloadLocation} folder_path created!") + + for file in datasetFiles: + print(f"Downloading MNIST dataset {file}... to path : {downloadLocation}") + response = requests.get(f"{MNIST_DATASET_URL}{file}", stream=True) + filePath = os.path.join(downloadLocation, file) + + #to download dataset file + try: + if response.status_code == 200: + open(filePath, 'wb').write(response.content) + print(f"{file}: Downloaded and saved zipped file to path - {filePath}") + else: + print(f"Failed to download file {file}") + except Exception as e: + print(e) + print(f"Downloaded MNIST dataset to... {downloadLocation}") + MNIST(self.data_dir, train=True, download=True) MNIST(self.data_dir, train=False, download=True) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index cbe55fed2..883457e95 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -79,7 +79,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Name: "job", Image: GetPyTorchImage(), Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index b8d3f4d06..21bd98ad8 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -17,7 +17,6 @@ limitations under the License. package e2e import ( - "encoding/base64" "testing" . "github.com/onsi/gomega" @@ -143,13 +142,6 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - { - Name: "init-myservice", - Image: "busybox:1.28", - Command: []string{"sh", "-c", "until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"}, - }, - }, Containers: []corev1.Container{ { Name: "ray-worker", @@ -230,21 +222,29 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Spec: rayv1.RayJobSpec{ Entrypoint: "python /home/ray/jobs/mnist.py", - RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` -{ - "pip": [ - "pytorch_lightning==1.5.10", - "torchmetrics==0.9.1", - "torchvision==0.12.0" - ], - "env_vars": { - } -} -`)), + RuntimeEnvYAML: ` + pip: + - pytorch_lightning==1.5.10 + - torchmetrics==0.9.1 + - torchvision==0.12.0 + env_vars: + MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `" +`, ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, }, ShutdownAfterJobFinishes: false, + SubmitterPodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Image: GetRayImage(), + Name: "rayjob-submitter-pod", + }, + }, + }, + }, }, } rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) @@ -256,6 +256,10 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) + // Wait for Ray job id to be available, this value is needed for writing logs in defer + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutShort). + Should(WithTransform(RayJobId, Not(BeEmpty()))) + // Retrieving the job logs once it has completed or timed out defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go index ac0783e00..8fb4e65fb 100644 --- a/test/upgrade/olm_upgrade_test.go +++ b/test/upgrade/olm_upgrade_test.go @@ -96,7 +96,8 @@ func TestMNISTCreateAppWrapper(t *testing.T) { Name: "job", Image: GetPyTorchImage(), Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ From a40442c3544d82d93b366ca912c971f5fd550bc8 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 25 Jan 2024 11:50:16 +0000 Subject: [PATCH 100/369] Upgrade protobuf dependency to address CVE --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fbac9a7ff..41b42bf37 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index e501c2351..e9ce71152 100644 --- a/go.sum +++ b/go.sum @@ -865,8 +865,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 195c9ecae7928f4c29fe8ce3b6c16fae2ae45b1f Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 25 Jan 2024 13:56:38 -0500 Subject: [PATCH 101/369] add roles for admin and editor to operator Signed-off-by: Kevin --- config/rbac/admin_role.yaml | 22 ++++++++++++++++++++++ config/rbac/editor_role.yaml | 24 ++++++++++++++++++++++++ config/rbac/kustomization.yaml | 2 ++ 3 files changed, 48 insertions(+) create mode 100644 config/rbac/admin_role.yaml create mode 100644 config/rbac/editor_role.yaml diff --git a/config/rbac/admin_role.yaml b/config/rbac/admin_role.yaml new file mode 100644 index 000000000..b4f4728ca --- /dev/null +++ b/config/rbac/admin_role.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: clusterrole-admin + labels: + rbac.authorization.kubernetes.io/aggregate-to-admin: "true" +rules: +- apiGroups: + - quota.codeflare.dev + resources: + - quotasubtrees + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/config/rbac/editor_role.yaml b/config/rbac/editor_role.yaml new file mode 100644 index 000000000..aa341297d --- /dev/null +++ b/config/rbac/editor_role.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: clusterrole-edit + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: + - workload.codeflare.dev + resources: + - schedulingspecs + - appwrappers + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index cdb307c82..a74a16f1d 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -4,6 +4,8 @@ resources: # if your manager will use a service account that exists at # runtime. Be sure to update RoleBinding and ClusterRoleBinding # subjects if changing service account names. +- admin_role.yaml +- editor_role.yaml - service_account.yaml - role.yaml - role_binding.yaml From f52d75d43628cfaffd7c6698f8abcb26e1554794 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 26 Jan 2024 16:48:15 +0000 Subject: [PATCH 102/369] Update dependency to address CVE --- go.mod | 21 ++++++++++----------- go.sum | 42 ++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 41b42bf37..c8cca86f1 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,6 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -75,10 +74,10 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.15.1 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.46.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect @@ -95,13 +94,13 @@ require ( go.opentelemetry.io/otel/trace v1.19.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index e9ce71152..a1ad6ba03 100644 --- a/go.sum +++ b/go.sum @@ -341,8 +341,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= @@ -397,27 +395,27 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -537,8 +535,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -613,16 +611,16 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -692,13 +690,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -708,8 +706,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ffbd647ab39d3075515c191d2f217d217eb0ce19 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 30 Jan 2024 17:17:37 +0000 Subject: [PATCH 103/369] Add Host to Container port mapping for KinD Cluster --- test/e2e/kind.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 5b39a164e..db53893c4 100755 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -24,6 +24,10 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP kubeadmConfigPatches: - | kind: InitConfiguration From f8839e9269aa16b0875f5ee42ec6d622bea71888 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 31 Jan 2024 15:41:02 +0100 Subject: [PATCH 104/369] Raise KinD Ingress controller version to v1.9.6 for manual setup --- test/e2e/kind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index db53893c4..7b7e9053e 100755 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -15,7 +15,7 @@ # limitations under the License. set -euo pipefail -: "${INGRESS_NGINX_VERSION:=controller-v1.6.4}" +: "${INGRESS_NGINX_VERSION:=controller-v1.9.6}" echo "Creating KinD cluster" cat < Date: Wed, 24 Jan 2024 23:02:10 +0000 Subject: [PATCH 105/369] Added e2e test for nodepools --- test/e2e/instascale_app_wrapper.go | 1 + test/e2e/instascale_nodepool_test.go | 66 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 test/e2e/instascale_nodepool_test.go diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index e030817f3..45305ddab 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -48,6 +48,7 @@ func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *cor Image: GetPyTorchImage(), Env: []corev1.EnvVar{ {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, Args: []string{"$PYTHONUSERBASE"}, diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go new file mode 100644 index 000000000..17945040c --- /dev/null +++ b/test/e2e/instascale_nodepool_test.go @@ -0,0 +1,66 @@ +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestInstascaleNodepool(t *testing.T) { + + test := With(t) + test.T().Parallel() + + clusterType := GetClusterType(test) + if clusterType != HypershiftCluster { + test.T().Skipf("Skipping test as not running on an Hypershift cluster, resolved cluster type: %s", clusterType) + } + + namespace := test.NewTestNamespace() + + // Test configuration + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + //create OCM connection + connection := CreateOCMConnection(test) + defer connection.Close() + + // check existing cluster resources + // look for a node pool with a label key equal to aw name - expect NOT to find it + test.Expect(GetNodePools(test, connection)). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + + // Setup batch job and AppWrapper + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + // look for a node pool with a label key equal to aw name - expect to find it + test.Eventually(NodePools(test, connection), TestTimeoutLong). + Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + + // assert that the AppWrapper goes to "Completed" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // look for a node pool with a label key equal to aw name - expect NOT to find it + test.Eventually(NodePools(test, connection), TestTimeoutLong). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + +} From d85339bfdf1041fd14524797a3a0312dae81280e Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 31 Jan 2024 13:38:35 +0000 Subject: [PATCH 106/369] address feedback --- test/e2e/instascale_nodepool_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go index 17945040c..8721663d0 100644 --- a/test/e2e/instascale_nodepool_test.go +++ b/test/e2e/instascale_nodepool_test.go @@ -1,6 +1,7 @@ package e2e import ( + "fmt" "testing" . "github.com/onsi/gomega" @@ -34,14 +35,15 @@ func TestInstascaleNodepool(t *testing.T) { connection := CreateOCMConnection(test) defer connection.Close() - // check existing cluster resources - // look for a node pool with a label key equal to aw name - expect NOT to find it - test.Expect(GetNodePools(test, connection)). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) - // Setup batch job and AppWrapper aw := instaScaleJobAppWrapper(test, namespace, cm) + expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) + // check existing cluster resources + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it + test.Expect(GetNodePools(test, connection)). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) + // apply AppWrapper to cluster _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) @@ -51,16 +53,16 @@ func TestInstascaleNodepool(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // look for a node pool with a label key equal to aw name - expect to find it + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect to find it test.Eventually(NodePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) // assert that the AppWrapper goes to "Completed" state test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for a node pool with a label key equal to aw name - expect NOT to find it + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it test.Eventually(NodePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) } From 1264faabc1835b3aafb5e11fc31c4b1bbb9a1382 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 1 Feb 2024 15:33:52 +0100 Subject: [PATCH 107/369] Adjust existing e2e tests to provide custom Pypi index URL --- README.md | 15 +++++++ go.mod | 2 +- go.sum | 4 +- test/e2e/mnist_pytorch_mcad_job_test.go | 2 + test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 + test/odh/mcad_ray_test.go | 42 ++++++++++++++++++- test/odh/notebook.go | 12 ++---- test/odh/resources/custom-nb-small.yaml | 2 +- test/odh/resources/mnist.py | 33 ++++++++++++++- test/odh/resources/mnist_ray_mini.ipynb | 5 ++- test/odh/resources/requirements.txt | 2 + test/odh/template.go | 40 ++++++++++++++++++ 12 files changed, 143 insertions(+), 18 deletions(-) create mode 100644 test/odh/template.go diff --git a/README.md b/README.md index 9488b3b46..de350d042 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,21 @@ The e2e tests can be executed locally by running the following commands: Alternatively, You can run the e2e test(s) from your IDE / debugger. +#### Testing on disconnected cluster + +To properly run e2e tests on disconnected cluster user has to provide additional environment variables to properly configure testing environment: + +- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job using MCAD +- `CODEFLARE_TEST_RAY_IMAGE` - image tag for Ray cluster image +- `MNIST_DATASET_URL` - URL where MNIST dataset is available +- `PIP_INDEX_URL` - URL where PyPI server with needed dependencies is running +- `PIP_TRUSTED_HOST` - PyPI server hostname + +For ODH tests additional environment variables are needed: + +- `NOTEBOOK_IMAGE_STREAM_NAME` - name of the ODH Notebook ImageStream to be used +- `ODH_NAMESPACE` - namespace where ODH is installed + ## Release 1. Invoke [project-codeflare-release.yaml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/project-codeflare-release.yml) diff --git a/go.mod b/go.mod index c8cca86f1..252653db6 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 + github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 github.com/project-codeflare/instascale v0.4.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 github.com/ray-project/kuberay/ray-operator v1.0.0 diff --git a/go.sum b/go.sum index a1ad6ba03..b36099d6c 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 h1:knUKEKvfEzVuSwQ4NAe2+I/Oxo4WztU5rYR8d/F66Lw= -github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= +github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 h1:30LG8751WElZmWA3mVS8l23l2oZnUCqbDkLCyy0U/p0= +github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 883457e95..a642d31ca 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -81,6 +81,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Env: []corev1.EnvVar{ {Name: "PYTHONUSERBASE", Value: "/workdir"}, {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, + {Name: "PIP_INDEX_URL", Value: GetPipIndexURL()}, + {Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 21bd98ad8..1118079ef 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -229,6 +229,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { - torchvision==0.12.0 env_vars: MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `" + PIP_INDEX_URL: "` + GetPipIndexURL() + `" + PIP_TRUSTED_HOST: "` + GetPipTrustedHost() + `" `, ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, diff --git a/test/odh/mcad_ray_test.go b/test/odh/mcad_ray_test.go index 770b64d9d..b72456c59 100644 --- a/test/odh/mcad_ray_test.go +++ b/test/odh/mcad_ray_test.go @@ -38,8 +38,8 @@ func TestMCADRay(t *testing.T) { config := CreateConfigMap(test, namespace.Name, map[string][]byte{ // MNIST Ray Notebook jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"), - "mnist.py": ReadFile(test, "resources/mnist.py"), - "requirements.txt": ReadFile(test, "resources/requirements.txt"), + "mnist.py": readMnistPy(test), + "requirements.txt": readRequirementsTxt(test), }) // Create RBAC, retrieve token for user with limited rights @@ -59,6 +59,11 @@ func TestMCADRay(t *testing.T) { APIGroups: []string{"route.openshift.io"}, Resources: []string{"routes"}, }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, + }, } // Create cluster wide RBAC, required for SDK OpenShift check @@ -96,3 +101,36 @@ func TestMCADRay(t *testing.T) { test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). Should(HaveLen(0)) } + +func readRequirementsTxt(test Test) []byte { + // Read the requirements.txt from resources and perform replacements for custom values using go template + props := struct { + PipIndexUrl string + PipTrustedHost string + }{ + PipIndexUrl: "--index " + GetPipIndexURL(), + } + + // Provide trusted host only if defined + if len(GetPipTrustedHost()) > 0 { + props.PipTrustedHost = "--trusted-host " + GetPipTrustedHost() + } + + template, err := files.ReadFile("resources/requirements.txt") + test.Expect(err).NotTo(HaveOccurred()) + + return ParseTemplate(test, template, props) +} + +func readMnistPy(test Test) []byte { + // Read the mnist.py from resources and perform replacements for custom values using go template + props := struct { + MnistDatasetURL string + }{ + MnistDatasetURL: GetMnistDatasetURL(), + } + template, err := files.ReadFile("resources/mnist.py") + test.Expect(err).NotTo(HaveOccurred()) + + return ParseTemplate(test, template, props) +} diff --git a/test/odh/notebook.go b/test/odh/notebook.go index 8c7b28275..70bdda82c 100644 --- a/test/odh/notebook.go +++ b/test/odh/notebook.go @@ -18,7 +18,6 @@ package odh import ( "bytes" - "html/template" gomega "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" @@ -44,6 +43,7 @@ type NotebookProps struct { OpenDataHubNamespace string ImageStreamName string ImageStreamTag string + RayImage string NotebookConfigMapName string NotebookConfigMapFileName string NotebookPVC string @@ -66,23 +66,19 @@ func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyt OpenDataHubNamespace: GetOpenDataHubNamespace(), ImageStreamName: GetNotebookImageStreamName(test), ImageStreamTag: recommendedTagName, + RayImage: GetRayImage(), NotebookConfigMapName: jupyterNotebookConfigMapName, NotebookConfigMapFileName: jupyterNotebookConfigMapFileName, NotebookPVC: notebookPVC.Name, } notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml") test.Expect(err).NotTo(gomega.HaveOccurred()) - parsedNotebookTemplate, err := template.New("notebook").Parse(string(notebookTemplate)) - test.Expect(err).NotTo(gomega.HaveOccurred()) - // Filter template and store results to the buffer - notebookBuffer := new(bytes.Buffer) - err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps) - test.Expect(err).NotTo(gomega.HaveOccurred()) + parsedNotebookTemplate := ParseTemplate(test, notebookTemplate, notebookProps) // Create Notebook CR notebookCR := &unstructured.Unstructured{} - err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR) + err = yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(parsedNotebookTemplate), 8192).Decode(notebookCR) test.Expect(err).NotTo(gomega.HaveOccurred()) _, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{}) test.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/test/odh/resources/custom-nb-small.yaml b/test/odh/resources/custom-nb-small.yaml index 95aaaf106..791a2d98a 100644 --- a/test/odh/resources/custom-nb-small.yaml +++ b/test/odh/resources/custom-nb-small.yaml @@ -55,7 +55,7 @@ spec: - name: OCP_TOKEN value: {{.KubernetesBearerToken}} image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} - command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} && sleep infinity"] + command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} -p ray_image {{.RayImage}} && sleep infinity"] # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ] imagePullPolicy: Always # livenessProbe: diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py index d6a211944..e88e8fc9a 100644 --- a/test/odh/resources/mnist.py +++ b/test/odh/resources/mnist.py @@ -15,6 +15,7 @@ import os import torch +import requests from pytorch_lightning import LightningModule, Trainer from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn @@ -32,6 +33,8 @@ print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) +MNIST_DATASET_URL = "{{.MnistDatasetURL}}" +print("MNIST_DATASET_URL: is ", MNIST_DATASET_URL) class LitMNIST(LightningModule): def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): @@ -110,8 +113,34 @@ def configure_optimizers(self): #################### def prepare_data(self): - # download - print("Downloading MNIST dataset...") + datasetFiles = [ + "t10k-images-idx3-ubyte", + "t10k-labels-idx1-ubyte", + "train-images-idx3-ubyte", + "train-labels-idx1-ubyte" + ] + + # Create required folder structure + downloadLocation = os.path.join(PATH_DATASETS, "MNIST", "raw") + os.makedirs(downloadLocation, exist_ok=True) + print(f"{downloadLocation} folder_path created!") + + for file in datasetFiles: + print(f"Downloading MNIST dataset {file}... to path : {downloadLocation}") + response = requests.get(f"{MNIST_DATASET_URL}{file}", stream=True) + filePath = os.path.join(downloadLocation, file) + + #to download dataset file + try: + if response.status_code == 200: + open(filePath, 'wb').write(response.content) + print(f"{file}: Downloaded and saved zipped file to path - {filePath}") + else: + print(f"Failed to download file {file}") + except Exception as e: + print(e) + print(f"Downloaded MNIST dataset to... {downloadLocation}") + MNIST(self.data_dir, train=True, download=True) MNIST(self.data_dir, train=False, download=True) diff --git a/test/odh/resources/mnist_ray_mini.ipynb b/test/odh/resources/mnist_ray_mini.ipynb index 38992cc7d..0d8fcc53a 100644 --- a/test/odh/resources/mnist_ray_mini.ipynb +++ b/test/odh/resources/mnist_ray_mini.ipynb @@ -27,7 +27,8 @@ "outputs": [], "source": [ "#parameters\n", - "namespace = \"default\"" + "namespace = \"default\"\n", + "ray_image = \"has to be specified\"" ] }, { @@ -40,7 +41,7 @@ "outputs": [], "source": [ "# Create our cluster and submit appwrapper\n", - "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False))" + "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False, image=ray_image))" ] }, { diff --git a/test/odh/resources/requirements.txt b/test/odh/resources/requirements.txt index 7266b064a..cf9fb5889 100644 --- a/test/odh/resources/requirements.txt +++ b/test/odh/resources/requirements.txt @@ -1,3 +1,5 @@ +{{.PipIndexUrl}} +{{.PipTrustedHost}} pytorch_lightning==1.5.10 ray_lightning torchmetrics==0.9.1 diff --git a/test/odh/template.go b/test/odh/template.go new file mode 100644 index 000000000..3ff4da17f --- /dev/null +++ b/test/odh/template.go @@ -0,0 +1,40 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "bytes" + "html/template" + + "github.com/onsi/gomega" + "github.com/project-codeflare/codeflare-common/support" +) + +func ParseTemplate(t support.Test, inputTemplate []byte, props interface{}) []byte { + t.T().Helper() + + // Parse input template + parsedTemplate, err := template.New("template").Parse(string(inputTemplate)) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + // Filter template and store results to the buffer + buffer := new(bytes.Buffer) + err = parsedTemplate.Execute(buffer, props) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + return buffer.Bytes() +} From 8255d814fc752edcc6f089231f0f467a1748cdbd Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 8 Feb 2024 13:01:53 +0100 Subject: [PATCH 108/369] Update README.md with release instructions --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de350d042..9a62419c0 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,9 @@ For ODH tests additional environment variables are needed: 2. Once all jobs within the action are completed, verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. 3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. 4. Once PR is merged, announce the new release in slack and mail lists, if any. -5. Update the Distributed Workloads component in ODH (also copy/update the compatibility matrix). This may require yaml and test updates depending on the release. Make sure to create a tag + release in the Distributed Workloads repository that matches the project-codeflare release version. -6. Update the readme/markdown/yaml in odh-manifests as required. +5. Release automation should open a PR with changes in [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Review the changes proposed by automation. If all the changes are correct then manually cherrypick all `CARRY` and `PATCH` commits from the current main branch, push the result to a dedicated branch and ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the `main` branch (branch protection has to be temporarily disabled). +6. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. +7. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. ### Releases involving part of the stack From 1558401299be8c8a1b823468e0ff8cec67c80cd8 Mon Sep 17 00:00:00 2001 From: abdhumal Date: Fri, 2 Feb 2024 18:01:14 +0530 Subject: [PATCH 109/369] add an automated CI workflow to sync ODH-notebooks with Codeflare-SDK release --- .github/workflows/odh-notebooks-sync.yml | 116 ++++++++++++++++++ .../workflows/project-codeflare-release.yml | 17 +++ 2 files changed, 133 insertions(+) create mode 100644 .github/workflows/odh-notebooks-sync.yml diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml new file mode 100644 index 000000000..789184c09 --- /dev/null +++ b/.github/workflows/odh-notebooks-sync.yml @@ -0,0 +1,116 @@ +# The aim of this GitHub workflow is to update the pipfile to sync with Codeflare-SDK release. +name: Sync with codeflare-sdk release +on: + workflow_dispatch: + inputs: + upstream-repository-organization: + required: true + description: "Owner of target upstream notebooks repository used to open a PR against" + default: "opendatahub-io" + + codeflare-repository-organization: + required: true + description: "Owner of origin notebooks repository used to open a PR" + default: "project-codeflare" + + codeflare_sdk_release_version: + required: true + description: "Provide version of the Codeflare-SDK release" + +env: + BRANCH_NAME: main + CODEFLARE_RELEASE_VERSION: ${{ github.event.inputs.codeflare_sdk_release_version }} + UPDATER_BRANCH: odh-sync-updater-${{ github.run_id }} + UPSTREAM_OWNER: ${{ github.event.inputs.upstream-repository-organization }} + REPO_OWNER: ${{ github.event.inputs.codeflare-repository-organization }} + REPO_NAME: notebooks + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.BRANCH_NAME }} + + - name: Clone repository and Sync + run: | + git clone https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git $REPO_NAME + cd $REPO_NAME + git remote add upstream https://github.com/$UPSTREAM_OWNER/$REPO_NAME.git + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" + git remote -v + git pull upstream main && git push origin main + + - name: Setup Python environment + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.9 + + - name: Install pipenv + run: pip install pipenv + + - name: Update Pipfiles in accordance with Codeflare-SDK latest release + run: | + # list all Pipfile paths having Codeflare-SDK listed + paths+=($(grep -rl 'codeflare-sdk = "~=.*"')) + # Extracting only directories from file paths, excluding a `.gitworkflow` directory + directories=() + exclude_directories=( + ".git/objects/pack" + ".github/workflows/", + ) + for path in "${paths[@]}"; do + current_dir=$(dirname "$path") + #Check if current_dir is not in exclude_directories list + if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then + #Check if Pipfile exists in current_dir + if [ -f "$current_dir/Pipfile" ];then + directories+=("$current_dir") + fi + fi + done + # Remove duplicates + directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + # Print the directories for verification + echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" + for dir in "${directories[@]}"; do + echo "- $dir" + done + # iterate over the directories and update Pipfile + counter=0 + total=${#directories[@]} + for dir in "${directories[@]}"; do + counter=$((counter+1)) + echo "--Processing directory $counter '$dir' of total $total" + cd "$dir" && pipenv install codeflare-sdk~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - + echo "$((total-counter)) directories remaining.." + done + + - name: Push changes + run: | + cd $REPO_NAME + git add . && git status && git checkout -b ${{ env.UPDATER_BRANCH }} && \ + git commit -am "Updated notebooks via ${{ env.UPDATER_BRANCH }} GitHub action" --signoff && + git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git + git push origin ${{ env.UPDATER_BRANCH }} + + - name: Create Pull Request + run: | + gh pr create --repo $UPSTREAM_OWNER/$REPO_NAME \ + --title "$pr_title" \ + --body "$pr_body" \ + --head $REPO_OWNER:$UPDATER_BRANCH \ + --base $BRANCH_NAME + env: + pr_title: "[Digest Updater Action] Update notebook's pipfile to sync with Codeflare-SDK release" + pr_body: | + :rocket: This is an automated Pull Request. + + This PR updates the `Pipfile` to sync with latest Codeflare-SDK release. + + :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.UPDATER_BRANCH }}` branch after merging the changes diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 77d0a0a1f..7ed8899cf 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -38,6 +38,10 @@ on: description: 'Owner of target community-operators-prod repository used to open a PR against' required: true default: 'redhat-openshift-ecosystem' + upstream-repository-organization: + description: 'Owner of target upstream notebooks repository used to open a PR against' + required: false + default: 'opendatahub-io' jobs: release-parameters: @@ -152,6 +156,19 @@ jobs: shell: bash if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} + - name: Sync ODH Notebooks + run: | + gh workflow run odh-notebooks-sync.yml \ + --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ + --ref ${{ github.ref }} \ + --field upstream-repository-organization=${{ github.event.inputs.upstream-repository-organization }} + --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ + --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + if: ${{ env.SDK_RELEASE_STATUS_CODE == '200' }} + release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] runs-on: ubuntu-latest From aa9c7ff410a516dce595591ede2e50ff62cd4e7a Mon Sep 17 00:00:00 2001 From: abdhumal Date: Fri, 9 Feb 2024 17:40:25 +0530 Subject: [PATCH 110/369] validate if codeflare-sdk version exists --- .github/workflows/odh-notebooks-sync.yml | 86 +++++++++++-------- .../workflows/project-codeflare-release.yml | 6 +- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml index 789184c09..016608f26 100644 --- a/.github/workflows/odh-notebooks-sync.yml +++ b/.github/workflows/odh-notebooks-sync.yml @@ -51,45 +51,61 @@ jobs: 3.8 3.9 - - name: Install pipenv - run: pip install pipenv + - name: Install pipenv and pip-versions + run: pip install pipenv pip-versions - name: Update Pipfiles in accordance with Codeflare-SDK latest release run: | - # list all Pipfile paths having Codeflare-SDK listed - paths+=($(grep -rl 'codeflare-sdk = "~=.*"')) - # Extracting only directories from file paths, excluding a `.gitworkflow` directory - directories=() - exclude_directories=( - ".git/objects/pack" - ".github/workflows/", - ) - for path in "${paths[@]}"; do - current_dir=$(dirname "$path") - #Check if current_dir is not in exclude_directories list - if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then - #Check if Pipfile exists in current_dir - if [ -f "$current_dir/Pipfile" ];then - directories+=("$current_dir") + package_name=codeflare-sdk + # Get the list of available versions for the package + if ! versions=$(pipenv run pip-versions list $package_name);then + echo "Failed to retrieve versions for $package_name" + exit 1 + fi + # Check if the desired version exists in the list + if echo "$versions" | grep -q "${CODEFLARE_RELEASE_VERSION}"; then + echo "Version ${CODEFLARE_RELEASE_VERSION} is available for $package_name" + # list all Pipfile paths having Codeflare-SDK listed + paths+=($(grep -rl "${package_name} = \"~=.*\"")) + # Extracting only directories from file paths, excluding a `.gitworkflow` directory + directories=() + exclude_directories=( + ".git/objects/pack" + ".github/workflows/", + ) + for path in "${paths[@]}"; do + current_dir=$(dirname "$path") + #Check if current_dir is not in exclude_directories list + if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then + #Check if Pipfile exists in current_dir + if [ -f "$current_dir/Pipfile" ];then + directories+=("$current_dir") + fi fi - fi - done - # Remove duplicates - directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) - # Print the directories for verification - echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" - for dir in "${directories[@]}"; do - echo "- $dir" - done - # iterate over the directories and update Pipfile - counter=0 - total=${#directories[@]} - for dir in "${directories[@]}"; do - counter=$((counter+1)) - echo "--Processing directory $counter '$dir' of total $total" - cd "$dir" && pipenv install codeflare-sdk~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - - echo "$((total-counter)) directories remaining.." - done + done + # Remove duplicates + directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + # Print the directories for verification + echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" + for dir in "${directories[@]}"; do + echo "- $dir" + done + # iterate over the directories and update Pipfile + counter=0 + total=${#directories[@]} + for dir in "${directories[@]}"; do + counter=$((counter+1)) + echo "--Processing directory $counter '$dir' of total $total" + cd "$dir" && pipenv install ${package_name}~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - + echo "$((total-counter)) directories remaining.." + done + else + versions_list=$(echo "$versions" | tr '\n' ' ' | sed 's/, $//') + versions="${versions_list%,}" + echo "Version '${CODEFLARE_RELEASE_VERSION}' is not available for $package_name" + echo "Available versions for $package_name: $versions" + exit 1 + fi - name: Push changes run: | diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 7ed8899cf..48a70ef47 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -38,10 +38,6 @@ on: description: 'Owner of target community-operators-prod repository used to open a PR against' required: true default: 'redhat-openshift-ecosystem' - upstream-repository-organization: - description: 'Owner of target upstream notebooks repository used to open a PR against' - required: false - default: 'opendatahub-io' jobs: release-parameters: @@ -161,7 +157,7 @@ jobs: gh workflow run odh-notebooks-sync.yml \ --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ --ref ${{ github.ref }} \ - --field upstream-repository-organization=${{ github.event.inputs.upstream-repository-organization }} + --field upstream-repository-organization=opendatahub-io --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} env: From d0274d0afec620162730af032b0687845e1a0d1a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 14 Feb 2024 15:06:08 +0100 Subject: [PATCH 111/369] Prepare Pytorch MNIST test image for disconnected testing --- .github/workflows/mnist-job-test-image.yml | 33 ++++ Makefile | 12 ++ test/pytorch_mnist_image/Dockerfile | 18 ++ test/pytorch_mnist_image/download_dataset.py | 21 +++ test/pytorch_mnist_image/entrypoint.sh | 3 + test/pytorch_mnist_image/mnist.py | 159 ++++++++++++++++++ .../mnist_pip_requirements.txt | 3 + 7 files changed, 249 insertions(+) create mode 100644 .github/workflows/mnist-job-test-image.yml create mode 100644 test/pytorch_mnist_image/Dockerfile create mode 100644 test/pytorch_mnist_image/download_dataset.py create mode 100755 test/pytorch_mnist_image/entrypoint.sh create mode 100644 test/pytorch_mnist_image/mnist.py create mode 100644 test/pytorch_mnist_image/mnist_pip_requirements.txt diff --git a/.github/workflows/mnist-job-test-image.yml b/.github/workflows/mnist-job-test-image.yml new file mode 100644 index 000000000..768104424 --- /dev/null +++ b/.github/workflows/mnist-job-test-image.yml @@ -0,0 +1,33 @@ +# This workflow will build the MNIST job test image and push it to the project-codeflare image registry + +name: MNIST Job Test Image + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'test/pytorch_mnist_image/**' + +jobs: + push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.20 + + - name: Login to Quay.io + uses: redhat-actions/podman-login@v1 + with: + username: ${{ secrets.QUAY_ID }} + password: ${{ secrets.QUAY_TOKEN }} + registry: quay.io + + - name: Image Build and Push + run: | + make image-mnist-job-test-push diff --git a/Makefile b/Makefile index dccaea286..3b70333b1 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,10 @@ ENVTEST_K8S_VERSION = 1.24.2 # used to build the manifests. ENV ?= default +# Image URL to build MNIST job test image +MNIST_JOB_TEST_VERSION ?= v0.0.2 +MNIST_JOB_TEST_IMG ?= $(IMAGE_ORG_BASE)/mnist-job-test:${MNIST_JOB_TEST_VERSION} + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -383,3 +387,11 @@ imports: openshift-goimports ## Organize imports in go files using openshift-goi .PHONY: verify-imports verify-imports: openshift-goimports ## Run import verifications. ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) + +.PHONY: image-mnist-job-test-build +image-mnist-job-test-build: ## Build container image with the MNIST job. + podman build -t ${MNIST_JOB_TEST_IMG} ./test/pytorch_mnist_image + +.PHONY: image-mnist-job-test-push +image-mnist-job-test-push: image-mnist-job-test-build ## Push container image with the MNIST job. + podman push ${MNIST_JOB_TEST_IMG} diff --git a/test/pytorch_mnist_image/Dockerfile b/test/pytorch_mnist_image/Dockerfile new file mode 100644 index 000000000..adbbd7cbc --- /dev/null +++ b/test/pytorch_mnist_image/Dockerfile @@ -0,0 +1,18 @@ +# Build the manager binary +FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime + +WORKDIR /test +COPY entrypoint.sh entrypoint.sh + +# Install MNIST requirements +COPY mnist_pip_requirements.txt requirements.txt +RUN pip install --requirement requirements.txt + +# Prepare MNIST script +COPY mnist.py mnist.py +COPY download_dataset.py download_dataset.py +RUN torchrun download_dataset.py + +USER 65532:65532 +WORKDIR /workdir +ENTRYPOINT ["/test/entrypoint.sh"] diff --git a/test/pytorch_mnist_image/download_dataset.py b/test/pytorch_mnist_image/download_dataset.py new file mode 100644 index 000000000..698f68f67 --- /dev/null +++ b/test/pytorch_mnist_image/download_dataset.py @@ -0,0 +1,21 @@ +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from torchvision.datasets import MNIST + +PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") +MNIST(PATH_DATASETS, train=True, download=True) +MNIST(PATH_DATASETS, train=False, download=True) diff --git a/test/pytorch_mnist_image/entrypoint.sh b/test/pytorch_mnist_image/entrypoint.sh new file mode 100755 index 000000000..7487b377a --- /dev/null +++ b/test/pytorch_mnist_image/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +torchrun /test/mnist.py diff --git a/test/pytorch_mnist_image/mnist.py b/test/pytorch_mnist_image/mnist.py new file mode 100644 index 000000000..134d56188 --- /dev/null +++ b/test/pytorch_mnist_image/mnist.py @@ -0,0 +1,159 @@ +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +import requests +from pytorch_lightning import LightningModule, Trainer +from pytorch_lightning.callbacks.progress import TQDMProgressBar +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader, random_split +from torchmetrics import Accuracy +from torchvision import transforms +from torchvision.datasets import MNIST + +PATH_WORKDIR = os.environ.get("PATH_WORKDIR", ".") +PATH_DATASETS = os.environ.get("PATH_DATASETS", "/test") +BATCH_SIZE = 256 if torch.cuda.is_available() else 64 +# %% + +print("prior to running the trainer") +print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) +print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) + +class LitMNIST(LightningModule): + def __init__(self, data_dir=PATH_WORKDIR, hidden_size=64, learning_rate=2e-4): + + super().__init__() + + # Set our init args as class attributes + self.data_dir = data_dir + self.hidden_size = hidden_size + self.learning_rate = learning_rate + + # Hardcode some dataset specific attributes + self.num_classes = 10 + self.dims = (1, 28, 28) + channels, width, height = self.dims + self.transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + ] + ) + + # Define PyTorch model + self.model = nn.Sequential( + nn.Flatten(), + nn.Linear(channels * width * height, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, self.num_classes), + ) + + self.val_accuracy = Accuracy() + self.test_accuracy = Accuracy() + + def forward(self, x): + x = self.model(x) + return F.log_softmax(x, dim=1) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.val_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("val_loss", loss, prog_bar=True) + self.log("val_acc", self.val_accuracy, prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.test_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("test_loss", loss, prog_bar=True) + self.log("test_acc", self.test_accuracy, prog_bar=True) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + #################### + # DATA RELATED HOOKS + #################### + + def prepare_data(self): + MNIST(PATH_DATASETS, train=True, download=True) + MNIST(PATH_DATASETS, train=False, download=True) + + def setup(self, stage=None): + + # Assign train/val datasets for use in dataloaders + if stage == "fit" or stage is None: + mnist_full = MNIST(PATH_DATASETS, train=True, transform=self.transform) + self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) + + # Assign test dataset for use in dataloader(s) + if stage == "test" or stage is None: + self.mnist_test = MNIST( + PATH_DATASETS, train=False, transform=self.transform + ) + + def train_dataloader(self): + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + + def val_dataloader(self): + return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) + + def test_dataloader(self): + return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) + + +# Init DataLoader from MNIST Dataset + +model = LitMNIST() + +print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) +print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) + +# Initialize a trainer +trainer = Trainer( + accelerator="auto", + # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs + max_epochs=5, + callbacks=[TQDMProgressBar(refresh_rate=20)], + num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), + devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + strategy="ddp", +) + +# Train the model ⚡ +trainer.fit(model) diff --git a/test/pytorch_mnist_image/mnist_pip_requirements.txt b/test/pytorch_mnist_image/mnist_pip_requirements.txt new file mode 100644 index 000000000..87edeef27 --- /dev/null +++ b/test/pytorch_mnist_image/mnist_pip_requirements.txt @@ -0,0 +1,3 @@ +pytorch_lightning==1.5.10 +torchmetrics==0.9.1 +torchvision==0.12.0 From 705ee164fb290eec9a165817ac8259fcc4b44ceb Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 15 Feb 2024 14:52:58 +0100 Subject: [PATCH 112/369] Update deprecated GitHub actions --- .github/workflows/e2e_tests.yaml | 8 ++++---- .github/workflows/mnist-job-test-image.yml | 15 ++++++++------ .github/workflows/olm_tests.yaml | 8 ++++---- .github/workflows/operator-image.yml | 17 +++++++++------- .github/workflows/precommit.yml | 6 +++--- .github/workflows/tag-and-build.yml | 21 +++++++++++--------- .github/workflows/unit_tests.yml | 6 +++--- .github/workflows/verify_generated_files.yml | 8 ++++---- 8 files changed, 49 insertions(+), 40 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0786c290c..0fc1ec88b 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -31,19 +31,19 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Checkout common repo code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'project-codeflare/codeflare-common' ref: 'main' path: 'common' - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 @@ -99,7 +99,7 @@ jobs: output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() && steps.deploy.outcome == 'success' with: name: logs diff --git a/.github/workflows/mnist-job-test-image.yml b/.github/workflows/mnist-job-test-image.yml index 768104424..4323bd5dd 100644 --- a/.github/workflows/mnist-job-test-image.yml +++ b/.github/workflows/mnist-job-test-image.yml @@ -17,17 +17,20 @@ jobs: - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Login to Quay.io - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.QUAY_ID }} - password: ${{ secrets.QUAY_TOKEN }} - registry: quay.io + id: podman-login-quay + run: | + podman login --username ${{ secrets.QUAY_ID }} --password ${{ secrets.QUAY_TOKEN }} quay.io - name: Image Build and Push run: | make image-mnist-job-test-push + + - name: Logout from Quay.io + if: always() && steps.podman-login-quay.outcome == 'success' + run: | + podman logout quay.io diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 6889604bb..aa8a3b172 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -30,19 +30,19 @@ jobs: CODEFLARE_TEST_TIMEOUT_LONG: "10m" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # fetching also previous commits to get tags - name: Checkout common repo code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'project-codeflare/codeflare-common' ref: 'main' path: 'common' - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 @@ -192,7 +192,7 @@ jobs: output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() && steps.deploy.outcome == 'success' with: name: logs diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index 0c155be1d..8c30a8c17 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -17,22 +17,25 @@ jobs: push: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Login to Quay.io - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.QUAY_ID }} - password: ${{ secrets.QUAY_TOKEN }} - registry: quay.io + id: podman-login-quay + run: | + podman login --username ${{ secrets.QUAY_ID }} --password ${{ secrets.QUAY_TOKEN }} quay.io - name: Image Build run: | make build make image-build -e IMG=quay.io/project-codeflare/codeflare-operator:dev make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:dev + + - name: Logout from Quay.io + if: always() && steps.podman-login-quay.outcome == 'success' + run: | + podman logout quay.io diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 13c4c7fca..2ca5869c6 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -21,15 +21,15 @@ jobs: volumes: - /cache steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Activate cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index f04d43d67..9dc290c48 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -59,10 +59,10 @@ jobs: PR_BRANCH_NAME: adjustments-release-${{ github.event.inputs.version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 @@ -79,7 +79,7 @@ jobs: GITHUB_TOKEN: ${{ github.TOKEN }} - name: Activate cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} @@ -108,11 +108,9 @@ jobs: shell: bash - name: Login to Quay.io - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.QUAY_ID }} - password: ${{ secrets.QUAY_TOKEN }} - registry: quay.io + id: podman-login-quay + run: | + podman login --username ${{ secrets.QUAY_ID }} --password ${{ secrets.QUAY_TOKEN }} quay.io - name: Align go.mod and go.sum dependencies for released components run: | @@ -140,7 +138,7 @@ jobs: - name: Commit changes in the code back to repository id: create-pr-branch - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} file_pattern: 'README.md *.yaml Makefile go.mod go.sum *.env' @@ -199,3 +197,8 @@ jobs: GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} + + - name: Logout from Quay.io + if: always() && steps.podman-login-quay.outcome == 'success' + run: | + podman logout quay.io diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 39ab0ef0f..747cee03d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,15 +22,15 @@ jobs: volumes: - /cache steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Activate cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index e2fa269cc..9027c1aad 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -20,9 +20,9 @@ jobs: verify-imports: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Verify that imports are organized @@ -31,9 +31,9 @@ jobs: verify-manifests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated From ac483fdcc2f8f0d3f3b2b4f2d57b507ce583cad8 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 19 Feb 2024 15:33:12 +0100 Subject: [PATCH 113/369] e2e tests: Use test image from quay.io/project-codeflare/demo-images --- test/odh/resources/mnist_mcad_mini.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb index 0b53324ab..b32a95ab9 100644 --- a/test/odh/resources/mnist_mcad_mini.ipynb +++ b/test/odh/resources/mnist_mcad_mini.ipynb @@ -19,7 +19,9 @@ "execution_count": null, "id": "47ca5c15", "metadata": { - "tags": ["parameters"] + "tags": [ + "parameters" + ] }, "outputs": [], "source": [ @@ -34,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "job = DDPJobDefinition(name=\"mnistjob\", script=\"mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/mnist-job-test:v0.0.1\").submit()" + "job = DDPJobDefinition(name=\"mnistjob\", script=\"/test/mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/demo-images:pytorch-mnist-v0.0.1\").submit()" ] }, { From 10bf0dd05f803c59cb565b24f46033697569d2c8 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 20 Feb 2024 15:54:59 +0000 Subject: [PATCH 114/369] Fix MCAD release --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 48a70ef47..b7abf8e8c 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -70,7 +70,7 @@ jobs: - name: Release MCAD run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 2e9384965b268416fa6ff9880bbe479d42a6718c Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 20 Feb 2024 17:08:16 +0100 Subject: [PATCH 115/369] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a62419c0..aadb28f81 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ For ODH tests additional environment variables are needed: 5. Release automation should open a PR with changes in [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Review the changes proposed by automation. If all the changes are correct then manually cherrypick all `CARRY` and `PATCH` commits from the current main branch, push the result to a dedicated branch and ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the `main` branch (branch protection has to be temporarily disabled). 6. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. 7. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. +8. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. ### Releases involving part of the stack From dfd6c619452f03fa2f9c1011f7b9840b3e4b499c Mon Sep 17 00:00:00 2001 From: dimakis Date: Tue, 20 Feb 2024 23:06:28 +0000 Subject: [PATCH 116/369] Update dependency versions for release v1.2.0 --- Makefile | 2 +- README.md | 6 +++--- config/crd/mcad/kustomization.yaml | 2 +- config/manager/params.env | 2 +- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 3b70333b1..e1419e1a1 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ INSTASCALE_VERSION ?= v0.4.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.39.0 +MCAD_VERSION ?= v1.40.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index aadb28f81..73bb55490 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.1.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.1.0) | -| Multi-Cluster App Dispatcher | [v1.39.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.39.0) | -| CodeFlare-SDK | [v0.13.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.13.0) | +| CodeFlare Operator | [v1.2.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.2.0) | +| Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | +| CodeFlare-SDK | [v0.14.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.14.0) | | InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | | KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 0e4ebfa7a..bbf6141cf 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.39.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.40.0 diff --git a/config/manager/params.env b/config/manager/params.env index ace2f2f29..c4c8b45f3 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.1.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.2.0 diff --git a/go.mod b/go.mod index 252653db6..7f846b2b4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 github.com/project-codeflare/instascale v0.4.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 diff --git a/go.sum b/go.sum index b36099d6c..60345cd59 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 h1:IkTmd/W/zxcsC5s4EbnW74PFpkQVEiTc/8rWWwFw0Ok= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 49b2f9da99305a9da7f5b12933509befe492156b Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Wed, 21 Feb 2024 15:13:59 +0530 Subject: [PATCH 117/369] fixed incorrect logic in Codeflare-SDK release sync with notebooks --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index b7abf8e8c..d5e0fe06f 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -163,7 +163,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash - if: ${{ env.SDK_RELEASE_STATUS_CODE == '200' }} + if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] From 6b5acfcd315ba71813ebe98ef6c996a5501092b8 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 20 Feb 2024 16:32:12 +0000 Subject: [PATCH 118/369] Display input for KubeRay parameter and check for existence --- .github/workflows/project-codeflare-release.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index d5e0fe06f..6e2d75742 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -51,12 +51,14 @@ jobs: echo "MCAD Version: ${{ github.event.inputs.mcad-version }}" echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}" echo "InstaScale Version: ${{ github.event.inputs.instascale-version }}" + echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}" echo "Is Stable: ${{ github.event.inputs.is-stable }}" echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" echo "Quay Organization: ${{ github.event.inputs.quay-organization }}" echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" release-mcad: + needs: check-kuberay-version runs-on: ubuntu-latest steps: @@ -120,6 +122,7 @@ jobs: if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} release-codeflare-sdk: + needs: check-kuberay-version runs-on: ubuntu-latest steps: @@ -186,3 +189,17 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + + check-kuberay-version: + runs-on: ubuntu-latest + + steps: + - name: Check if KubeRay release exists + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/ray-project/kuberay/releases/tag/${{ github.event.inputs.kuberay-version }}) + if [[ "$status_code" == "200" ]]; then + echo "KubeRay release with version ${{ github.event.inputs.kuberay-version }} exists and available to use." + else + echo "KubeRay release with version ${{ github.event.inputs.kuberay-version }} does not exist. Please select an existing version." + exit 1 + fi From fa8863602903a319605529a87b8561082eda502f Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Fri, 9 Feb 2024 16:25:16 +0000 Subject: [PATCH 119/369] add: update machinepool tests to identify machines based on label --- test/e2e/instascale_machinepool_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 69fa037d3..64e53f8a8 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "fmt" "testing" . "github.com/onsi/gomega" @@ -49,14 +50,15 @@ func TestInstascaleMachinePool(t *testing.T) { connection := CreateOCMConnection(test) defer connection.Close() - // check existing cluster machine pool resources - // look for machine pool with aw name - expect not to find it - test.Expect(GetMachinePools(test, connection)). - ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) - // Setup batch job and AppWrapper aw := instaScaleJobAppWrapper(test, namespace, cm) + expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) + // check existing cluster machine pool resources + // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it + test.Expect(GetMachinePools(test, connection)). + ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) + // apply AppWrapper to cluster _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) @@ -66,15 +68,15 @@ func TestInstascaleMachinePool(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // look for machine pool with aw name - expect to find it + // look for a machine pool with a label key equal to aw.name-aw.namespace - expect to find it test.Eventually(MachinePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) + Should(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for machine pool with aw name - expect not to find it + // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it test.Eventually(MachinePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) + ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) } From 8aeaefc831335c978a4109d5cdf6654664855063 Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Tue, 27 Feb 2024 16:35:46 +0530 Subject: [PATCH 120/369] remove Sync ODH-notebooks CI workflow to add this step in Codeflare-SDK repository --- .github/workflows/odh-notebooks-sync.yml | 132 ------------------ .../workflows/project-codeflare-release.yml | 13 -- 2 files changed, 145 deletions(-) delete mode 100644 .github/workflows/odh-notebooks-sync.yml diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml deleted file mode 100644 index 016608f26..000000000 --- a/.github/workflows/odh-notebooks-sync.yml +++ /dev/null @@ -1,132 +0,0 @@ -# The aim of this GitHub workflow is to update the pipfile to sync with Codeflare-SDK release. -name: Sync with codeflare-sdk release -on: - workflow_dispatch: - inputs: - upstream-repository-organization: - required: true - description: "Owner of target upstream notebooks repository used to open a PR against" - default: "opendatahub-io" - - codeflare-repository-organization: - required: true - description: "Owner of origin notebooks repository used to open a PR" - default: "project-codeflare" - - codeflare_sdk_release_version: - required: true - description: "Provide version of the Codeflare-SDK release" - -env: - BRANCH_NAME: main - CODEFLARE_RELEASE_VERSION: ${{ github.event.inputs.codeflare_sdk_release_version }} - UPDATER_BRANCH: odh-sync-updater-${{ github.run_id }} - UPSTREAM_OWNER: ${{ github.event.inputs.upstream-repository-organization }} - REPO_OWNER: ${{ github.event.inputs.codeflare-repository-organization }} - REPO_NAME: notebooks - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ env.BRANCH_NAME }} - - - name: Clone repository and Sync - run: | - git clone https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git $REPO_NAME - cd $REPO_NAME - git remote add upstream https://github.com/$UPSTREAM_OWNER/$REPO_NAME.git - git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" - git config --global user.name "codeflare-machine-account" - git remote -v - git pull upstream main && git push origin main - - - name: Setup Python environment - uses: actions/setup-python@v4 - with: - python-version: | - 3.8 - 3.9 - - - name: Install pipenv and pip-versions - run: pip install pipenv pip-versions - - - name: Update Pipfiles in accordance with Codeflare-SDK latest release - run: | - package_name=codeflare-sdk - # Get the list of available versions for the package - if ! versions=$(pipenv run pip-versions list $package_name);then - echo "Failed to retrieve versions for $package_name" - exit 1 - fi - # Check if the desired version exists in the list - if echo "$versions" | grep -q "${CODEFLARE_RELEASE_VERSION}"; then - echo "Version ${CODEFLARE_RELEASE_VERSION} is available for $package_name" - # list all Pipfile paths having Codeflare-SDK listed - paths+=($(grep -rl "${package_name} = \"~=.*\"")) - # Extracting only directories from file paths, excluding a `.gitworkflow` directory - directories=() - exclude_directories=( - ".git/objects/pack" - ".github/workflows/", - ) - for path in "${paths[@]}"; do - current_dir=$(dirname "$path") - #Check if current_dir is not in exclude_directories list - if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then - #Check if Pipfile exists in current_dir - if [ -f "$current_dir/Pipfile" ];then - directories+=("$current_dir") - fi - fi - done - # Remove duplicates - directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) - # Print the directories for verification - echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" - for dir in "${directories[@]}"; do - echo "- $dir" - done - # iterate over the directories and update Pipfile - counter=0 - total=${#directories[@]} - for dir in "${directories[@]}"; do - counter=$((counter+1)) - echo "--Processing directory $counter '$dir' of total $total" - cd "$dir" && pipenv install ${package_name}~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - - echo "$((total-counter)) directories remaining.." - done - else - versions_list=$(echo "$versions" | tr '\n' ' ' | sed 's/, $//') - versions="${versions_list%,}" - echo "Version '${CODEFLARE_RELEASE_VERSION}' is not available for $package_name" - echo "Available versions for $package_name: $versions" - exit 1 - fi - - - name: Push changes - run: | - cd $REPO_NAME - git add . && git status && git checkout -b ${{ env.UPDATER_BRANCH }} && \ - git commit -am "Updated notebooks via ${{ env.UPDATER_BRANCH }} GitHub action" --signoff && - git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git - git push origin ${{ env.UPDATER_BRANCH }} - - - name: Create Pull Request - run: | - gh pr create --repo $UPSTREAM_OWNER/$REPO_NAME \ - --title "$pr_title" \ - --body "$pr_body" \ - --head $REPO_OWNER:$UPDATER_BRANCH \ - --base $BRANCH_NAME - env: - pr_title: "[Digest Updater Action] Update notebook's pipfile to sync with Codeflare-SDK release" - pr_body: | - :rocket: This is an automated Pull Request. - - This PR updates the `Pipfile` to sync with latest Codeflare-SDK release. - - :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.UPDATER_BRANCH }}` branch after merging the changes diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 6e2d75742..50fc26ae1 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -155,19 +155,6 @@ jobs: shell: bash if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} - - name: Sync ODH Notebooks - run: | - gh workflow run odh-notebooks-sync.yml \ - --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ - --ref ${{ github.ref }} \ - --field upstream-repository-organization=opendatahub-io - --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ - --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} - release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] runs-on: ubuntu-latest From 7a9b5bb40ef79aa0788d0279d935895e57ab7c0a Mon Sep 17 00:00:00 2001 From: Jiri Petrlik Date: Tue, 27 Feb 2024 16:24:39 +0100 Subject: [PATCH 121/369] RHOAIENG-3771 - Reduce execution time of E2E tests By reducing number of epochs and number of training samples in each epoch it was possible to reduce test execution time from more than 10 minutes to less than 2 minutes. --- test/e2e/mnist.py | 7 ++++--- test/odh/resources/mnist.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index 4cfe0b43b..5ac266652 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -20,7 +20,7 @@ from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split +from torch.utils.data import DataLoader, random_split, RandomSampler from torchmetrics import Accuracy from torchvision import transforms from torchvision.datasets import MNIST @@ -158,7 +158,7 @@ def setup(self, stage=None): ) def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE, sampler=RandomSampler(self.mnist_train, num_samples=1000)) def val_dataloader(self): return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) @@ -178,10 +178,11 @@ def test_dataloader(self): trainer = Trainer( accelerator="auto", # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=5, + max_epochs=3, callbacks=[TQDMProgressBar(refresh_rate=20)], num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + replace_sampler_ddp=False, strategy="ddp", ) diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py index e88e8fc9a..85d420f48 100644 --- a/test/odh/resources/mnist.py +++ b/test/odh/resources/mnist.py @@ -20,7 +20,7 @@ from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split +from torch.utils.data import DataLoader, random_split, RandomSampler from torchmetrics import Accuracy from torchvision import transforms from torchvision.datasets import MNIST @@ -158,7 +158,7 @@ def setup(self, stage=None): ) def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE, sampler=RandomSampler(self.mnist_train, num_samples=1000)) def val_dataloader(self): return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) @@ -178,10 +178,11 @@ def test_dataloader(self): trainer = Trainer( accelerator="auto", # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=2, + max_epochs=3, callbacks=[TQDMProgressBar(refresh_rate=20)], num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + replace_sampler_ddp=False, strategy="ddp", ) From bb2460797387b02d918d79c13360801b7de8434d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 28 Feb 2024 14:14:48 +0100 Subject: [PATCH 122/369] Update README.md with ODH/CFO build instructions --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 73bb55490..4f9cd0248 100644 --- a/README.md +++ b/README.md @@ -108,9 +108,10 @@ For ODH tests additional environment variables are needed: 3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. 4. Once PR is merged, announce the new release in slack and mail lists, if any. 5. Release automation should open a PR with changes in [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Review the changes proposed by automation. If all the changes are correct then manually cherrypick all `CARRY` and `PATCH` commits from the current main branch, push the result to a dedicated branch and ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the `main` branch (branch protection has to be temporarily disabled). -6. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. -7. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. -8. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. +6. Build ODH/CFO image by triggering [Build and Push action](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/build-and-push.yaml) +7. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. +8. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. +9. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. ### Releases involving part of the stack From 7419586f1a169fe956f3dd476f88ca3a4a807138 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 8 Mar 2024 17:17:58 +0100 Subject: [PATCH 123/369] e2e tests: Fix finishing condition in ODH e2e test --- test/odh/resources/mnist_mcad_mini.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb index b32a95ab9..341eb12ce 100644 --- a/test/odh/resources/mnist_mcad_mini.ipynb +++ b/test/odh/resources/mnist_mcad_mini.ipynb @@ -50,7 +50,7 @@ "while not finished:\n", " sleep(1)\n", " try:\n", - " finished = (\"Epoch 4: 100%\" in job.logs())\n", + " finished = (\"Epoch 2: 100%\" in job.logs())\n", " except:\n", " finished = False" ] From 2b5d9b838ad2141f15e97ccfd8b705cf82cf395c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 25 Mar 2024 10:27:48 +0000 Subject: [PATCH 124/369] Fix e2e test - invalid go version error --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e1419e1a1..d2cd22d0e 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,7 @@ $(YQ): $(LOCALBIN) .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) - test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.17 .PHONY: openshift-goimports openshift-goimports: $(OPENSHIFT-GOIMPORTS) ## Download openshift-goimports locally if necessary. From 91f8bf3aeb8fb2a2836a8c51cbbeb9f5e8d300ad Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 25 Jan 2024 13:48:14 -0500 Subject: [PATCH 125/369] add raycluster controller to CFO Signed-off-by: Kevin --- Dockerfile | 1 + PROJECT | 10 + config/rbac/kustomization.yaml | 2 + config/rbac/mcad_manager_role.yaml | 223 ++++++++++++++ config/rbac/mcad_manager_role_binding.yaml | 12 + config/rbac/role.yaml | 179 +---------- controllers/raycluster_controller.go | 338 +++++++++++++++++++++ controllers/raycluster_controller_test.go | 222 ++++++++++++++ controllers/suite_test.go | 135 ++++++++ go.mod | 8 +- go.sum | 10 +- main.go | 33 ++ 12 files changed, 1001 insertions(+), 172 deletions(-) create mode 100644 config/rbac/mcad_manager_role.yaml create mode 100644 config/rbac/mcad_manager_role_binding.yaml create mode 100644 controllers/raycluster_controller.go create mode 100644 controllers/raycluster_controller_test.go create mode 100644 controllers/suite_test.go diff --git a/Dockerfile b/Dockerfile index f5c51c3f5..26f4c23db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN go mod download # Copy the Go sources COPY main.go main.go COPY pkg/ pkg/ +COPY controllers/ controllers/ # Build USER root diff --git a/PROJECT b/PROJECT index 57b3ef28a..cb6903086 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: codeflare.dev layout: - go.kubebuilder.io/v3 @@ -6,4 +10,10 @@ plugins: scorecard.sdk.operatorframework.io/v2: {} projectName: codeflare-operator repo: github.com/project-codeflare/codeflare-operator +resources: +- controller: true + domain: codeflare.dev + group: ray + kind: RayCluster + version: v1 version: "3" diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index a74a16f1d..12d4bb247 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,6 +7,8 @@ resources: - admin_role.yaml - editor_role.yaml - service_account.yaml +- mcad_manager_role.yaml +- mcad_manager_role_binding.yaml - role.yaml - role_binding.yaml - instascale_role.yaml diff --git a/config/rbac/mcad_manager_role.yaml b/config/rbac/mcad_manager_role.yaml new file mode 100644 index 000000000..b414b8e23 --- /dev/null +++ b/config/rbac/mcad_manager_role.yaml @@ -0,0 +1,223 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manual-manager-role +rules: +- apiGroups: + - '*' + resources: + - deployments + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - config.openshift.io + resources: + - clusterversions + verbs: + - get + - list +- apiGroups: + - coordination.k8s.io + resources: + - kube-scheduler + - leases + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - bindings + - pods/binding + verbs: + - create +- apiGroups: + - "" + resources: + - configmaps + - nodes + - persistentvolumeclaims + - persistentvolumes + - secrets + - serviceaccounts + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + - kube-scheduler + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - kube-scheduler + verbs: + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - replicationcontrollers + verbs: + - get + - list + - watch +- apiGroups: + - events.k8s.io + resources: + - events + - kube-scheduler + verbs: + - create + - patch + - update +- apiGroups: + - machine.openshift.io + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - scheduling.sigs.k8s.io + resources: + - podgroups + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - csidrivers + - csinodes + - csistoragecapacities + verbs: + - get + - list + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + - appwrappers/finalizers + - appwrappers/status + - schedulingspecs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - quota.codeflare.dev + resources: + - quotasubtrees + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/config/rbac/mcad_manager_role_binding.yaml b/config/rbac/mcad_manager_role_binding.yaml new file mode 100644 index 000000000..af6c74ae0 --- /dev/null +++ b/config/rbac/mcad_manager_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manual-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manual-manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 07954b370..c7ef78b42 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,219 +5,62 @@ metadata: creationTimestamp: null name: manager-role rules: -- apiGroups: - - '*' - resources: - - deployments - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - replicasets - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list -- apiGroups: - - coordination.k8s.io - resources: - - kube-scheduler - - leases - verbs: - - create - - get - - update - apiGroups: - "" resources: - - bindings - - pods/binding - verbs: - - create -- apiGroups: - - "" - resources: - - configmaps - - nodes - - persistentvolumeclaims - - persistentvolumes - secrets - - serviceaccounts - - services verbs: - create - delete - get - - list - patch - - update - - watch - apiGroups: - "" resources: - - endpoints - - kube-scheduler - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update -- apiGroups: - - "" - resources: - - kube-scheduler - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods + - serviceaccounts verbs: - - create - delete - - deletecollection - get - - list - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/status - verbs: - - patch - - update - apiGroups: - "" resources: - - replicationcontrollers - verbs: - - get - - list - - watch -- apiGroups: - - events.k8s.io - resources: - - events - - kube-scheduler - verbs: - - create - - patch - - update -- apiGroups: - - machine.openshift.io - resources: - - '*' + - services verbs: - - create - delete - get - - list - patch - - update - - watch - apiGroups: - - scheduling.sigs.k8s.io + - ray.io resources: - - podgroups + - rayclusters verbs: - create - delete - - deletecollection - get - list - patch - update - watch - apiGroups: - - storage.k8s.io + - ray.io resources: - - csidrivers - - csinodes - - csistoragecapacities + - rayclusters/finalizers verbs: - - get - - list - - watch + - update - apiGroups: - - workload.codeflare.dev + - ray.io resources: - - appwrappers - - appwrappers/finalizers - - appwrappers/status - - schedulingspecs + - rayclusters/status verbs: - - create - - delete - - deletecollection - get - - list - patch - update - - watch - apiGroups: - - quota.codeflare.dev + - route.openshift.io resources: - - quotasubtrees + - routes verbs: - - create - delete - - deletecollection - get - - list - patch - - update - - watch diff --git a/controllers/raycluster_controller.go b/controllers/raycluster_controller.go new file mode 100644 index 000000000..00d7d559d --- /dev/null +++ b/controllers/raycluster_controller.go @@ -0,0 +1,338 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "strconv" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + coreapply "k8s.io/client-go/applyconfigurations/core/v1" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" + rbacapply "k8s.io/client-go/applyconfigurations/rbac/v1" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + routev1 "github.com/openshift/api/route/v1" + routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" + routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" +) + +// RayClusterReconciler reconciles a RayCluster object +type RayClusterReconciler struct { + client.Client + kubeClient *kubernetes.Clientset + routeClient *routev1client.RouteV1Client + Scheme *runtime.Scheme + CookieSalt string +} + +const ( + requeueTime = 10 + controllerName = "codeflare-raycluster-controller" + oauthAnnotation = "codeflare.dev/oauth=true" + CodeflareOAuthFinalizer = "codeflare.dev/oauth-finalizer" + OAuthServicePort = 443 + OAuthServicePortName = "oauth-proxy" + strTrue = "true" + strFalse = "false" + logRequeueing = "requeueing" +) + +var ( + deletePolicy = metav1.DeletePropagationForeground + deleteOptions = client.DeleteOptions{PropagationPolicy: &deletePolicy} +) + +//+kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update +//+kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get +//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get +//+kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get +//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the RayCluster object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.15.3/pkg/reconcile + +func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := ctrl.LoggerFrom(ctx) + + var cluster rayv1.RayCluster + + if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { + if !errors.IsNotFound(err) { + logger.Error(err, "Error getting RayCluster resource") + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if cluster.ObjectMeta.DeletionTimestamp.IsZero() { + if !controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { + logger.Info("Add a finalizer", "finalizer", CodeflareOAuthFinalizer) + controllerutil.AddFinalizer(&cluster, CodeflareOAuthFinalizer) + if err := r.Update(ctx, &cluster); err != nil { + // this log is info level since errors are not fatal and are expected + logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } + } else if controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { + err := r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, + ) + if err != nil { + logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + controllerutil.RemoveFinalizer(&cluster, CodeflareOAuthFinalizer) + if err := r.Update(ctx, &cluster); err != nil { + logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + logger.Info("Successfully removed finalizer.", logRequeueing, strFalse) + return ctrl.Result{}, nil + } + + val, ok := cluster.ObjectMeta.Annotations["codeflare.dev/oauth"] + boolVal, err := strconv.ParseBool(val) + if err != nil { + logger.Error(err, "Could not convert codeflare.dev/oauth value to bool", "codeflare.dev/oauth", val) + } + if !ok || err != nil || !boolVal { + logger.Info("Removing all OAuth Objects") + err := r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthSecretNameFromCluster(&cluster)}, &corev1.Secret{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth Secret, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceNameFromCluster(&cluster)}, &corev1.Service{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth Service, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceAccountNameFromCluster(&cluster)}, &corev1.ServiceAccount{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth ServiceAccount, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth CRB, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: routeNameFromCluster(&cluster)}, &routev1.Route{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth Route, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + return ctrl.Result{}, nil + } + + _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Route") + } + + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to create OAuth Secret") + } + + _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Service") + } + + _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ServiceAccount") + } + + _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ClusterRoleBinding") + } + + return ctrl.Result{}, nil +} + +func crbNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} +} + +func (r *RayClusterReconciler) deleteIfNotExist(ctx context.Context, namespacedName types.NamespacedName, obj client.Object) error { + err := r.Client.Get(ctx, namespacedName, obj) + if err != nil && errors.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + return r.Client.Delete(ctx, obj, &deleteOptions) +} + +func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.ClusterRoleBindingApplyConfiguration { + return rbacapply.ClusterRoleBinding( + crbNameFromCluster(cluster)). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSubjects( + rbacapply.Subject(). + WithKind("ServiceAccount"). + WithName(oauthServiceAccountNameFromCluster(cluster)). + WithNamespace(cluster.Namespace), + ). + WithRoleRef( + rbacapply.RoleRef(). + WithAPIGroup("rbac.authorization.k8s.io"). + WithKind("ClusterRole"). + WithName("system:auth-delegator"), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-oauth-proxy" +} + +func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountApplyConfiguration { + return coreapply.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithAnnotations(map[string]string{ + "serviceaccounts.openshift.io/oauth-redirectreference.first": "" + + `{"kind":"OAuthRedirectReference","apiVersion":"v1",` + + `"reference":{"kind":"Route","name":"` + routeNameFromCluster(cluster) + `"}}`, + }). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func routeNameFromCluster(cluster *rayv1.RayCluster) string { + return "ray-dashboard-" + cluster.Name +} + +func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { + return routeapply.Route(routeNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSpec(routeapply.RouteSpec(). + WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). + WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((OAuthServicePortName)))). + WithTLS(routeapply.TLSConfig(). + WithInsecureEdgeTerminationPolicy(routev1.InsecureEdgeTerminationPolicyRedirect). + WithTermination(routev1.TLSTerminationReencrypt), + ), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-oauth" +} + +func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string { + return cluster.Name + "-proxy-tls-secret" +} + +func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfiguration { + return coreapply.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithAnnotations(map[string]string{"service.beta.openshift.io/serving-cert-secret-name": oauthServiceTLSSecretName(cluster)}). + WithSpec( + coreapply.ServiceSpec(). + WithPorts( + coreapply.ServicePort(). + WithName(OAuthServicePortName). + WithPort(OAuthServicePort). + WithTargetPort(intstr.FromString(OAuthServicePortName)). + WithProtocol(corev1.ProtocolTCP), + ). + WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-oauth-config" +} + +// desiredOAuthSecret defines the desired OAuth secret object +func desiredOAuthSecret(cluster *rayv1.RayCluster, r *RayClusterReconciler) *coreapply.SecretApplyConfiguration { + // Generate the cookie secret for the OAuth proxy + hasher := sha1.New() // REVIEW is SHA1 okay here? + hasher.Write([]byte(cluster.Name + r.CookieSalt)) + cookieSecret := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + + return coreapply.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithStringData(map[string]string{"cookie_secret": cookieSecret}). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) + // Create a Kubernetes secret to store the cookie secret +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.kubeClient = kubernetes.NewForConfigOrDie(mgr.GetConfig()) + r.routeClient = routev1client.NewForConfigOrDie(mgr.GetConfig()) + b := make([]byte, 16) + _, err := rand.Read(b) + if err != nil { + return err + } + r.CookieSalt = string(b) + return ctrl.NewControllerManagedBy(mgr). + Named(controllerName). + For(&rayv1.RayCluster{}). + Complete(r) +} diff --git a/controllers/raycluster_controller_test.go b/controllers/raycluster_controller_test.go new file mode 100644 index 000000000..0facb0cb5 --- /dev/null +++ b/controllers/raycluster_controller_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "math/rand" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + routev1 "github.com/openshift/api/route/v1" +) + +func stringInList(l []string, s string) bool { + for _, i := range l { + if i == s { + return true + } + } + return false +} + +var letters = []rune("abcdefghijklmnopqrstuvwxyz") +var r = rand.New(rand.NewSource(time.Now().UnixNano())) + +func randSeq(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letters[r.Intn(len(letters))] + } + return string(b) +} + +var _ = Describe("RayCluster controller", func() { + Context("RayCluster controller test", func() { + var rayClusterName = "test-raycluster" + var typeNamespaceName types.NamespacedName + ctx := context.Background() + BeforeEach(func() { + By("Generate random number so each run is creating unique") + rString := randSeq(10) + rayClusterName = rayClusterName + "-" + rString + typeNamespaceName = types.NamespacedName{Name: rayClusterName, Namespace: rayClusterName} + By("Creating a namespace for running the tests.") + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + }, + } + var err error + err = k8sClient.Create(ctx, namespace) + Expect(err).To(Not(HaveOccurred())) + + By("creating a basic instance of the RayCluster CR") + raycluster := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + Namespace: rayClusterName, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + corev1.Container{}, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + }, + } + err = k8sClient.Get(ctx, typeNamespaceName, &rayv1.RayCluster{}) + Expect(errors.IsNotFound(err)).To(Equal(true)) + err = k8sClient.Create(ctx, raycluster) + Expect(err).To(Not(HaveOccurred())) + }) + + AfterEach(func() { + By("removing the instance of the RayCluster used") + // err := clientSet.CoreV1().Namespaces().Delete(ctx, RayClusterName, metav1.DeleteOptions{}) + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + if err != nil { + Expect(errors.IsNotFound(err)).To(Equal(true)) + } else { + Expect(err).To(Not(HaveOccurred())) + _ = k8sClient.Delete(ctx, &foundRayCluster) + } + Eventually(func() bool { + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + return errors.IsNotFound(err) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + + It("should have oauth finalizer set", func() { + foundRayCluster := rayv1.RayCluster{} + Eventually(func() bool { + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + return stringInList(foundRayCluster.Finalizers, CodeflareOAuthFinalizer) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + + Context("Cluster has OAuth annotation", func() { + BeforeEach(func() { + By("adding OAuth annotation to RayCluster") + Eventually(func() error { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + if foundRayCluster.Annotations == nil { + foundRayCluster.Annotations = map[string]string{"codeflare.dev/oauth": "true"} + } else { + foundRayCluster.Annotations["codeflare.dev/oauth"] = "'true'" + } + return k8sClient.Update(ctx, &foundRayCluster) + }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) + By("waiting for dependent resources to be created") + Eventually(func() error { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + if err != nil { + return err + } + return nil + }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) + }) + + It("should set owner references for all resources", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).ToNot(HaveOccurred()) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should delete OAuth resources when annotation is removed", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + delete(foundRayCluster.Annotations, "codeflare.dev/oauth") + err = k8sClient.Update(ctx, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + + It("should remove CRB when the RayCluster is deleted", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Delete(ctx, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + }) + }) +}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go new file mode 100644 index 000000000..0110179f7 --- /dev/null +++ b/controllers/suite_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "io" + "net/http" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + routev1 "github.com/openshift/api/route/v1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +const ( + RayClusterCRDFileDownload = "https://raw.githubusercontent.com/ray-project/kuberay/master/ray-operator/config/crd/bases/ray.io_rayclusters.yaml" + RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/route.crd.yaml" +) + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + var err error + var fRoute, fRaycluster *os.File + + By("Creating and downloading necessary crds") + err = os.Mkdir("./test-crds", os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + fRoute, err = os.Create("./test-crds/route.yaml") + Expect(err).ToNot(HaveOccurred()) + defer fRoute.Close() + resp, err := http.Get(RouteCRDFileDownload) + Expect(err).ToNot(HaveOccurred()) + _, err = io.Copy(fRoute, resp.Body) + Expect(err).ToNot(HaveOccurred()) + fRaycluster, err = os.Create("./test-crds/raycluster.yaml") + Expect(err).ToNot(HaveOccurred()) + defer fRaycluster.Close() + resp, err = http.Get(RayClusterCRDFileDownload) + Expect(err).ToNot(HaveOccurred()) + _, err = io.Copy(fRaycluster, resp.Body) + Expect(err).ToNot(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "config", "crd"), + filepath.Join(".", "test-crds"), + }, + ErrorIfCRDPathMissing: true, + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + clientSet, err := kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + err = rayv1.AddToScheme(scheme.Scheme) + Expect(err).To(Not(HaveOccurred())) + err = routev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).NotTo(HaveOccurred()) + err = (&RayClusterReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + kubeClient: clientSet, + CookieSalt: "foo", + }).SetupWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + go func() { + defer GinkgoRecover() + err = k8sManager.Start(context.Background()) + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := os.RemoveAll("./test-crds") + Expect(err).NotTo(HaveOccurred()) + err = testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/go.mod b/go.mod index 7f846b2b4..93fa2c9fc 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module github.com/project-codeflare/codeflare-operator go 1.20 require ( + github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 + github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c + github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/project-codeflare/instascale v0.4.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 github.com/ray-project/kuberay/ray-operator v1.0.0 @@ -46,6 +48,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/glog v1.1.2 // indirect @@ -55,6 +58,7 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -72,7 +76,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect - github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -102,6 +105,7 @@ require ( golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect diff --git a/go.sum b/go.sum index 60345cd59..7d5f4c95a 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,7 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -216,6 +217,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -366,6 +368,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -384,8 +387,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 h1:30LG8751WElZmWA3mVS8l23l2oZnUCqbDkLCyy0U/p0= -github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= +github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a h1:Yk9J5qXjp+yfSRCzS0EElrhpTgfYJ+S+W/z84cmlmX4= +github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 h1:IkTmd/W/zxcsC5s4EbnW74PFpkQVEiTc/8rWWwFw0Ok= @@ -450,6 +453,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -569,6 +573,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -764,6 +769,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 5e2bb6e37..58aa02893 100644 --- a/main.go +++ b/main.go @@ -30,13 +30,16 @@ import ( quotasubtreev1alpha1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1alpha1" mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -51,7 +54,9 @@ import ( configv1 "github.com/openshift/api/config/v1" machinev1beta1 "github.com/openshift/api/machine/v1beta1" + routev1 "github.com/openshift/api/route/v1" + cfoControllers "github.com/project-codeflare/codeflare-operator/controllers" "github.com/project-codeflare/codeflare-operator/pkg/config" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -75,6 +80,10 @@ func init() { // InstaScale utilruntime.Must(configv1.Install(scheme)) utilruntime.Must(machinev1beta1.Install(scheme)) + // Ray + utilruntime.Must(rayv1.AddToScheme(scheme)) + // OpenShift Route + utilruntime.Must(routev1.Install(scheme)) } func main() { @@ -171,6 +180,13 @@ func main() { exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") } + if v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")); v { + rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} + exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") + } else if err != nil { + exitOnError(err, "Could not determine if RayCluster CR present on cluster.") + } + exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") @@ -221,6 +237,23 @@ func createConfigMap(ctx context.Context, client kubernetes.Interface, ns, name return err } +func HasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) { + gv, kind := gvk.ToAPIVersionAndKind() + if resources, err := dc.ServerResourcesForGroupVersion(gv); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } else { + for _, res := range resources.APIResources { + if res.Kind == kind { + return true, nil + } + } + } + return false, nil +} + func namespaceOrDie() string { // This way assumes you've set the NAMESPACE environment variable either manually, when running // the operator standalone, or using the downward API, when running the operator in-cluster. From 4f0c9c1df24fe712c43d718f1340dd1c5f95825f Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 2 Apr 2024 10:09:46 -0400 Subject: [PATCH 126/369] make oauth part of config and remove annotation check Signed-off-by: Kevin --- controllers/raycluster_controller.go | 48 +------- controllers/raycluster_controller_test.go | 132 ++++++++-------------- main.go | 4 +- pkg/config/config.go | 2 + 4 files changed, 54 insertions(+), 132 deletions(-) diff --git a/controllers/raycluster_controller.go b/controllers/raycluster_controller.go index 00d7d559d..54ebf07ae 100644 --- a/controllers/raycluster_controller.go +++ b/controllers/raycluster_controller.go @@ -21,7 +21,6 @@ import ( "crypto/rand" "crypto/sha1" "encoding/base64" - "strconv" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -128,52 +127,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - val, ok := cluster.ObjectMeta.Annotations["codeflare.dev/oauth"] - boolVal, err := strconv.ParseBool(val) - if err != nil { - logger.Error(err, "Could not convert codeflare.dev/oauth value to bool", "codeflare.dev/oauth", val) - } - if !ok || err != nil || !boolVal { - logger.Info("Removing all OAuth Objects") - err := r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthSecretNameFromCluster(&cluster)}, &corev1.Secret{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth Secret, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceNameFromCluster(&cluster)}, &corev1.Service{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth Service, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceAccountNameFromCluster(&cluster)}, &corev1.ServiceAccount{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth ServiceAccount, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth CRB, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: routeNameFromCluster(&cluster)}, &routev1.Route{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth Route, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - return ctrl.Result{}, nil - } - - _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth Route") } diff --git a/controllers/raycluster_controller_test.go b/controllers/raycluster_controller_test.go index 0facb0cb5..fbbaaa466 100644 --- a/controllers/raycluster_controller_test.go +++ b/controllers/raycluster_controller_test.go @@ -125,98 +125,62 @@ var _ = Describe("RayCluster controller", func() { }, SpecTimeout(time.Second*10)).Should(Equal(true)) }) - Context("Cluster has OAuth annotation", func() { - BeforeEach(func() { - By("adding OAuth annotation to RayCluster") - Eventually(func() error { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - if foundRayCluster.Annotations == nil { - foundRayCluster.Annotations = map[string]string{"codeflare.dev/oauth": "true"} - } else { - foundRayCluster.Annotations["codeflare.dev/oauth"] = "'true'" - } - return k8sClient.Update(ctx, &foundRayCluster) - }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) - By("waiting for dependent resources to be created") - Eventually(func() error { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) - if err != nil { - return err - } - return nil - }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) - }) - - It("should set owner references for all resources", func() { + It("should create all oauth resources", func() { + Eventually(func() error { foundRayCluster := rayv1.RayCluster{} err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).ToNot(HaveOccurred()) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) - Expect(err).To(Not(HaveOccurred())) - }) + if err != nil { + return err + } + return nil + }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) + }) - It("should delete OAuth resources when annotation is removed", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - delete(foundRayCluster.Annotations, "codeflare.dev/oauth") - err = k8sClient.Update(ctx, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - }) - - It("should remove CRB when the RayCluster is deleted", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Delete(ctx, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - }) + It("should set owner references for all resources", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).ToNot(HaveOccurred()) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should remove CRB when the RayCluster is deleted", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Delete(ctx, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) }) }) }) diff --git a/main.go b/main.go index 58aa02893..951dbf988 100644 --- a/main.go +++ b/main.go @@ -134,6 +134,7 @@ func main() { MaxScaleoutAllowed: 5, }, }, + RayClusterOAuth: pointer.Bool(true), } kubeConfig, err := ctrl.GetConfig() @@ -180,7 +181,8 @@ func main() { exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") } - if v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")); v { + v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + if v && *cfg.RayClusterOAuth { rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 295e3e000..79a944bfe 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -36,6 +36,8 @@ type CodeFlareOperatorConfiguration struct { // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` + + RayClusterOAuth *bool `json:"rayClusterOAuth,omitempty"` } type InstaScaleConfiguration struct { From 9ea041fc68aad6ba5eb9d9509d783b0307905ca3 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 2 Apr 2024 12:26:51 -0400 Subject: [PATCH 127/369] address comments from PR review Signed-off-by: Kevin --- controllers/raycluster_controller.go | 36 +++++++++++----------------- main.go | 6 +++-- pkg/config/config.go | 6 ++++- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/controllers/raycluster_controller.go b/controllers/raycluster_controller.go index 54ebf07ae..8d7299b5e 100644 --- a/controllers/raycluster_controller.go +++ b/controllers/raycluster_controller.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" coreapply "k8s.io/client-go/applyconfigurations/core/v1" v1 "k8s.io/client-go/applyconfigurations/meta/v1" @@ -56,12 +55,9 @@ type RayClusterReconciler struct { const ( requeueTime = 10 controllerName = "codeflare-raycluster-controller" - oauthAnnotation = "codeflare.dev/oauth=true" - CodeflareOAuthFinalizer = "codeflare.dev/oauth-finalizer" + CodeflareOAuthFinalizer = "openshift.ai/oauth-finalizer" OAuthServicePort = 443 OAuthServicePortName = "oauth-proxy" - strTrue = "true" - strFalse = "false" logRequeueing = "requeueing" ) @@ -106,24 +102,30 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) controllerutil.AddFinalizer(&cluster, CodeflareOAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { // this log is info level since errors are not fatal and are expected - logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, strTrue) + logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } } } else if controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { - err := r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, - ) + err := client.IgnoreNotFound(r.Client.Delete( + ctx, + &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: crbNameFromCluster(&cluster), + }, + }, + &deleteOptions, + )) if err != nil { - logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, strTrue) + logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } controllerutil.RemoveFinalizer(&cluster, CodeflareOAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { - logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, strTrue) + logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - logger.Info("Successfully removed finalizer.", logRequeueing, strFalse) + logger.Info("Successfully removed finalizer.", logRequeueing, false) return ctrl.Result{}, nil } @@ -159,16 +161,6 @@ func crbNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} } -func (r *RayClusterReconciler) deleteIfNotExist(ctx context.Context, namespacedName types.NamespacedName, obj client.Object) error { - err := r.Client.Get(ctx, namespacedName, obj) - if err != nil && errors.IsNotFound(err) { - return nil - } else if err != nil { - return err - } - return r.Client.Delete(ctx, obj, &deleteOptions) -} - func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.ClusterRoleBindingApplyConfiguration { return rbacapply.ClusterRoleBinding( crbNameFromCluster(cluster)). diff --git a/main.go b/main.go index 951dbf988..99b293cdb 100644 --- a/main.go +++ b/main.go @@ -134,7 +134,9 @@ func main() { MaxScaleoutAllowed: 5, }, }, - RayClusterOAuth: pointer.Bool(true), + KubeRay: &config.KubeRayConfiguration{ + RayDashboardOAuthEnabled: pointer.Bool(true), + }, } kubeConfig, err := ctrl.GetConfig() @@ -182,7 +184,7 @@ func main() { } v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) - if v && *cfg.RayClusterOAuth { + if v && *cfg.KubeRay.RayDashboardOAuthEnabled { rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 79a944bfe..10b547676 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -37,7 +37,11 @@ type CodeFlareOperatorConfiguration struct { // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` - RayClusterOAuth *bool `json:"rayClusterOAuth,omitempty"` + KubeRay *KubeRayConfiguration `json:"kuberay,omitempty"` +} + +type KubeRayConfiguration struct { + RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` } type InstaScaleConfiguration struct { From 5e5f5b30b9908befb0dcff0b58c1cd2d8e40895d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 3 Apr 2024 14:45:58 +0200 Subject: [PATCH 128/369] Move controllers package into pkg --- Dockerfile | 1 - main.go | 4 ++-- {controllers => pkg/controllers}/raycluster_controller.go | 0 .../controllers}/raycluster_controller_test.go | 0 {controllers => pkg/controllers}/suite_test.go | 0 5 files changed, 2 insertions(+), 3 deletions(-) rename {controllers => pkg/controllers}/raycluster_controller.go (100%) rename {controllers => pkg/controllers}/raycluster_controller_test.go (100%) rename {controllers => pkg/controllers}/suite_test.go (100%) diff --git a/Dockerfile b/Dockerfile index 26f4c23db..f5c51c3f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ RUN go mod download # Copy the Go sources COPY main.go main.go COPY pkg/ pkg/ -COPY controllers/ controllers/ # Build USER root diff --git a/main.go b/main.go index 99b293cdb..04cd33ec2 100644 --- a/main.go +++ b/main.go @@ -56,8 +56,8 @@ import ( machinev1beta1 "github.com/openshift/api/machine/v1beta1" routev1 "github.com/openshift/api/route/v1" - cfoControllers "github.com/project-codeflare/codeflare-operator/controllers" "github.com/project-codeflare/codeflare-operator/pkg/config" + "github.com/project-codeflare/codeflare-operator/pkg/controllers" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports @@ -185,7 +185,7 @@ func main() { v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if v && *cfg.KubeRay.RayDashboardOAuthEnabled { - rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} + rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") diff --git a/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go similarity index 100% rename from controllers/raycluster_controller.go rename to pkg/controllers/raycluster_controller.go diff --git a/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go similarity index 100% rename from controllers/raycluster_controller_test.go rename to pkg/controllers/raycluster_controller_test.go diff --git a/controllers/suite_test.go b/pkg/controllers/suite_test.go similarity index 100% rename from controllers/suite_test.go rename to pkg/controllers/suite_test.go From 2f100afe80d753dfd13518cc74088df0fa2153b6 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 3 Apr 2024 14:49:39 +0200 Subject: [PATCH 129/369] Scope RayCluster finalizer into ray.openshift.ai --- pkg/controllers/raycluster_controller.go | 44 +++++++++---------- pkg/controllers/raycluster_controller_test.go | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 8d7299b5e..53999591c 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -53,12 +53,12 @@ type RayClusterReconciler struct { } const ( - requeueTime = 10 - controllerName = "codeflare-raycluster-controller" - CodeflareOAuthFinalizer = "openshift.ai/oauth-finalizer" - OAuthServicePort = 443 - OAuthServicePortName = "oauth-proxy" - logRequeueing = "requeueing" + requeueTime = 10 + controllerName = "codeflare-raycluster-controller" + oAuthFinalizer = "ray.openshift.ai/oauth-finalizer" + oAuthServicePort = 443 + oAuthServicePortName = "oauth-proxy" + logRequeueing = "requeueing" ) var ( @@ -66,13 +66,13 @@ var ( deleteOptions = client.DeleteOptions{PropagationPolicy: &deletePolicy} ) -//+kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update -//+kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get -//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get -//+kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get -//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get +// +kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -97,16 +97,16 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if cluster.ObjectMeta.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { - logger.Info("Add a finalizer", "finalizer", CodeflareOAuthFinalizer) - controllerutil.AddFinalizer(&cluster, CodeflareOAuthFinalizer) + if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { + logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) + controllerutil.AddFinalizer(&cluster, oAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { // this log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } } - } else if controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { + } else if controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { err := client.IgnoreNotFound(r.Client.Delete( ctx, &rbacv1.ClusterRoleBinding{ @@ -120,7 +120,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - controllerutil.RemoveFinalizer(&cluster, CodeflareOAuthFinalizer) + controllerutil.RemoveFinalizer(&cluster, oAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err @@ -208,7 +208,7 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfig WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(routeapply.RouteSpec(). WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). - WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((OAuthServicePortName)))). + WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))). WithTLS(routeapply.TLSConfig(). WithInsecureEdgeTerminationPolicy(routev1.InsecureEdgeTerminationPolicyRedirect). WithTermination(routev1.TLSTerminationReencrypt), @@ -235,9 +235,9 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfi coreapply.ServiceSpec(). WithPorts( coreapply.ServicePort(). - WithName(OAuthServicePortName). - WithPort(OAuthServicePort). - WithTargetPort(intstr.FromString(OAuthServicePortName)). + WithName(oAuthServicePortName). + WithPort(oAuthServicePort). + WithTargetPort(intstr.FromString(oAuthServicePortName)). WithProtocol(corev1.ProtocolTCP), ). WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), diff --git a/pkg/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go index fbbaaa466..6592aa292 100644 --- a/pkg/controllers/raycluster_controller_test.go +++ b/pkg/controllers/raycluster_controller_test.go @@ -121,7 +121,7 @@ var _ = Describe("RayCluster controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) Expect(err).To(Not(HaveOccurred())) - return stringInList(foundRayCluster.Finalizers, CodeflareOAuthFinalizer) + return stringInList(foundRayCluster.Finalizers, oAuthFinalizer) }, SpecTimeout(time.Second*10)).Should(Equal(true)) }) From 22d75f6b8ab1f6ee36c0340b9c0a842f8c4048a3 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 3 Apr 2024 18:49:55 +0200 Subject: [PATCH 130/369] Add missing RBAC for oauth-proxy ClusterRoleBinding --- config/rbac/role.yaml | 8 ++++++++ pkg/controllers/raycluster_controller.go | 1 + 2 files changed, 9 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c7ef78b42..e5a2ca9f4 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -56,6 +56,14 @@ rules: - get - patch - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + verbs: + - delete + - get + - patch - apiGroups: - route.openshift.io resources: diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 53999591c..9e45bf78d 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -73,6 +73,7 @@ var ( // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get // +kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get // +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=patch;delete;get // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From a8e2a9315e545fa5a0cfd88a42adcd988e3a48ed Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 4 Apr 2024 16:03:29 -0400 Subject: [PATCH 131/369] Disable MCAD by default Signed-off-by: Anish Asthana --- main.go | 15 +++++++++------ pkg/config/config.go | 4 +++- test/e2e/setup.sh | 10 ++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 04cd33ec2..42cc5a3e2 100644 --- a/main.go +++ b/main.go @@ -127,7 +127,8 @@ func main() { }, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, - MCAD: &mcadconfig.MCADConfiguration{}, + MCADEnabled: pointer.Bool(false), + MCAD: &mcadconfig.MCADConfiguration{}, InstaScale: &config.InstaScaleConfiguration{ Enabled: pointer.Bool(false), InstaScaleConfiguration: instascaleconfig.InstaScaleConfiguration{ @@ -167,12 +168,14 @@ func main() { }) exitOnError(err, "unable to start manager") - mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) - if mcadQueueController == nil { - // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object - os.Exit(1) + if pointer.BoolDeref(cfg.MCADEnabled, false) { + mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) + if mcadQueueController == nil { + // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object + os.Exit(1) + } + mcadQueueController.Run(ctx.Done()) } - mcadQueueController.Run(ctx.Done()) if pointer.BoolDeref(cfg.InstaScale.Enabled, false) { instaScaleController := &instascale.AppWrapperReconciler{ diff --git a/pkg/config/config.go b/pkg/config/config.go index 10b547676..36234a3c6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -32,7 +32,9 @@ type CodeFlareOperatorConfiguration struct { ControllerManager `json:",inline"` // The MCAD controller configuration - MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` + // MCADEnabled defaults to false + MCADEnabled *bool `json:"mcadEnabled,omitempty"` + MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index d5c99cbf8..1f8922aa8 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -56,3 +56,13 @@ roleRef: kind: ClusterRole name: mcad-controller-rayclusters EOF + +cat < Date: Fri, 5 Apr 2024 09:32:19 +0200 Subject: [PATCH 132/369] Enable MCAD in e2e tests --- config/e2e/config.yaml | 8 ++++++++ config/e2e/kustomization.yaml | 1 + main.go | 9 +++++---- pkg/config/config.go | 13 ++++++++++--- test/e2e/setup.sh | 10 ---------- 5 files changed, 24 insertions(+), 17 deletions(-) create mode 100644 config/e2e/config.yaml diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml new file mode 100644 index 000000000..4b01a4b0e --- /dev/null +++ b/config/e2e/config.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: codeflare-operator-config +data: + config.yaml: | + mcad: + enabled: true diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index e06123b2a..772370da6 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,6 +1,7 @@ namespace: openshift-operators bases: +- config.yaml - ../default patches: diff --git a/main.go b/main.go index 42cc5a3e2..05a53f074 100644 --- a/main.go +++ b/main.go @@ -127,8 +127,9 @@ func main() { }, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, - MCADEnabled: pointer.Bool(false), - MCAD: &mcadconfig.MCADConfiguration{}, + MCAD: &config.MCADConfiguration{ + Enabled: pointer.Bool(false), + }, InstaScale: &config.InstaScaleConfiguration{ Enabled: pointer.Bool(false), InstaScaleConfiguration: instascaleconfig.InstaScaleConfiguration{ @@ -168,8 +169,8 @@ func main() { }) exitOnError(err, "unable to start manager") - if pointer.BoolDeref(cfg.MCADEnabled, false) { - mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) + if pointer.BoolDeref(cfg.MCAD.Enabled, false) { + mcadQueueController := mcad.NewJobController(mgr.GetConfig(), &cfg.MCAD.MCADConfiguration, &mcadconfig.MCADConfigurationExtended{}) if mcadQueueController == nil { // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object os.Exit(1) diff --git a/pkg/config/config.go b/pkg/config/config.go index 36234a3c6..85a03385c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -32,9 +32,7 @@ type CodeFlareOperatorConfiguration struct { ControllerManager `json:",inline"` // The MCAD controller configuration - // MCADEnabled defaults to false - MCADEnabled *bool `json:"mcadEnabled,omitempty"` - MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` + MCAD *MCADConfiguration `json:"mcad,omitempty"` // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` @@ -46,6 +44,15 @@ type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` } +type MCADConfiguration struct { + // enabled controls whether the MCAD controller is started. + // It defaults to false. + Enabled *bool `json:"enabled,omitempty"` + + // The InstaScale controller configuration + mcad.MCADConfiguration `json:",inline,omitempty"` +} + type InstaScaleConfiguration struct { // enabled controls whether the InstaScale controller is started. // It may default to true on platforms that InstaScale supports. diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 1f8922aa8..d5c99cbf8 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -56,13 +56,3 @@ roleRef: kind: ClusterRole name: mcad-controller-rayclusters EOF - -cat < Date: Fri, 5 Apr 2024 10:03:18 +0200 Subject: [PATCH 133/369] Enable MCAD in OLM tests --- .github/workflows/olm_tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index aa8a3b172..b6932d893 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -92,6 +92,10 @@ jobs: echo Waiting for Deployment to be ready timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s + + echo Patch the CodeFlare operator ConfigMap + kubectl patch -n '${{ env.SUBSCRIPTION_NAMESPACE }}' cm codeflare-operator-config --type merge -p '{"data":{"config.yaml":"mcad:\n enabled: true"}}' + env: CATALOG_SOURCE_NAME: "codeflare-olm-test" CATALOG_SOURCE_NAMESPACE: "olm" From 465da2068475f437dba4b41736fda7dc3c62d25a Mon Sep 17 00:00:00 2001 From: Christian Zaccaria <73656840+ChristianZaccaria@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:31:21 +0100 Subject: [PATCH 134/369] Refactor creation logic of ingress/routes into RayCluster Controller (#493) * WIP - Creation/Deletion of ingress and routes in RayCluster Controller * Always run RayCluster Controller * Add support file for RC Controller * Disable OAuth for e2e tests * Add OAuth config to reconciler struct --- .github/workflows/e2e_tests.yaml | 1 + main.go | 4 +- pkg/controllers/raycluster_controller.go | 102 +++++++++---- pkg/controllers/support.go | 181 +++++++++++++++++++++++ 4 files changed, 258 insertions(+), 30 deletions(-) create mode 100644 pkg/controllers/support.go diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0fc1ec88b..2e8fdbc1f 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -60,6 +60,7 @@ jobs: run: | echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator + sed -i 's/RayDashboardOAuthEnabled: pointer.Bool(true)/RayDashboardOAuthEnabled: pointer.Bool(false)/' main.go make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager diff --git a/main.go b/main.go index 05a53f074..658b55e90 100644 --- a/main.go +++ b/main.go @@ -188,8 +188,8 @@ func main() { } v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) - if v && *cfg.KubeRay.RayDashboardOAuthEnabled { - rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} + if v { + rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 9e45bf78d..23104c2ab 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -41,6 +41,8 @@ import ( routev1 "github.com/openshift/api/route/v1" routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" + + "github.com/project-codeflare/codeflare-operator/pkg/config" ) // RayClusterReconciler reconciles a RayCluster object @@ -50,15 +52,17 @@ type RayClusterReconciler struct { routeClient *routev1client.RouteV1Client Scheme *runtime.Scheme CookieSalt string + Config *config.CodeFlareOperatorConfiguration } const ( - requeueTime = 10 - controllerName = "codeflare-raycluster-controller" - oAuthFinalizer = "ray.openshift.ai/oauth-finalizer" - oAuthServicePort = 443 - oAuthServicePortName = "oauth-proxy" - logRequeueing = "requeueing" + requeueTime = 10 + controllerName = "codeflare-raycluster-controller" + oAuthFinalizer = "ray.openshift.ai/oauth-finalizer" + oAuthServicePort = 443 + oAuthServicePortName = "oauth-proxy" + ingressServicePortName = "dashboard" + logRequeueing = "requeueing" ) var ( @@ -97,6 +101,10 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } + isLocalInteractive := annotationBoolVal(ctx, &cluster, "sdk.codeflare.dev/local_interactive", false) + ingressDomain := "" // FIX - CFO will retrieve it. + isOpenShift, ingressHost := getClusterType(ctx, r.kubeClient, &cluster, ingressDomain) + if cluster.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) @@ -130,29 +138,63 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth Route") - } + if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && isOpenShift { + logger.Info("Creating OAuth Objects") + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Route") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to create OAuth Secret") - } + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to create OAuth Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth Service") - } + _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Service") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth ServiceAccount") - } + _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ServiceAccount") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth ClusterRoleBinding") + _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ClusterRoleBinding") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + + if isLocalInteractive { + logger.Info("Creating RayClient Route") + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Route") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } + + } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !isOpenShift { + logger.Info("Creating Dashboard Ingress") + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, ingressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + // This log is info level since errors are not fatal and are expected + logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + if isLocalInteractive && ingressDomain != "" { + logger.Info("Creating RayClient Ingress") + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Ingress") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } } return ctrl.Result{}, nil @@ -193,19 +235,23 @@ func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountA WithAnnotations(map[string]string{ "serviceaccounts.openshift.io/oauth-redirectreference.first": "" + `{"kind":"OAuthRedirectReference","apiVersion":"v1",` + - `"reference":{"kind":"Route","name":"` + routeNameFromCluster(cluster) + `"}}`, + `"reference":{"kind":"Route","name":"` + dashboardNameFromCluster(cluster) + `"}}`, }). WithOwnerReferences( v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } -func routeNameFromCluster(cluster *rayv1.RayCluster) string { +func dashboardNameFromCluster(cluster *rayv1.RayCluster) string { return "ray-dashboard-" + cluster.Name } +func rayClientNameFromCluster(cluster *rayv1.RayCluster) string { + return "rayclient-" + cluster.Name +} + func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { - return routeapply.Route(routeNameFromCluster(cluster), cluster.Namespace). + return routeapply.Route(dashboardNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(routeapply.RouteSpec(). WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go new file mode 100644 index 000000000..348cd03de --- /dev/null +++ b/pkg/controllers/support.go @@ -0,0 +1,181 @@ +package controllers + +import ( + "context" + "fmt" + "strconv" + "strings" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" + networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + + routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" +) + +func serviceNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-head-svc" +} + +func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { + return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSpec(routeapply.RouteSpec(). + WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace). + WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)). + WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))). + WithTLS(routeapply.TLSConfig().WithTermination("passthrough")), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +// Create an Ingress object for the RayCluster +func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *networkingv1ac.IngressApplyConfiguration { + return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithAnnotations(map[string]string{ + "nginx.ingress.kubernetes.io/rewrite-target": "/", + "nginx.ingress.kubernetes.io/ssl-redirect": "true", + "nginx.ingress.kubernetes.io/ssl-passthrough": "true", + }). + WithOwnerReferences(v1.OwnerReference(). + WithAPIVersion(cluster.APIVersion). + WithKind(cluster.Kind). + WithName(cluster.Name). + WithUID(types.UID(cluster.UID))). + WithSpec(networkingv1ac.IngressSpec(). + WithIngressClassName("nginx"). + WithRules(networkingv1ac.IngressRule(). + WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingressDomain). + WithHTTP(networkingv1ac.HTTPIngressRuleValue(). + WithPaths(networkingv1ac.HTTPIngressPath(). + WithPath("/"). + WithPathType(networkingv1.PathTypeImplementationSpecific). + WithBackend(networkingv1ac.IngressBackend(). + WithService(networkingv1ac.IngressServiceBackend(). + WithName(serviceNameFromCluster(cluster)). + WithPort(networkingv1ac.ServiceBackendPort(). + WithNumber(10001), + ), + ), + ), + ), + ), + ), + ) +} + +// Create an Ingress object for the RayCluster +func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { + return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithOwnerReferences(v1.OwnerReference(). + WithAPIVersion(cluster.APIVersion). + WithKind(cluster.Kind). + WithName(cluster.Name). + WithUID(types.UID(cluster.UID))). + WithSpec(networkingv1ac.IngressSpec(). + WithRules(networkingv1ac.IngressRule(). + WithHost(ingressHost). // KinD hostname or ingressDomain + WithHTTP(networkingv1ac.HTTPIngressRuleValue(). + WithPaths(networkingv1ac.HTTPIngressPath(). + WithPath("/"). + WithPathType(networkingv1.PathTypePrefix). + WithBackend(networkingv1ac.IngressBackend(). + WithService(networkingv1ac.IngressServiceBackend(). + WithName(serviceNameFromCluster(cluster)). + WithPort(networkingv1ac.ServiceBackendPort(). + WithName(ingressServicePortName), + ), + ), + ), + ), + ), + ), + ) +} + +// isOnKindCluster checks if the current cluster is a KinD cluster. +// It searches for a node with a label commonly used by KinD clusters. +func isOnKindCluster(clientset *kubernetes.Clientset) (bool, error) { + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: "kubernetes.io/hostname=kind-control-plane", + }) + if err != nil { + return false, err + } + // If we find one or more nodes with the label, assume it's a KinD cluster. + return len(nodes.Items) > 0, nil +} + +// getDiscoveryClient returns a discovery client for the current reconciler +func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) { + return discovery.NewDiscoveryClientForConfig(config) +} + +// Check where we are running. We are trying to distinguish here whether +// this is vanilla kubernetes cluster or Openshift +func getClusterType(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) (bool, string) { + // The discovery package is used to discover APIs supported by a Kubernetes API server. + logger := ctrl.LoggerFrom(ctx) + config, err := ctrl.GetConfig() + if err != nil && config == nil { + logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes") + return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + } + dclient, err := getDiscoveryClient(config) + if err != nil && dclient == nil { + logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes") + return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + } + apiGroupList, err := dclient.ServerGroups() + if err != nil { + logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") + return false, "" + } + for i := 0; i < len(apiGroupList.Groups); i++ { + if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { + logger.Info("We detected being on OpenShift!") + return true, "" + } + } + onKind, _ := isOnKindCluster(clientset) + if onKind && ingressDomain == "" { + logger.Info("We detected being on a KinD cluster!") + return false, "kind" + } + logger.Info("We detected being on Vanilla Kubernetes!") + return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) +} + +func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { + if r.Config != nil && r.Config.KubeRay != nil && r.Config.KubeRay.RayDashboardOAuthEnabled != nil { + return *r.Config.KubeRay.RayDashboardOAuthEnabled + } + return true +} + +func annotationBoolVal(ctx context.Context, cluster *rayv1.RayCluster, annotation string, defaultValue bool) bool { + logger := ctrl.LoggerFrom(ctx) + val, exists := cluster.ObjectMeta.Annotations[annotation] + if !exists || val == "" { + return defaultValue + } + boolVal, err := strconv.ParseBool(val) + if err != nil { + logger.Error(err, "Could not convert annotation value to bool", "annotation", annotation, "value", val) + return defaultValue + } + return boolVal +} From be7ac07a9ebcc06b8c51e5fdb120f6155c265fbd Mon Sep 17 00:00:00 2001 From: Mark Campbell Date: Fri, 5 Apr 2024 17:02:35 +0100 Subject: [PATCH 135/369] Removed InstaScale & MCAD from release workflow (#492) --- .../workflows/project-codeflare-release.yml | 76 +------------------ .github/workflows/tag-and-build.yml | 19 ----- 2 files changed, 2 insertions(+), 93 deletions(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 50fc26ae1..017c7b023 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -10,15 +10,9 @@ on: replaces: description: 'The previous operator semantic version that this release replaces (for example: v0.0.0)' required: true - mcad-version: - description: 'Version of multi-cluster-app-dispatcher to be released (for example: v0.0.0)' - required: true codeflare-sdk-version: description: 'Version of CodeFlare-SDK to be released (for example: v0.0.0)' required: true - instascale-version: - description: 'Version of InstaScale to be released (for example: v0.0.0)' - required: true kuberay-version: description: 'Tested version of KubeRay (for example: v0.0.0)' required: true @@ -48,79 +42,13 @@ jobs: echo "Below are the release parameters set for the workflow:" echo "Operator Version: ${{ github.event.inputs.operator-version }}" echo "Replaces: ${{ github.event.inputs.replaces }}" - echo "MCAD Version: ${{ github.event.inputs.mcad-version }}" echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}" - echo "InstaScale Version: ${{ github.event.inputs.instascale-version }}" echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}" echo "Is Stable: ${{ github.event.inputs.is-stable }}" echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" echo "Quay Organization: ${{ github.event.inputs.quay-organization }}" echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" - release-mcad: - needs: check-kuberay-version - runs-on: ubuntu-latest - - steps: - - name: Check if MCAD release does exist - run: | - status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/${{ github.event.inputs.mcad-version }}) - if [[ "$status_code" == "200" ]]; then - echo "MCAD release with version ${{ github.event.inputs.mcad-version }} already exist. Will not create MCAD release." - fi - echo "MCAD_RELEASE_STATUS_CODE=$status_code" >> $GITHUB_ENV - - - name: Release MCAD - run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.MCAD_RELEASE_STATUS_CODE != '200' }} - - - name: Wait for MCAD run to finish - run: | - # wait for a while for Run to be started - sleep 5 - run_id=$(gh run list --workflow mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --limit 1 --json databaseId --jq .[].databaseId) - gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --interval 10 --exit-status - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.MCAD_RELEASE_STATUS_CODE != '200' }} - - release-instascale: - needs: release-mcad - runs-on: ubuntu-latest - - steps: - - name: Check if Instascale release does exist - run: | - status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/instascale/releases/tag/${{ github.event.inputs.instascale-version }}) - if [[ "$status_code" == "200" ]]; then - echo "Instascale release with version ${{ github.event.inputs.instascale-version }} already exist. Will not create Instascale release." - fi - echo "INSTASCALE_RELEASE_STATUS_CODE=$status_code" >> $GITHUB_ENV - - - name: Release InstaScale - run: | - gh workflow run instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --ref ${{ github.ref }} --field tag=${{ github.event.inputs.instascale-version }} --field mcad-version=${{ github.event.inputs.mcad-version }} - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} - - - name: Wait for InstaScale run to finish - run: | - # wait for a while for Run to be started - sleep 5 - run_id=$(gh run list --workflow instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --limit 1 --json databaseId --jq .[].databaseId) - gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --interval 10 --exit-status - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} - release-codeflare-sdk: needs: check-kuberay-version runs-on: ubuntu-latest @@ -156,13 +84,13 @@ jobs: if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} release-codeflare-operator: - needs: [release-mcad, release-instascale, release-codeflare-sdk] + needs: [release-codeflare-sdk] runs-on: ubuntu-latest steps: - name: Release CodeFlare operator run: | - gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 9dc290c48..870ac35c1 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -12,18 +12,10 @@ on: description: 'The previous semantic version that this tag replaces.' required: true default: 'v0.0.0-dev' - mcad-version: - description: 'Published version of multi-cluster-app-dispatcher' - required: true - default: 'v0.0.0-dev' codeflare-sdk-version: description: 'Published version of CodeFlare-SDK' required: true default: 'v0.0.0-dev' - instascale-version: - description: 'Published version of InstaScale' - required: true - default: 'v0.0.0-dev' kuberay-version: description: 'Tested version of KubeRay' required: true @@ -90,18 +82,9 @@ jobs: - name: Adjust Compatibility Matrix in readme run: | sed -i -E "s|(.*CodeFlare Operator.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.version }}\2${{ github.event.inputs.version }}\3|" README.md - sed -i -E "s|(.*Multi-Cluster App Dispatcher.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.mcad-version }}\2${{ github.event.inputs.mcad-version }}\3|" README.md sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md - sed -i -E "s|(.*InstaScale.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.instascale-version }}\2${{ github.event.inputs.instascale-version }}\3|" README.md sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md - - name: Adjust MCAD, SDK and InstaScale dependencies in the code - run: | - sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile - sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile - sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile - sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile - - name: Update image version in params.env run: | VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env @@ -192,8 +175,6 @@ jobs: env: VERSION: ${{ github.event.inputs.version }} PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} - INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} - MCAD_VERSION: ${{ github.event.inputs.mcad-version }} GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} From c771aae4dc990526b8b3b4d3817805fa479bded2 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 5 Apr 2024 13:04:51 -0400 Subject: [PATCH 136/369] Clean removal of MCADv1 and Instascale (#501) * Clean removal of MCAD and Instascale * entirely remove the MCAD/Instascale config structs * untested port of odh ray test to non-appwrapper version --- Makefile | 18 - config/crd/crd-appwrapper.yml | 462 ------------------ config/crd/crd-quotasubtree.yml | 115 ----- config/crd/crd-schedulingspec.yml | 80 --- config/crd/kustomization.yaml | 3 - config/crd/mcad/kustomization.yaml | 4 - config/e2e/config.yaml | 8 - config/e2e/kustomization.yaml | 1 - go.mod | 46 +- go.sum | 117 +---- main.go | 41 -- pkg/config/config.go | 28 -- test/e2e/instascale_app_wrapper.go | 140 ------ test/e2e/instascale_machinepool_test.go | 82 ---- test/e2e/instascale_machineset_test.go | 59 --- test/e2e/instascale_nodepool_test.go | 68 --- test/e2e/mnist_pytorch_mcad_job_test.go | 175 ------- ...est.go => mnist_rayjob_raycluster_test.go} | 50 +- test/odh/mcad_ray_test.go | 39 +- test/odh/pytorch_mcad_test.go | 77 --- test/upgrade/olm_upgrade_test.go | 216 -------- 21 files changed, 39 insertions(+), 1790 deletions(-) delete mode 100644 config/crd/crd-appwrapper.yml delete mode 100644 config/crd/crd-quotasubtree.yml delete mode 100644 config/crd/crd-schedulingspec.yml delete mode 100644 config/crd/mcad/kustomization.yaml delete mode 100644 config/e2e/config.yaml delete mode 100644 test/e2e/instascale_app_wrapper.go delete mode 100644 test/e2e/instascale_machinepool_test.go delete mode 100644 test/e2e/instascale_machineset_test.go delete mode 100644 test/e2e/instascale_nodepool_test.go delete mode 100644 test/e2e/mnist_pytorch_mcad_job_test.go rename test/e2e/{mnist_rayjob_mcad_raycluster_test.go => mnist_rayjob_raycluster_test.go} (81%) delete mode 100644 test/odh/pytorch_mcad_test.go delete mode 100644 test/upgrade/olm_upgrade_test.go diff --git a/Makefile b/Makefile index d2cd22d0e..e1b77a8e1 100644 --- a/Makefile +++ b/Makefile @@ -11,16 +11,6 @@ PREVIOUS_VERSION ?= v0.0.0-dev VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) -# INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.4.0 -INSTASCALE_REPO ?= github.com/project-codeflare/instascale - -# MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.40.0 -MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher -# Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) -MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} - # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v1.0.0 @@ -139,9 +129,6 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen kustomize install-yq ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml - $(KUSTOMIZE) build config/crd/mcad | $(YQ) -s '"crd-" + .spec.names.singular' --no-doc - mv crd-*.yml config/crd .PHONY: fmt fmt: ## Run go fmt against code. @@ -156,8 +143,6 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. - go get $(MCAD_REPO)@$(MCAD_VERSION) - go get $(INSTASCALE_REPO)@$(INSTASCALE_VERSION) go get github.com/ray-project/kuberay/ray-operator@$(KUBERAY_VERSION) go mod tidy @@ -166,8 +151,6 @@ build: fmt vet ## Build manager binary. go build \ -ldflags " \ -X 'main.OperatorVersion=$(BUILD_VERSION)' \ - -X 'main.McadVersion=$(MCAD_VERSION)' \ - -X 'main.InstaScaleVersion=$(INSTASCALE_VERSION)' \ -X 'main.BuildDate=$(BUILD_DATE)' \ " \ -o bin/manager main.go @@ -208,7 +191,6 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/${ENV} | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml deleted file mode 100644 index 9140857a0..000000000 --- a/config/crd/crd-appwrapper.yml +++ /dev/null @@ -1,462 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: appwrappers.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: AppWrapper - listKind: AppWrapperList - plural: appwrappers - singular: appwrapper - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Definition of AppWrapper class - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AppWrapperSpec describes how the App Wrapper will look like. - properties: - priority: - format: int32 - type: integer - priorityslope: - format: float - type: number - resources: - description: a collection of AppWrapperResource - properties: - GenericItems: - items: - description: AppWrapperGenericResource is App Wrapper aggregation resource - properties: - allocated: - description: The number of allocated replicas from this resource type - format: int32 - type: integer - completionstatus: - description: 'Optional field that drives completion status of this AppWrapper. This field within an item of an AppWrapper determines the full state of the AppWrapper. The completionstatus field contains a list of conditions that make the associate item considered completed, for instance: - completion conditions could be "Complete" or "Failed". The associated item''s level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this option is set and the conditionstatus is met the entire AppWrapper state will be changed to one of the valid AppWrapper completion state. Note: - this is an AND operation for all items where this option is set. See the list of AppWrapper states for a list of valid complete states.' - type: string - custompodresources: - description: Optional section that specifies resource requirements for non-standard k8s resources, follows same format as that of standard k8s resources. - items: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, quantity) pairs. - type: object - replicas: - type: integer - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'todo: replace with Containers []Container Contain v1.ResourceRequirements' - type: object - required: - - replicas - - requests - type: object - type: array - generictemplate: - description: The template for the resource; it is now a raw text because we don't know for what resource it should be instantiated - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - minavailable: - description: The minimal available pods to run for this AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - type: object - type: array - type: object - schedulingSpec: - description: SchedSpec specifies the parameters used for scheduling generic items wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number of running pods. - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. - type: integer - type: object - type: object - selector: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - service: - description: AppWrapperService is App Wrapper service definition - properties: - spec: - description: ServiceSpec describes the attributes that a user creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer. Default is "true". It may be set to "false" if the cluster load-balancer does not rely on NodePorts. If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. - type: boolean - clusterIP: - description: 'clusterIP is the IP address of the service and is usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be blank) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are "None", empty string (""), or a valid IP address. Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - clusterIPs: - description: "ClusterIPs is a list of IP addresses assigned to this service, and are usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be empty) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. If this field is not specified, it will be initialized from the clusterIP field. If this field is specified, clients must ensure that clusterIPs[0] and clusterIP have the same value. \n This field may hold a maximum of two entries (dual-stack IPs, in either order). These IPs must correspond to the values of the ipFamilies field. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system. - items: - type: string - type: array - externalName: - description: externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: externalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure the service in a way that assumes that external load balancers will take care of balancing the service traffic between nodes, and so each node will deliver traffic only to the node-local endpoints of the service, without masquerading the client source IP. (Traffic mistakenly sent to a node with no endpoints will be dropped.) The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). Note that traffic sent to an External IP or LoadBalancer IP from within the cluster will always get "Cluster" semantics, but clients sending to a NodePort from within the cluster may need to take traffic policy into account when picking a node. - type: string - healthCheckNodePort: - description: healthCheckNodePort specifies the healthcheck nodePort for the service. This only applies when type is set to LoadBalancer and externalTrafficPolicy is set to Local. If a value is specified, is in-range, and is not in use, it will be used. If not specified, a value will be automatically allocated. External systems (e.g. load-balancers) can use this port to determine if a given node holds endpoints for this service or not. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type). This field cannot be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: InternalTrafficPolicy describes how nodes distribute service traffic they receive on the ClusterIP. If set to "Local", the proxy will assume that pods only want to talk to endpoints of the service on the same node as the pod, dropping the traffic if there are no local endpoints. The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). - type: string - ipFamilies: - description: "IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply to \"headless\" services. This field will be wiped when updating a Service to type ExternalName. \n This field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field." - items: - description: IPFamily represents the IP Family (IPv4 or IPv6). This type is used to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this field will be set to SingleStack. Services can be "SingleStack" (a single IP family), "PreferDualStack" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or "RequireDualStack" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName. - type: string - loadBalancerClass: - description: loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature. Deprecated: This field was under-specified and its meaning varies across implementations, and it cannot support dual-stack. As of Kubernetes v1.24, users are encouraged to use implementation-specific annotations when available. This field may be removed in a future API version.' - type: string - loadBalancerSourceRanges: - description: 'If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' - items: - type: string - type: array - ports: - description: 'The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - items: - description: ServicePort contains information on service's port. - properties: - appProtocol: - description: The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and https://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. - type: string - name: - description: The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service. - type: string - nodePort: - description: 'The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not specified, a port will be allocated if this Service requires one. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: 'Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod''s container ports. If this is not specified, the value of the ''port'' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: publishNotReadyAddresses indicates that any agent which deals with endpoints for this Service should disregard any indications of ready/not-ready. The primary use case for setting this field is for a StatefulSet's Headless Service to propagate SRV DNS records for its Pods for the purpose of peer discovery. The Kubernetes controllers that generate Endpoints and EndpointSlice resources for Services interpret this to mean that all endpoints are considered "ready" even if the Pods themselves are not. Agents which consume only Kubernetes generated endpoints through the Endpoints or EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: 'Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: 'Supports "ClientIP" and "None". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client IP based session affinity. - properties: - timeoutSeconds: - description: timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - type: - description: 'type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object or EndpointSlice objects. If clusterIP is "None", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a virtual IP. "NodePort" builds on ClusterIP and allocates a port on every node which routes to the same endpoints as the clusterIP. "LoadBalancer" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the same endpoints as the clusterIP. "ExternalName" aliases this service to the specified externalName. Several other fields do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' - type: string - type: object - required: - - spec - type: object - required: - - resources - type: object - status: - description: AppWrapperStatus represents the current state of a AppWrapper - properties: - Succeeded: - description: The number of resources which reached phase Succeeded. - format: int32 - type: integer - canrun: - description: Can run? - type: boolean - conditions: - description: Represents the latest available observations of the AppWrapper's current condition. - items: - description: AppWrapperCondition describes the state of an AppWrapper at a certain point. - properties: - lastTransitionMicroTime: - description: Last time the condition transitioned from one status to another. - format: date-time - type: string - lastUpdateMicroTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human-readable message indicating details about the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of AppWrapper condition. - type: string - required: - - status - - type - type: object - type: array - controllerfirstdispatchtimestamp: - description: Microsecond level timestamp when controller first dispatches the AppWrapper - format: date-time - type: string - controllerfirsttimestamp: - description: Microsecond level timestamp when controller first sees QueueJob (by Informer) - format: date-time - type: string - failed: - description: The number of resources which reached phase Failed. - format: int32 - type: integer - filterignore: - description: Tell Informer to ignore this update message (do not generate a controller event) - type: boolean - isdispatched: - description: Is Dispatched? - type: boolean - local: - description: Indicate if message is a duplicate (for Informer to recognize duplicate messages) - type: boolean - message: - type: string - numberOfRequeueings: - description: Field to keep track of how many times a requeuing event has been triggered - type: integer - pending: - description: The number of pending pods. - format: int32 - type: integer - pendingpodconditions: - description: Represents the latest available observations of pods belonging to the AppWrapper. - items: - properties: - conditions: - items: - description: PodCondition contains details for the current condition of this pod. - properties: - lastProbeTime: - description: Last time we probed the condition. - format: date-time - type: string - lastTransitionTime: - description: Last time the condition transitioned from one status to another. - format: date-time - type: string - message: - description: Human-readable message indicating details about last transition. - type: string - reason: - description: Unique, one-word, CamelCase reason for the condition's last transition. - type: string - status: - description: 'Status is the status of the condition. Can be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - type: - description: 'Type is the type of the condition. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - required: - - status - - type - type: object - type: array - podname: - type: string - type: object - type: array - queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, ... - type: string - requeueingTimeInSeconds: - description: Field to keep track of total number of seconds spent in requeueing - type: integer - running: - format: int32 - type: integer - sender: - description: Indicate sender of this message (extremely useful for debugging) - type: string - state: - description: State - Pending, Running, Failed, Deleted - type: string - systempriority: - description: System defined Priority - format: float - type: number - template: - description: The minimal available resources to run for this AppWrapper (is this different from the MinAvailable from JobStatus) - format: int32 - type: integer - totalcpu: - description: The number of CPU consumed by all pods belonging to the AppWrapper. - format: int32 - type: integer - totalgpu: - description: The total number of GPUs consumed by all pods belonging to the AppWrapper. - format: int32 - type: integer - totalmemory: - description: The amount of memory consumed by all pods belonging to the AppWrapper. - format: int32 - type: integer - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/crd-quotasubtree.yml b/config/crd/crd-quotasubtree.yml deleted file mode 100644 index 924aaacd7..000000000 --- a/config/crd/crd-quotasubtree.yml +++ /dev/null @@ -1,115 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: quotasubtrees.quota.codeflare.dev -spec: - group: quota.codeflare.dev - names: - kind: QuotaSubtree - listKind: QuotaSubtreeList - plural: quotasubtrees - singular: quotasubtree - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: QuotaSubtree is a specification for a quota subtree resource - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: QuotaSubtreeSpec is the spec for a resource plan - properties: - children: - items: - description: Child is the spec for a QuotaSubtree resource - properties: - name: - type: string - namespace: - type: string - path: - type: string - quotas: - description: Quota is the spec for a QuotaSubtree resource - properties: - disabled: - type: boolean - hardLimit: - type: boolean - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - type: array - parent: - type: string - parentNamespace: - type: string - type: object - status: - description: QuotaSubtreeStatus is the status for a QuotaSubtree resource - properties: - children: - items: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - type: array - totalAllocation: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - required: - - children - - totalAllocation - type: object - required: - - spec - type: object - served: true - storage: true diff --git a/config/crd/crd-schedulingspec.yml b/config/crd/crd-schedulingspec.yml deleted file mode 100644 index 373dc89d8..000000000 --- a/config/crd/crd-schedulingspec.yml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: schedulingspecs.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: SchedulingSpec - listKind: SchedulingSpecList - plural: schedulingspecs - singular: schedulingspec - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. - type: integer - type: object - type: object - required: - - metadata - type: object - served: true - storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index faaff445f..3b60faba5 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,8 +2,5 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- crd-appwrapper.yml -- crd-quotasubtree.yml -- crd-schedulingspec.yml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml deleted file mode 100644 index bbf6141cf..000000000 --- a/config/crd/mcad/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.40.0 diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml deleted file mode 100644 index 4b01a4b0e..000000000 --- a/config/e2e/config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: codeflare-operator-config -data: - config.yaml: | - mcad: - enabled: true diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index 772370da6..e06123b2a 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,7 +1,6 @@ namespace: openshift-operators bases: -- config.yaml - ../default patches: diff --git a/go.mod b/go.mod index 93fa2c9fc..80f3a52b8 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,6 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a - github.com/project-codeflare/instascale v0.4.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 @@ -25,25 +23,16 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 require ( - github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.3.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect @@ -54,53 +43,32 @@ require ( github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/cel-go v0.12.7 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.7 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect - go.etcd.io/etcd/client/v3 v3.5.7 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect - go.opentelemetry.io/otel v1.19.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect - go.opentelemetry.io/otel/metric v1.19.0 // indirect - go.opentelemetry.io/otel/sdk v1.19.0 // indirect - go.opentelemetry.io/otel/trace v1.19.0 // indirect - go.opentelemetry.io/proto/otlp v1.0.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.18.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -108,22 +76,12 @@ require ( golang.org/x/tools v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.7 // indirect - k8s.io/apiserver v0.27.8 // indirect - k8s.io/kms v0.27.8 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/metrics v0.26.2 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect - sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 7d5f4c95a..51ec31f47 100644 --- a/go.sum +++ b/go.sum @@ -13,15 +13,12 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -34,12 +31,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -47,10 +41,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -58,8 +48,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -77,24 +65,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= -github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= -github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -103,13 +82,10 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -126,11 +102,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -143,7 +116,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -187,9 +159,6 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/cel-go v0.12.7 h1:jM6p55R0MKBg79hZjn1zs2OlrywZ1Vk00rxVvad1/O0= -github.com/google/cel-go v0.12.7/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -226,18 +195,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -245,8 +203,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= @@ -300,7 +256,6 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -345,8 +300,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -359,12 +312,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= @@ -389,10 +340,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a h1:Yk9J5qXjp+yfSRCzS0EElrhpTgfYJ+S+W/z84cmlmX4= github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= -github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= -github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 h1:IkTmd/W/zxcsC5s4EbnW74PFpkQVEiTc/8rWWwFw0Ok= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -427,7 +376,6 @@ github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDN github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -435,14 +383,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -459,11 +402,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -471,41 +413,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= -go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= -go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= -go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= -go.etcd.io/etcd/client/v2 v2.305.7 h1:AELPkjNR3/igjbO7CjyF1fPuVPjrblliiKj+Y6xSGOU= -go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4= -go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= -go.etcd.io/etcd/pkg/v3 v3.5.7 h1:obOzeVwerFwZ9trMWapU/VjDcYUJb5OfgC1zqEGWO/0= -go.etcd.io/etcd/raft/v3 v3.5.7 h1:aN79qxLmV3SvIq84aNTliYGmjwsW6NqJSnqmI1HLJKc= -go.etcd.io/etcd/server/v3 v3.5.7 h1:BTBD8IJUV7YFgsczZMHhMTS67XuA4KpRquL0MFOJGRk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -540,7 +453,6 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -637,8 +549,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -833,12 +743,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -854,8 +758,6 @@ google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -882,9 +784,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -913,31 +812,21 @@ k8s.io/apiextensions-apiserver v0.27.7 h1:YqIOwZAUokzxJIjunmUd4zS1v3JhK34EPXn+pP k8s.io/apiextensions-apiserver v0.27.7/go.mod h1:x0p+b5a955lfPz9gaDeBy43obM12s+N9dNHK6+dUL+g= k8s.io/apimachinery v0.27.8 h1:Xg+ogjDm8s7KmV3vZGf7uOZ0jrC6FPy2Lk/h7BIRmvg= k8s.io/apimachinery v0.27.8/go.mod h1:EIXLxLt/b1muPITiF5zlrcm7I+YnXsIgM+0GdnPTQvA= -k8s.io/apiserver v0.27.8 h1:fdrEm98wl1lxFoF8tCdQiYQOfmU3TOGkQr4YTLYHwzc= -k8s.io/apiserver v0.27.8/go.mod h1:8yLtDnOdRq9fubaea0zQ3AVheQUinb5Y/RqBUdQMT0Y= k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.27.8 h1:D7s2vZAFWKD1+1Yg6Y4C+l7k+ZjEsMc7BjY0QRKAOY8= -k8s.io/kms v0.27.8/go.mod h1:+yIM/3y+UgdaZSQaMJObNONp0i237t/dVrxswECh7a0= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/metrics v0.26.2 h1:2gUvUWWnHPdE2tyA5DvyHC8HGryr+izhY9i5dzLP06s= -k8s.io/metrics v0.26.2/go.mod h1:PX1wm9REV9hSGuw9GcXTFNDgab1KRXck3mNeiLYbRho= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= -sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 h1:puQ5YlyBjhxg+OQ1YPMJXwtk7WhC4E6AlWIQ9pC8jws= -sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3/go.mod h1:9nUXR/EgdYZto1aQ6yhwOksPR7J979jSyOqic1IgaOo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/main.go b/main.go index 658b55e90..752f146f2 100644 --- a/main.go +++ b/main.go @@ -24,12 +24,6 @@ import ( "strings" "time" - instascale "github.com/project-codeflare/instascale/controllers" - instascaleconfig "github.com/project-codeflare/instascale/pkg/config" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - quotasubtreev1alpha1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1alpha1" - mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" - mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" @@ -52,8 +46,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/yaml" - configv1 "github.com/openshift/api/config/v1" - machinev1beta1 "github.com/openshift/api/machine/v1beta1" routev1 "github.com/openshift/api/route/v1" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -74,12 +66,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - // MCAD - utilruntime.Must(mcadv1beta1.AddToScheme(scheme)) - utilruntime.Must(quotasubtreev1alpha1.AddToScheme(scheme)) - // InstaScale - utilruntime.Must(configv1.Install(scheme)) - utilruntime.Must(machinev1beta1.Install(scheme)) // Ray utilruntime.Must(rayv1.AddToScheme(scheme)) // OpenShift Route @@ -127,15 +113,6 @@ func main() { }, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, - MCAD: &config.MCADConfiguration{ - Enabled: pointer.Bool(false), - }, - InstaScale: &config.InstaScaleConfiguration{ - Enabled: pointer.Bool(false), - InstaScaleConfiguration: instascaleconfig.InstaScaleConfiguration{ - MaxScaleoutAllowed: 5, - }, - }, KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: pointer.Bool(true), }, @@ -169,24 +146,6 @@ func main() { }) exitOnError(err, "unable to start manager") - if pointer.BoolDeref(cfg.MCAD.Enabled, false) { - mcadQueueController := mcad.NewJobController(mgr.GetConfig(), &cfg.MCAD.MCADConfiguration, &mcadconfig.MCADConfigurationExtended{}) - if mcadQueueController == nil { - // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object - os.Exit(1) - } - mcadQueueController.Run(ctx.Done()) - } - - if pointer.BoolDeref(cfg.InstaScale.Enabled, false) { - instaScaleController := &instascale.AppWrapperReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Config: cfg.InstaScale.InstaScaleConfiguration, - } - exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") - } - v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if v { rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg} diff --git a/pkg/config/config.go b/pkg/config/config.go index 85a03385c..124bb0ebb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,6 @@ limitations under the License. package config import ( - instascale "github.com/project-codeflare/instascale/pkg/config" - mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" - configv1alpha1 "k8s.io/component-base/config/v1alpha1" ) @@ -31,12 +28,6 @@ type CodeFlareOperatorConfiguration struct { // ControllerManager returns the configurations for controllers ControllerManager `json:",inline"` - // The MCAD controller configuration - MCAD *MCADConfiguration `json:"mcad,omitempty"` - - // The InstaScale controller configuration - InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` - KubeRay *KubeRayConfiguration `json:"kuberay,omitempty"` } @@ -44,25 +35,6 @@ type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` } -type MCADConfiguration struct { - // enabled controls whether the MCAD controller is started. - // It defaults to false. - Enabled *bool `json:"enabled,omitempty"` - - // The InstaScale controller configuration - mcad.MCADConfiguration `json:",inline,omitempty"` -} - -type InstaScaleConfiguration struct { - // enabled controls whether the InstaScale controller is started. - // It may default to true on platforms that InstaScale supports. - // Otherwise, it defaults to false. - Enabled *bool `json:"enabled,omitempty"` - - // The InstaScale controller configuration - instascale.InstaScaleConfiguration `json:",inline,omitempty"` -} - type ControllerManager struct { // Metrics contains the controller metrics configuration // +optional diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go deleted file mode 100644 index 45305ddab..000000000 --- a/test/e2e/instascale_app_wrapper.go +++ /dev/null @@ -1,140 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { - // Batch Job - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "job", - Image: GetPyTorchImage(), - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, - }, - Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - Args: []string{"$PYTHONUSERBASE"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - }, - } - - // AppWrapper - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-instascale", - Namespace: namespace.Name, - Labels: map[string]string{ - "orderedinstance": "g4dn.xlarge", - }, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - "nvidia.com/gpu": resource.MustParse("1"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - "nvidia.com/gpu": resource.MustParse("1"), - }, - }, - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, job), - CompletionStatus: "Complete", - }, - }, - }, - }, - } - - return aw -} diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go deleted file mode 100644 index 64e53f8a8..000000000 --- a/test/e2e/instascale_machinepool_test.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestInstascaleMachinePool(t *testing.T) { - test := With(t) - test.T().Parallel() - - clusterType := GetClusterType(test) - if clusterType != OsdCluster { - test.T().Skipf("Skipping test as not running on an OSD cluster, resolved cluster type: %s", clusterType) - } - - namespace := test.NewTestNamespace() - - // Test configuration - cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }) - - //create OCM connection - connection := CreateOCMConnection(test) - defer connection.Close() - - // Setup batch job and AppWrapper - aw := instaScaleJobAppWrapper(test, namespace, cm) - - expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) - // check existing cluster machine pool resources - // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it - test.Expect(GetMachinePools(test, connection)). - ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) - - // apply AppWrapper to cluster - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - - // assert that AppWrapper goes to "Running" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - // look for a machine pool with a label key equal to aw.name-aw.namespace - expect to find it - test.Eventually(MachinePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) - - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - - // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it - test.Eventually(MachinePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) - -} diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go deleted file mode 100644 index 827fd4044..000000000 --- a/test/e2e/instascale_machineset_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package e2e - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestInstascaleMachineSet(t *testing.T) { - test := With(t) - test.T().Parallel() - - // skip test if not using machine sets - clusterType := GetClusterType(test) - if clusterType != OcpCluster { - test.T().Skipf("Skipping test as not running on an OCP cluster, resolved cluster type: %s", clusterType) - } - - namespace := test.NewTestNamespace() - - // Test configuration - cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }) - - // Setup batch job and AppWrapper - aw := instaScaleJobAppWrapper(test, namespace, cm) - - // look for machine set with aw name - expect to find it - test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal(aw.Name)))) - // look for machine belonging to the machine set, there should be none - test.Expect(GetMachines(test, aw.Name)).Should(BeEmpty()) - - // apply AppWrapper to cluster - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - // assert that AppWrapper goes to "Running" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - // look for machine belonging to the machine set - expect to find it - test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(HaveLen(1)) - - // assert that the AppWrapper goes to "Completed" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - - // look for machine belonging to the machine set - there should be none - test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(BeEmpty()) - -} diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go deleted file mode 100644 index 8721663d0..000000000 --- a/test/e2e/instascale_nodepool_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package e2e - -import ( - "fmt" - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestInstascaleNodepool(t *testing.T) { - - test := With(t) - test.T().Parallel() - - clusterType := GetClusterType(test) - if clusterType != HypershiftCluster { - test.T().Skipf("Skipping test as not running on an Hypershift cluster, resolved cluster type: %s", clusterType) - } - - namespace := test.NewTestNamespace() - - // Test configuration - cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }) - - //create OCM connection - connection := CreateOCMConnection(test) - defer connection.Close() - - // Setup batch job and AppWrapper - aw := instaScaleJobAppWrapper(test, namespace, cm) - - expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) - // check existing cluster resources - // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it - test.Expect(GetNodePools(test, connection)). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) - - // apply AppWrapper to cluster - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - - // assert that AppWrapper goes to "Running" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect to find it - test.Eventually(NodePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) - - // assert that the AppWrapper goes to "Completed" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - - // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it - test.Eventually(NodePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) - -} diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go deleted file mode 100644 index a642d31ca..000000000 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. -func TestMNISTPyTorchMCAD(t *testing.T) { - test := With(t) - test.T().Parallel() - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-mcad", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // Batch Job - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "job", - Image: GetPyTorchImage(), - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, - {Name: "PIP_INDEX_URL", Value: GetPipIndexURL()}, - {Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()}, - }, - Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - }, - } - - // Create an AppWrapper resource - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - DesiredAvailable: 1, - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, job), - CompletionStatus: "Complete", - }, - }, - }, - }, - } - - _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) - - test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( - Or( - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), - )) - - // Assert the job has completed successfully - test.Expect(GetAppWrapper(test, namespace, aw.Name)). - To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) -} diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go similarity index 81% rename from test/e2e/mnist_rayjob_mcad_raycluster_test.go rename to test/e2e/mnist_rayjob_raycluster_test.go index 1118079ef..46f37d7a5 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" @@ -29,13 +28,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, -// and asserts successful completion of the training job. -func TestMNISTRayJobMCADRayCluster(t *testing.T) { +// Trains the MNIST dataset as a RayJob, executed by a Ray cluster +// directly managed by Kueue, and asserts successful completion of the training job. +func TestMNISTRayJobRayCluster(t *testing.T) { test := With(t) test.T().Parallel() - // Create a namespace + // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() // MNIST training script @@ -172,44 +171,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, } - // Create an AppWrapper resource - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ray-cluster", - Namespace: namespace.Name, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - DesiredAvailable: 1, - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 2, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, rayCluster), - }, - }, - }, - }, - } - - aw, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + _, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name) - test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + test.T().Logf("Waiting for RayCluster %s/%s to be running", rayCluster.Namespace, rayCluster.Name) + test.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) rayJob := &rayv1.RayJob{ TypeMeta: metav1.TypeMeta{ diff --git a/test/odh/mcad_ray_test.go b/test/odh/mcad_ray_test.go index b72456c59..967fc54cb 100644 --- a/test/odh/mcad_ray_test.go +++ b/test/odh/mcad_ray_test.go @@ -20,11 +20,13 @@ import ( "testing" . "github.com/onsi/gomega" + gomega "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestMCADRay(t *testing.T) { @@ -44,14 +46,9 @@ func TestMCADRay(t *testing.T) { // Create RBAC, retrieve token for user with limited rights policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, { Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + APIGroups: []string{rayv1.GroupVersion.Group}, Resources: []string{"rayclusters", "rayclusters/status"}, }, { @@ -87,18 +84,17 @@ func TestMCADRay(t *testing.T) { // Create Notebook CR createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) - // Make sure the AppWrapper is created and running - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + // Make sure the RayCluster is created and running + test.Eventually(rayClusters(test, namespace), TestTimeoutLong). Should( And( HaveLen(1), - ContainElement(WithTransform(AppWrapperName, HavePrefix("mnisttest"))), - ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ContainElement(WithTransform(AppWrapperState, Equal(rayv1.Ready))), ), ) - // Make sure the AppWrapper finishes and is deleted - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + // Make sure the RayCluster finishes and is deleted + test.Eventually(rayClusters(test, namespace), TestTimeoutLong). Should(HaveLen(0)) } @@ -134,3 +130,18 @@ func readMnistPy(test Test) []byte { return ParseTemplate(test, template, props) } + +// TODO: This belongs on codeflare-common/support/ray.go +func rayClusters(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []*rayv1.RayCluster { + return func(g gomega.Gomega) []*rayv1.RayCluster { + rcs, err := t.Client().Ray().RayV1().RayClusters(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + rcsp := []*rayv1.RayCluster{} + for _, v := range rcs.Items { + rcsp = append(rcsp, &v) + } + + return rcsp + } +} diff --git a/test/odh/pytorch_mcad_test.go b/test/odh/pytorch_mcad_test.go deleted file mode 100644 index 0dd33a363..000000000 --- a/test/odh/pytorch_mcad_test.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - rbacv1 "k8s.io/api/rbac/v1" -) - -func TestMnistPyTorchMCAD(t *testing.T) { - test := With(t) - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - jupyterNotebookConfigMapFileName := "mnist_mcad_mini.ipynb" - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // MNIST MCAD Notebook - jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_mcad_mini.ipynb"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - // Needed for job.logs() - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"pods/log"}, - }, - } - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - token := CreateToken(test, namespace.Name, sa) - - // Create Notebook CR - createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) - - // Make sure the AppWrapper is created and running - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). - Should( - And( - HaveLen(1), - ContainElement(WithTransform(AppWrapperName, HavePrefix("mnistjob"))), - ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), - ), - ) - - // Make sure the AppWrapper finishes and is deleted - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). - Should(HaveLen(0)) -} diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go deleted file mode 100644 index 8fb4e65fb..000000000 --- a/test/upgrade/olm_upgrade_test.go +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - . "github.com/project-codeflare/codeflare-operator/test/e2e" -) - -var ( - namespaceName = "test-ns-olmupgrade" - appWrapperName = "mnist" - jobName = "mnist-job" -) - -func TestMNISTCreateAppWrapper(t *testing.T) { - test := With(t) - test.T().Parallel() - - // Create a namespace - namespace := CreateTestNamespaceWithName(test, namespaceName) - - // Delete namespace only if test failed - defer func() { - if t.Failed() { - DeleteTestNamespace(test, namespace) - } else { - StoreNamespaceLogs(test, namespace) - } - }() - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-mcad", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // Batch Job - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: jobName, - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "job", - Image: GetPyTorchImage(), - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, - }, - Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - Suspend: Ptr(true), - }, - } - - // Create an AppWrapper resource - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: appWrapperName, - Namespace: namespace.Name, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - DesiredAvailable: 1, - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, job), - CompletionStatus: "Complete", - }, - }, - }, - }, - } - - _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created MCAD AppWrapper %s/%s successfully", aw.Namespace, aw.Name) - - test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - -} - -func TestMNISTCheckAppWrapperStatus(t *testing.T) { - test := With(t) - test.T().Parallel() - - // get namespace - namespace := GetNamespaceWithName(test, namespaceName) - - //delete the namespace after test complete - defer DeleteTestNamespace(test, namespace) - - // Patch job to resume execution - patch := []byte(`[{"op":"replace","path":"/spec/suspend","value": false}]`) - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Patch(test.Ctx(), jobName, types.JSONPatchType, patch, metav1.PatchOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - test.T().Logf("Waiting for AppWrapper %s/%s to complete", namespace.Name, appWrapperName) - test.Eventually(AppWrapper(test, namespace, appWrapperName), TestTimeoutShort).Should( - Or( - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), - )) - - // Assert the AppWrapper has completed successfully - test.Expect(GetAppWrapper(test, namespace, appWrapperName)). - To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - -} From faff28ab80941db913f509ffdc22eb9cffbed7c8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:01:04 +0200 Subject: [PATCH 137/369] Update dependency versions for release v1.3.0 (#502) Co-authored-by: Bobbins228 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4f9cd0248..0fd608826 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.2.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.2.0) | +| CodeFlare Operator | [v1.3.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.0) | | Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | -| CodeFlare-SDK | [v0.14.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.14.0) | +| CodeFlare-SDK | [v0.15.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.0) | | InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | -| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | +| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index c4c8b45f3..120dba668 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.2.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.0 From 67122ad788019518cfba2d7e546104616756a3c1 Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Mon, 8 Apr 2024 12:45:46 +0200 Subject: [PATCH 138/369] fix(manifests): remove emptry CRD and reference (#504) --- config/crd/kustomization.yaml | 6 ------ config/default/kustomization.yaml | 1 - 2 files changed, 7 deletions(-) delete mode 100644 config/crd/kustomization.yaml diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml deleted file mode 100644 index 3b60faba5..000000000 --- a/config/crd/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: - -#+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 870c75a5c..6e926aed2 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -14,7 +14,6 @@ commonLabels: app.kubernetes.io/part-of: codeflare bases: -- ../crd - ../rbac - ../manager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. From ac0ddb5cf552a3f858966d45caa2e0af025c8429 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 8 Apr 2024 16:31:53 +0200 Subject: [PATCH 139/369] Fix OLM upgrade test --- .github/workflows/olm_tests.yaml | 44 +++----------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index b6932d893..472eda20f 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -92,10 +92,6 @@ jobs: echo Waiting for Deployment to be ready timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s - - echo Patch the CodeFlare operator ConfigMap - kubectl patch -n '${{ env.SUBSCRIPTION_NAMESPACE }}' cm codeflare-operator-config --type merge -p '{"data":{"config.yaml":"mcad:\n enabled: true"}}' - env: CATALOG_SOURCE_NAME: "codeflare-olm-test" CATALOG_SOURCE_NAMESPACE: "olm" @@ -107,10 +103,6 @@ jobs: CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version) echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV - - name: Deploy CodeFlare stack - run: | - make setup-e2e - - name: Build operator and catalog image run: | make image-push @@ -126,13 +118,6 @@ jobs: BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" - - name: Run OLM Upgrade e2e AppWrapper creation test - run: | - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMNISTCreateAppWrapper -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - - name: Update Operator to the built version run: | ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) @@ -162,38 +147,18 @@ jobs: SUBSCRIPTION_NAME: "codeflare-operator" SUBSCRIPTION_NAMESPACE: "openshift-operators" - - name: Run OLM Upgrade e2e Appwrapper Job status test to monitor training - run: | - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMNISTCheckAppWrapperStatus -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - - - name: Run e2e tests against built operator - run: | - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - - set -euo pipefail - go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing CodeFlare operator logs" - kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - - - name: Print KubeRay operator logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing KubeRay operator logs" - kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + mkdir logs + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee logs/codeflare-operator.log - name: Export all KinD pod logs uses: ./common/github-actions/kind-export-logs if: always() && steps.deploy.outcome == 'success' with: - output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} + output-directory: logs - name: Upload logs uses: actions/upload-artifact@v4 @@ -201,5 +166,4 @@ jobs: with: name: logs retention-days: 10 - path: | - ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log + path: logs/**/*.log From 5cc6fd962421184251972da137bfebdfd80eb2d3 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Fri, 12 Apr 2024 13:23:57 +0530 Subject: [PATCH 140/369] Update kuberay version to latest --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e1b77a8e1..fa92d560c 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.0.0 +KUBERAY_VERSION ?= v1.1.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 From 58b23ee3517fa029d3e81012396ca376ec8d4fa5 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 11 Apr 2024 16:09:41 +0200 Subject: [PATCH 141/369] Add missing roles to create ClusterRoleBinding for CodeFlare operator Cleanup after MCAD removal --- ...eflare-operator.clusterserviceversion.yaml | 18 +- config/rbac/admin_role.yaml | 22 -- config/rbac/editor_role.yaml | 24 -- config/rbac/instascale_role.yaml | 43 ---- config/rbac/instascale_role_binding.yaml | 12 - config/rbac/kustomization.yaml | 8 - .../rbac/mcad-controller-ray-clusterrole.yaml | 43 ---- ...cad-controller-ray-clusterrolebinding.yaml | 12 - config/rbac/mcad_manager_role.yaml | 223 ------------------ config/rbac/mcad_manager_role_binding.yaml | 12 - config/rbac/role.yaml | 31 +++ pkg/controllers/raycluster_controller.go | 11 +- 12 files changed, 39 insertions(+), 420 deletions(-) delete mode 100644 config/rbac/admin_role.yaml delete mode 100644 config/rbac/editor_role.yaml delete mode 100644 config/rbac/instascale_role.yaml delete mode 100644 config/rbac/instascale_role_binding.yaml delete mode 100644 config/rbac/mcad-controller-ray-clusterrole.yaml delete mode 100644 config/rbac/mcad-controller-ray-clusterrolebinding.yaml delete mode 100644 config/rbac/mcad_manager_role.yaml delete mode 100644 config/rbac/mcad_manager_role_binding.yaml diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 5531e61a8..04a0b301d 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -50,23 +50,7 @@ metadata: namespace: placeholder spec: apiservicedefinitions: {} - customresourcedefinitions: - owned: - - description: AppWrapper is the Schema for the AppWrapper API - displayName: AppWrapper - kind: AppWrappers - name: appwrappers.workload.codeflare.dev - version: v1beta1 - - description: SchedulingSpec is the Schema for the SchedulingSpec API - displayName: SchedulingSpec - kind: SchedulingSpecs - name: schedulingspecs.workload.codeflare.dev - version: v1beta1 - - description: QuotaSubtree is the Schema for the QuotaSubtree API - displayName: QuotaSubtree - kind: QuotaSubtrees - name: quotasubtrees.quota.codeflare.dev - version: v1 + customresourcedefinitions: {} description: CodeFlare allows you to scale complex pipelines anywhere displayName: CodeFlare Operator icon: diff --git a/config/rbac/admin_role.yaml b/config/rbac/admin_role.yaml deleted file mode 100644 index b4f4728ca..000000000 --- a/config/rbac/admin_role.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: clusterrole-admin - labels: - rbac.authorization.kubernetes.io/aggregate-to-admin: "true" -rules: -- apiGroups: - - quota.codeflare.dev - resources: - - quotasubtrees - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/config/rbac/editor_role.yaml b/config/rbac/editor_role.yaml deleted file mode 100644 index aa341297d..000000000 --- a/config/rbac/editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: clusterrole-edit - labels: - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" -rules: -- apiGroups: - - workload.codeflare.dev - resources: - - schedulingspecs - - appwrappers - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/config/rbac/instascale_role.yaml b/config/rbac/instascale_role.yaml deleted file mode 100644 index dec6b720a..000000000 --- a/config/rbac/instascale_role.yaml +++ /dev/null @@ -1,43 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: instascale-role -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - update -- apiGroups: - - "" - resources: - - secrets - verbs: - - get -- apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list - - watch -- apiGroups: - - machine.openshift.io - resources: - - machines - - machinesets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch diff --git a/config/rbac/instascale_role_binding.yaml b/config/rbac/instascale_role_binding.yaml deleted file mode 100644 index 00a7d43f7..000000000 --- a/config/rbac/instascale_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: instascale-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: instascale-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 12d4bb247..166fe7986 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -4,16 +4,8 @@ resources: # if your manager will use a service account that exists at # runtime. Be sure to update RoleBinding and ClusterRoleBinding # subjects if changing service account names. -- admin_role.yaml -- editor_role.yaml - service_account.yaml -- mcad_manager_role.yaml -- mcad_manager_role_binding.yaml - role.yaml - role_binding.yaml -- instascale_role.yaml -- instascale_role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -- mcad-controller-ray-clusterrolebinding.yaml -- mcad-controller-ray-clusterrole.yaml diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml deleted file mode 100644 index 18e3d98f3..000000000 --- a/config/rbac/mcad-controller-ray-clusterrole.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: mcad-controller-ray-clusterrole -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - - rayclusters/finalizers - - rayclusters/status - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - route.openshift.io - resources: - - routes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml deleted file mode 100644 index da9e8c023..000000000 --- a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: mcad-controller-ray-clusterrolebinding -subjects: - - kind: ServiceAccount - name: controller-manager - namespace: system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: mcad-controller-ray-clusterrole diff --git a/config/rbac/mcad_manager_role.yaml b/config/rbac/mcad_manager_role.yaml deleted file mode 100644 index b414b8e23..000000000 --- a/config/rbac/mcad_manager_role.yaml +++ /dev/null @@ -1,223 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manual-manager-role -rules: -- apiGroups: - - '*' - resources: - - deployments - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - replicasets - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list -- apiGroups: - - coordination.k8s.io - resources: - - kube-scheduler - - leases - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - bindings - - pods/binding - verbs: - - create -- apiGroups: - - "" - resources: - - configmaps - - nodes - - persistentvolumeclaims - - persistentvolumes - - secrets - - serviceaccounts - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - endpoints - - kube-scheduler - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update -- apiGroups: - - "" - resources: - - kube-scheduler - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/status - verbs: - - patch - - update -- apiGroups: - - "" - resources: - - replicationcontrollers - verbs: - - get - - list - - watch -- apiGroups: - - events.k8s.io - resources: - - events - - kube-scheduler - verbs: - - create - - patch - - update -- apiGroups: - - machine.openshift.io - resources: - - '*' - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - scheduling.sigs.k8s.io - resources: - - podgroups - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - - csinodes - - csistoragecapacities - verbs: - - get - - list - - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers - - appwrappers/finalizers - - appwrappers/status - - schedulingspecs - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch -- apiGroups: - - quota.codeflare.dev - resources: - - quotasubtrees - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/config/rbac/mcad_manager_role_binding.yaml b/config/rbac/mcad_manager_role_binding.yaml deleted file mode 100644 index af6c74ae0..000000000 --- a/config/rbac/mcad_manager_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manual-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manual-manager-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e5a2ca9f4..70a6a861d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,18 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - "" resources: @@ -19,17 +31,31 @@ rules: resources: - serviceaccounts verbs: + - create - delete - get - patch + - update - apiGroups: - "" resources: - services verbs: + - create + - delete + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create - delete - get - patch + - update - apiGroups: - ray.io resources: @@ -61,14 +87,19 @@ rules: resources: - clusterrolebindings verbs: + - create - delete - get - patch + - update - apiGroups: - route.openshift.io resources: - routes + - routes/custom-host verbs: + - create - delete - get - patch + - update diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 23104c2ab..a098f9b27 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -73,11 +73,14 @@ var ( // +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes;routes/custom-host,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get -// +kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get -// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=patch;delete;get +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create; +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create; // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From 61d7a48861afa2ccf20b4196e45c26dfc7c50b9f Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 11 Apr 2024 18:41:10 +0100 Subject: [PATCH 142/369] RC Controller - Avoid multiple calls to DiscoveryAPI --- main.go | 6 +++--- pkg/controllers/raycluster_controller.go | 25 ++++++++++++++---------- pkg/controllers/support.go | 25 +++++++++++++++--------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/main.go b/main.go index 752f146f2..8a129e806 100644 --- a/main.go +++ b/main.go @@ -146,9 +146,9 @@ func main() { }) exitOnError(err, "unable to start manager") - v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) - if v { - rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg} + ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + if ok { + rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index a098f9b27..769a5e099 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -48,11 +48,13 @@ import ( // RayClusterReconciler reconciles a RayCluster object type RayClusterReconciler struct { client.Client - kubeClient *kubernetes.Clientset - routeClient *routev1client.RouteV1Client - Scheme *runtime.Scheme - CookieSalt string - Config *config.CodeFlareOperatorConfiguration + kubeClient *kubernetes.Clientset + routeClient *routev1client.RouteV1Client + Scheme *runtime.Scheme + CookieSalt string + Config *config.KubeRayConfiguration + IsOpenShift bool + IsOpenShiftInitialized bool } const ( @@ -105,8 +107,10 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } isLocalInteractive := annotationBoolVal(ctx, &cluster, "sdk.codeflare.dev/local_interactive", false) - ingressDomain := "" // FIX - CFO will retrieve it. - isOpenShift, ingressHost := getClusterType(ctx, r.kubeClient, &cluster, ingressDomain) + if !r.IsOpenShiftInitialized { + r.IsOpenShift = isOpenShift(ctx, r.kubeClient, &cluster) + r.IsOpenShiftInitialized = true + } if cluster.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { @@ -141,7 +145,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && isOpenShift { + if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && r.IsOpenShift { logger.Info("Creating OAuth Objects") _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { @@ -182,9 +186,10 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !isOpenShift { + } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { + ingressDomain := "" logger.Info("Creating Dashboard Ingress") - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, ingressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, getIngressHost(ctx, r.kubeClient, &cluster, ingressDomain)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 348cd03de..44b21bda9 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -126,42 +126,49 @@ func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) // Check where we are running. We are trying to distinguish here whether // this is vanilla kubernetes cluster or Openshift -func getClusterType(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) (bool, string) { +func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster) bool { // The discovery package is used to discover APIs supported by a Kubernetes API server. logger := ctrl.LoggerFrom(ctx) config, err := ctrl.GetConfig() if err != nil && config == nil { logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes") - return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + return false } dclient, err := getDiscoveryClient(config) if err != nil && dclient == nil { logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes") - return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + return false } apiGroupList, err := dclient.ServerGroups() if err != nil { logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") - return false, "" + return false } for i := 0; i < len(apiGroupList.Groups); i++ { if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { logger.Info("We detected being on OpenShift!") - return true, "" + return true } } + logger.Info("We detected being on Vanilla Kubernetes!") + return false +} + +// getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. +func getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) string { + logger := ctrl.LoggerFrom(ctx) onKind, _ := isOnKindCluster(clientset) if onKind && ingressDomain == "" { logger.Info("We detected being on a KinD cluster!") - return false, "kind" + return "kind" } logger.Info("We detected being on Vanilla Kubernetes!") - return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + return fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) } func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { - if r.Config != nil && r.Config.KubeRay != nil && r.Config.KubeRay.RayDashboardOAuthEnabled != nil { - return *r.Config.KubeRay.RayDashboardOAuthEnabled + if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { + return *r.Config.RayDashboardOAuthEnabled } return true } From da57f8c3b29f50322ff9867853dc15706bd6d9d0 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Fri, 12 Apr 2024 11:33:27 +0100 Subject: [PATCH 143/369] Add TODO notifier on ingressDomain --- pkg/controllers/raycluster_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 769a5e099..db13f0854 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -187,7 +187,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { - ingressDomain := "" + ingressDomain := "" // TODO: ingressDomain should be retrieved by the CFO and used here to fix local_interactive. Jira: https://issues.redhat.com/browse/RHOAIENG-5330 logger.Info("Creating Dashboard Ingress") _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, getIngressHost(ctx, r.kubeClient, &cluster, ingressDomain)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { From 75a2d234e0f54ff241eca245fe86d8afebe79f74 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 15 Apr 2024 16:45:11 +0200 Subject: [PATCH 144/369] Add a ConfigMap for e2e tests --- .github/workflows/e2e_tests.yaml | 1 - config/e2e/config.yaml | 8 ++++++++ config/e2e/kustomization.yaml | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 config/e2e/config.yaml diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 2e8fdbc1f..0fc1ec88b 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -60,7 +60,6 @@ jobs: run: | echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator - sed -i 's/RayDashboardOAuthEnabled: pointer.Bool(true)/RayDashboardOAuthEnabled: pointer.Bool(false)/' main.go make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml new file mode 100644 index 000000000..83320bb0c --- /dev/null +++ b/config/e2e/config.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: codeflare-operator-config +data: + config.yaml: | + kuberay: + rayDashboardOAuthEnabled: false diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index e06123b2a..772370da6 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,6 +1,7 @@ namespace: openshift-operators bases: +- config.yaml - ../default patches: From 689611140bbe12f2576216472d6975d42da6e87d Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 15 Apr 2024 11:46:43 +0100 Subject: [PATCH 145/369] Rayclient Route to resolve its own ingress domain, and enable local_interactive by default --- pkg/controllers/raycluster_controller.go | 15 ++++++--------- pkg/controllers/support.go | 16 ---------------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index db13f0854..f60845eb5 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -106,7 +106,6 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } - isLocalInteractive := annotationBoolVal(ctx, &cluster, "sdk.codeflare.dev/local_interactive", false) if !r.IsOpenShiftInitialized { r.IsOpenShift = isOpenShift(ctx, r.kubeClient, &cluster) r.IsOpenShiftInitialized = true @@ -177,13 +176,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{RequeueAfter: requeueTime}, err } - if isLocalInteractive { - logger.Info("Creating RayClient Route") - _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update RayClient Route") - return ctrl.Result{RequeueAfter: requeueTime}, err - } + logger.Info("Creating RayClient Route") + _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Route") + return ctrl.Result{RequeueAfter: requeueTime}, err } } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { @@ -195,7 +192,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - if isLocalInteractive && ingressDomain != "" { + if ingressDomain != "" { logger.Info("Creating RayClient Ingress") _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 44b21bda9..3749fdf03 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "strconv" "strings" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -30,7 +29,6 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(routeapply.RouteSpec(). - WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace). WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)). WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))). WithTLS(routeapply.TLSConfig().WithTermination("passthrough")), @@ -172,17 +170,3 @@ func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { } return true } - -func annotationBoolVal(ctx context.Context, cluster *rayv1.RayCluster, annotation string, defaultValue bool) bool { - logger := ctrl.LoggerFrom(ctx) - val, exists := cluster.ObjectMeta.Annotations[annotation] - if !exists || val == "" { - return defaultValue - } - boolVal, err := strconv.ParseBool(val) - if err != nil { - logger.Error(err, "Could not convert annotation value to bool", "annotation", annotation, "value", val) - return defaultValue - } - return boolVal -} From 26be493635fff84b651c1fd1258d12040413a7ca Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 15 Apr 2024 16:09:39 +0100 Subject: [PATCH 146/369] Add ingressDomain to configmap to work with local and workflow e2e tests --- config/e2e/config.yaml | 1 + main.go | 1 + pkg/config/config.go | 2 ++ pkg/controllers/raycluster_controller.go | 18 ++++++------ pkg/controllers/support.go | 36 ++++++++---------------- 5 files changed, 24 insertions(+), 34 deletions(-) diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 83320bb0c..6b8f55a0c 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -6,3 +6,4 @@ data: config.yaml: | kuberay: rayDashboardOAuthEnabled: false + ingressDomain: "kind" diff --git a/main.go b/main.go index 8a129e806..58c6c73cb 100644 --- a/main.go +++ b/main.go @@ -115,6 +115,7 @@ func main() { }, KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: pointer.Bool(true), + IngressDomain: "fake.domain", }, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 124bb0ebb..c71678f11 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -33,6 +33,8 @@ type CodeFlareOperatorConfiguration struct { type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` + + IngressDomain string `json:"ingressDomain,omitempty"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index f60845eb5..9fe6dfeb0 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -184,21 +184,21 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { - ingressDomain := "" // TODO: ingressDomain should be retrieved by the CFO and used here to fix local_interactive. Jira: https://issues.redhat.com/browse/RHOAIENG-5330 + logger.Info("We detected being on Vanilla Kubernetes!") logger.Info("Creating Dashboard Ingress") - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, getIngressHost(ctx, r.kubeClient, &cluster, ingressDomain)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + dashboardName := dashboardNameFromCluster(&cluster) + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - if ingressDomain != "" { - logger.Info("Creating RayClient Ingress") - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update RayClient Ingress") - return ctrl.Result{RequeueAfter: requeueTime}, err - } + logger.Info("Creating RayClient Ingress") + rayClientName := rayClientNameFromCluster(&cluster) + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Ingress") + return ctrl.Result{RequeueAfter: requeueTime}, err } } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 3749fdf03..495639454 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -8,7 +8,6 @@ import ( rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" v1 "k8s.io/client-go/applyconfigurations/meta/v1" @@ -39,7 +38,7 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf } // Create an Ingress object for the RayCluster -func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *networkingv1ac.IngressApplyConfiguration { +func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithAnnotations(map[string]string{ @@ -55,7 +54,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *n WithSpec(networkingv1ac.IngressSpec(). WithIngressClassName("nginx"). WithRules(networkingv1ac.IngressRule(). - WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingressDomain). + WithHost(ingressHost). WithHTTP(networkingv1ac.HTTPIngressRuleValue(). WithPaths(networkingv1ac.HTTPIngressPath(). WithPath("/"). @@ -85,7 +84,7 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo WithUID(types.UID(cluster.UID))). WithSpec(networkingv1ac.IngressSpec(). WithRules(networkingv1ac.IngressRule(). - WithHost(ingressHost). // KinD hostname or ingressDomain + WithHost(ingressHost). // Full Hostname WithHTTP(networkingv1ac.HTTPIngressRuleValue(). WithPaths(networkingv1ac.HTTPIngressPath(). WithPath("/"). @@ -104,19 +103,6 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ) } -// isOnKindCluster checks if the current cluster is a KinD cluster. -// It searches for a node with a label commonly used by KinD clusters. -func isOnKindCluster(clientset *kubernetes.Clientset) (bool, error) { - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: "kubernetes.io/hostname=kind-control-plane", - }) - if err != nil { - return false, err - } - // If we find one or more nodes with the label, assume it's a KinD cluster. - return len(nodes.Items) > 0, nil -} - // getDiscoveryClient returns a discovery client for the current reconciler func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) { return discovery.NewDiscoveryClientForConfig(config) @@ -153,15 +139,15 @@ func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster * } // getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. -func getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) string { - logger := ctrl.LoggerFrom(ctx) - onKind, _ := isOnKindCluster(clientset) - if onKind && ingressDomain == "" { - logger.Info("We detected being on a KinD cluster!") - return "kind" +func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) string { + ingressDomain := "fake.domain" + if r.Config != nil && r.Config.IngressDomain != "" { + ingressDomain = r.Config.IngressDomain } - logger.Info("We detected being on Vanilla Kubernetes!") - return fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + if ingressDomain == "kind" { + return ingressDomain + } + return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain) } func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { From 61c7bca73efd9a37cd647fdc2159e437355855ff Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 15 Apr 2024 17:57:55 +0100 Subject: [PATCH 147/369] Return error message if IngressDomain configuration not set --- main.go | 2 +- pkg/config/config.go | 2 +- pkg/controllers/raycluster_controller.go | 12 ++++++++++-- pkg/controllers/support.go | 10 ++++++---- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 58c6c73cb..09ebe2599 100644 --- a/main.go +++ b/main.go @@ -115,7 +115,7 @@ func main() { }, KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: pointer.Bool(true), - IngressDomain: "fake.domain", + IngressDomain: "", }, } diff --git a/pkg/config/config.go b/pkg/config/config.go index c71678f11..08e2579b9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -34,7 +34,7 @@ type CodeFlareOperatorConfiguration struct { type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` - IngressDomain string `json:"ingressDomain,omitempty"` + IngressDomain string `json:"ingressDomain"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 9fe6dfeb0..4f60545f7 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -187,7 +187,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Info("We detected being on Vanilla Kubernetes!") logger.Info("Creating Dashboard Ingress") dashboardName := dashboardNameFromCluster(&cluster) - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + dashboardIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName) + if err != nil { + return ctrl.Result{RequeueAfter: requeueTime}, err + } + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, dashboardIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) @@ -195,7 +199,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } logger.Info("Creating RayClient Ingress") rayClientName := rayClientNameFromCluster(&cluster) - _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + rayClientIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName) + if err != nil { + return ctrl.Result{RequeueAfter: requeueTime}, err + } + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, rayClientIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update RayClient Ingress") return ctrl.Result{RequeueAfter: requeueTime}, err diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 495639454..7871e66a8 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -139,15 +139,17 @@ func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster * } // getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. -func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) string { - ingressDomain := "fake.domain" +func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { + ingressDomain := "" if r.Config != nil && r.Config.IngressDomain != "" { ingressDomain = r.Config.IngressDomain + } else { + return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") } if ingressDomain == "kind" { - return ingressDomain + return ingressDomain, nil } - return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain) + return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil } func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { From bc16f94bf2006335cdce7c10546ac36d3b8b6b70 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 15 Apr 2024 20:50:05 +0200 Subject: [PATCH 148/369] Adjust Kuberay dependency in the Makefile --- .github/workflows/tag-and-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 870ac35c1..a60dceb7b 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -85,6 +85,10 @@ jobs: sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md + - name: Adjust Kuberay dependency in the code + run: | + sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile + - name: Update image version in params.env run: | VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env From 8b0902e1426cc6af01e1bb8eaf780f21d8b3e82d Mon Sep 17 00:00:00 2001 From: sutaakar Date: Mon, 15 Apr 2024 19:13:35 +0000 Subject: [PATCH 149/369] Update dependency versions for release v1.3.1 --- Makefile | 2 +- README.md | 6 +++--- config/manager/params.env | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index fa92d560c..e1b77a8e1 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.1.0 +KUBERAY_VERSION ?= v1.0.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/README.md b/README.md index 0fd608826..ef2b44d5a 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.3.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.0) | +| CodeFlare Operator | [v1.3.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.1) | | Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | -| CodeFlare-SDK | [v0.15.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.0) | +| CodeFlare-SDK | [v0.15.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.1) | | InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | -| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | +| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 120dba668..36832f916 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.1 From c37c4b232b3ef68ae4cc23baeb4a9bf55b775d60 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 15 Apr 2024 21:08:06 +0200 Subject: [PATCH 150/369] E2E tests: Deploy Kuberay before starting CodeFlare operator --- .github/workflows/e2e_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0fc1ec88b..30ef1ea99 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -58,15 +58,15 @@ jobs: - name: Deploy CodeFlare stack id: deploy run: | + echo Setting up CodeFlare stack + make setup-e2e + echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager - echo Setting up CodeFlare stack - make setup-e2e - - name: Run e2e tests run: | export CODEFLARE_TEST_TIMEOUT_SHORT=1m From f41244597acbc1dc9e2c274fbf5615857b2387b5 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 15 Apr 2024 23:24:06 +0200 Subject: [PATCH 151/369] Use Dnsmasq to provide DNS for KinD --- pkg/controllers/support.go | 3 -- test/e2e/mnist_rayjob_raycluster_test.go | 45 +++++++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 7871e66a8..344e6215f 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -146,9 +146,6 @@ func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *ku } else { return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") } - if ingressDomain == "kind" { - return ingressDomain, nil - } return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil } diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 46f37d7a5..b3cee102c 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -17,6 +17,9 @@ limitations under the License. package e2e import ( + "crypto/tls" + "net/http" + "net/url" "testing" . "github.com/onsi/gomega" @@ -221,7 +224,7 @@ func TestMNISTRayJobRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) - rayDashboardURL := ExposeService(test, "ray-dashboard", namespace.Name, "raycluster-head-svc", "dashboard") + rayDashboardURL := getRayDashboardURL(test, rayCluster.Namespace, rayCluster.Name) test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) @@ -241,3 +244,43 @@ func TestMNISTRayJobRayCluster(t *testing.T) { test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) } + +func getRayDashboardURL(test Test, namespace, rayClusterName string) url.URL { + dashboardName := "ray-dashboard-" + rayClusterName + + if IsOpenShift(test) { + route := GetRoute(test, namespace, dashboardName) + hostname := route.Status.Ingress[0].Host + + // Wait for expected HTTP code + test.T().Logf("Waiting for Route %s/%s to be available", route.Namespace, route.Name) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + test.Eventually(func() (int, error) { + resp, err := client.Get("https://" + hostname) + if err != nil { + return -1, err + } + return resp.StatusCode, nil + }, TestTimeoutShort).Should(Not(Equal(503))) + + return url.URL{ + Scheme: "https", + Host: hostname, + } + } + + ingress := GetIngress(test, namespace, dashboardName) + + test.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) + test.Eventually(Ingress(test, ingress.Namespace, ingress.Name), TestTimeoutShort). + Should(WithTransform(LoadBalancerIngresses, HaveLen(1))) + + return url.URL{ + Scheme: "http", + Host: ingress.Spec.Rules[0].Host, + } +} From 3f1166cd29ca9f53682bd02978a14d3834608933 Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Tue, 16 Apr 2024 11:10:26 +0200 Subject: [PATCH 152/369] fix(manifests): missing namespace in CM stack-config Signed-off-by: Wen Zhou --- config/manager/params.env | 1 + 1 file changed, 1 insertion(+) diff --git a/config/manager/params.env b/config/manager/params.env index 36832f916..8843b5e31 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1,2 @@ codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.1 +namespace=opendatahub From 8b3b27c9ea4989a089de953aa2030cb165393d53 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 09:30:32 +0200 Subject: [PATCH 153/369] Remove owner reference for ClusterRoleBinding to RayCluster --- pkg/controllers/raycluster_controller.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 4f60545f7..551367e49 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -232,9 +232,6 @@ func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.Cluste WithAPIGroup("rbac.authorization.k8s.io"). WithKind("ClusterRole"). WithName("system:auth-delegator"), - ). - WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } From 3fa247f1c0c0f94c1f5e8e4e388c235a65bacc1d Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 12:13:04 +0100 Subject: [PATCH 154/369] add: mutating webhook --- PROJECT | 4 + config/certmanager/certificate.yaml | 3 + config/certmanager/kustomization.yaml | 5 + config/certmanager/kustomizeconfig.yaml | 16 +++ config/default/kustomization.yaml | 11 +- config/default/manager_webhook_patch.yaml | 23 ++++ config/default/webhookcainjection_patch.yaml | 15 +++ config/webhook/kustomization.yaml | 6 + config/webhook/kustomizeconfig.yaml | 25 ++++ config/webhook/manifests.yaml | 27 ++++ config/webhook/service.yaml | 22 ++++ main.go | 3 + pkg/controllers/raycluster_webhook.go | 122 +++++++++++++++++++ 13 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 config/certmanager/certificate.yaml create mode 100644 config/certmanager/kustomization.yaml create mode 100644 config/certmanager/kustomizeconfig.yaml create mode 100644 config/default/manager_webhook_patch.yaml create mode 100644 config/default/webhookcainjection_patch.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/kustomizeconfig.yaml create mode 100644 config/webhook/manifests.yaml create mode 100644 config/webhook/service.yaml create mode 100644 pkg/controllers/raycluster_webhook.go diff --git a/PROJECT b/PROJECT index cb6903086..3e821aa02 100644 --- a/PROJECT +++ b/PROJECT @@ -15,5 +15,9 @@ resources: domain: codeflare.dev group: ray kind: RayCluster + path: github.com/project-codeflare/codeflare-operator/pkg/controllers version: v1 + webhooks: + defaulting: true + webhookVersion: v1 version: "3" diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 000000000..451153488 --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,3 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 000000000..bebea5a59 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 000000000..e631f7773 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 6e926aed2..fccc7164e 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -14,10 +14,15 @@ commonLabels: app.kubernetes.io/part-of: codeflare bases: -- ../rbac -- ../manager + - ../rbac + - ../manager + - ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: -- metrics_service.yaml + - metrics_service.yaml + +patches: +- path: manager_webhook_patch.yaml +- path: webhookcainjection_patch.yaml diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 000000000..5d6b541dc --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: codeflare-operator-raycluster-webhook-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 000000000..2770f6a9c --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: codeflare-operator + app.kubernetes.io/part-of: codeflare-operator + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + service.beta.openshift.io/inject-cabundle: "true" + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 000000000..9cf26134e --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 000000000..25e21e3c9 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 000000000..faf91696b --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-ray-io-v1-raycluster + failurePolicy: Fail + name: mraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 000000000..9caf0911a --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: service + app.kubernetes.io/part-of: codeflare-operator + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: codeflare-operator + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: openshift-operators + annotations: + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/part-of: codeflare + app.kubernetes.io/name: codeflare-operator diff --git a/main.go b/main.go index 09ebe2599..40f916a50 100644 --- a/main.go +++ b/main.go @@ -147,6 +147,9 @@ func main() { }) exitOnError(err, "unable to start manager") + rayClusterDefaulter := &controllers.RayClusterDefaulter{} + exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook") + ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if ok { rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay} diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go new file mode 100644 index 000000000..5cabf2fae --- /dev/null +++ b/pkg/controllers/raycluster_webhook.go @@ -0,0 +1,122 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// log is for logging in this package. +var rayclusterlog = logf.Log.WithName("raycluster-resource") + +func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&rayv1.RayCluster{}). + WithDefaulter(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 + +type RayClusterDefaulter struct{} + +var _ webhook.CustomDefaulter = &RayClusterDefaulter{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { + raycluster := obj.(*rayv1.RayCluster) + + rayclusterlog.Info("default", "name", raycluster.Name) + // Check and add OAuth proxy if it does not exist. + alreadyExists := false + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.Info("OAuth sidecar already exists, no patch needed") + alreadyExists = true + break // exits the for loop + } + } + + if !alreadyExists { + rayclusterlog.Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + "--added-label=True", + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + } + + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, + }, + } + + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + } + } + return nil +} From 2dde5a99d30fbc7e6ac5b6896486b10e04400139 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 14:32:42 +0100 Subject: [PATCH 155/369] remove: added label --- pkg/controllers/raycluster_webhook.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 5cabf2fae..63581907c 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -75,7 +75,6 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e "--tls-key=/etc/tls/private/tls.key", "--cookie-secret=$(COOKIE_SECRET)", "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - "--added-label=True", }, Env: []corev1.EnvVar{ { From 4afea61ea3d42b57e3d193e5a2d5537812271e5e Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 16:31:05 +0100 Subject: [PATCH 156/369] add: address feedback on pr --- config/certmanager/certificate.yaml | 3 --- config/certmanager/kustomization.yaml | 5 ----- config/certmanager/kustomizeconfig.yaml | 16 ---------------- config/default/webhookcainjection_patch.yaml | 10 +--------- config/webhook/service.yaml | 11 ++--------- pkg/controllers/raycluster_webhook.go | 4 +++- 6 files changed, 6 insertions(+), 43 deletions(-) delete mode 100644 config/certmanager/certificate.yaml delete mode 100644 config/certmanager/kustomization.yaml delete mode 100644 config/certmanager/kustomizeconfig.yaml diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml deleted file mode 100644 index 451153488..000000000 --- a/config/certmanager/certificate.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a59..000000000 --- a/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index e631f7773..000000000 --- a/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml index 2770f6a9c..39db86347 100644 --- a/config/default/webhookcainjection_patch.yaml +++ b/config/default/webhookcainjection_patch.yaml @@ -2,14 +2,6 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - labels: - app.kubernetes.io/name: mutatingwebhookconfiguration - app.kubernetes.io/instance: mutating-webhook-configuration - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/managed-by: kustomize name: mutating-webhook-configuration annotations: - service.beta.openshift.io/inject-cabundle: "true" - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert + service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index 9caf0911a..ada0e3281 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -1,22 +1,15 @@ apiVersion: v1 kind: Service metadata: - labels: - app.kubernetes.io/name: service - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/instance: webhook-service - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/managed-by: kustomize name: webhook-service namespace: openshift-operators annotations: - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert spec: ports: - port: 443 protocol: TCP targetPort: 9443 selector: - app.kubernetes.io/part-of: codeflare + app.kubernetes.io/part-of: codeflare app.kubernetes.io/name: codeflare-operator diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 63581907c..bce533222 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -18,7 +18,9 @@ package controllers import ( "context" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -32,7 +34,7 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(r). + WithDefaulter(&RayClusterDefaulter{}). Complete() } From 51057f4238da323909ba25efb961d2261336772a Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Wed, 17 Apr 2024 11:26:03 +0100 Subject: [PATCH 157/369] add: check for isRayDashboardOAuthEnabledWebhook before applying patch --- pkg/controllers/raycluster_webhook.go | 128 ++++++++++++++------------ pkg/controllers/support.go | 7 ++ 2 files changed, 76 insertions(+), 59 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index bce533222..3406cd9b6 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -26,6 +26,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/project-codeflare/codeflare-operator/pkg/config" ) // log is for logging in this package. @@ -34,13 +36,19 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&RayClusterDefaulter{}). + WithDefaulter(&RayClusterDefaulter{ + Config: r.Config, + rayDashboardOauthEnabled: r.isRayDashboardOAuthEnabledWebhook(), + }). Complete() } //+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 -type RayClusterDefaulter struct{} +type RayClusterDefaulter struct { + Config *config.KubeRayConfiguration + rayDashboardOauthEnabled bool +} var _ webhook.CustomDefaulter = &RayClusterDefaulter{} @@ -48,75 +56,77 @@ var _ webhook.CustomDefaulter = &RayClusterDefaulter{} func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) - rayclusterlog.Info("default", "name", raycluster.Name) - // Check and add OAuth proxy if it does not exist. - alreadyExists := false - for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { - if container.Name == "oauth-proxy" { - rayclusterlog.Info("OAuth sidecar already exists, no patch needed") - alreadyExists = true - break // exits the for loop + if r.rayDashboardOauthEnabled { + rayclusterlog.Info("default", "name", raycluster.Name) + // Check and add OAuth proxy if it does not exist. + alreadyExists := false + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.Info("OAuth sidecar already exists, no patch needed") + alreadyExists = true + break // exits the for loop + } } - } - if !alreadyExists { - rayclusterlog.Info("Adding OAuth sidecar container") - // definition of the new container - newOAuthSidecar := corev1.Container{ - Name: "oauth-proxy", - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", - Ports: []corev1.ContainerPort{ - {ContainerPort: 8443, Name: "oauth-proxy"}, - }, - Args: []string{ - "--https-address=:8443", - "--provider=openshift", - "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", - "--upstream=http://localhost:8265", - "--tls-cert=/etc/tls/private/tls.crt", - "--tls-key=/etc/tls/private/tls.key", - "--cookie-secret=$(COOKIE_SECRET)", - "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - }, - Env: []corev1.EnvVar{ - { - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", + if !alreadyExists { + rayclusterlog.Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", }, - Key: "cookie_secret", }, }, }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "proxy-tls-secret", - MountPath: "/etc/tls/private", - ReadOnly: true, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, }, - }, - } + } - // Adding the new OAuth sidecar container - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - tlsSecretVolume := corev1.Volume{ - Name: "proxy-tls-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: raycluster.Name + "-proxy-tls-secret", + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, }, - }, - } + } - raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) - // Ensure the service account is set - if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + } } } return nil diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 344e6215f..698bd8da3 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -155,3 +155,10 @@ func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { } return true } + +func (r *RayClusterDefaulter) isRayDashboardOAuthEnabledWebhook() bool { + if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { + return *r.Config.RayDashboardOAuthEnabled + } + return true +} From 77accff191dd2574e020cf5a4f8043b31a9c0ef1 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Wed, 17 Apr 2024 12:47:55 +0100 Subject: [PATCH 158/369] update: placement of cookie secret in the mutating webhook for rayclusters --- pkg/controllers/raycluster_webhook.go | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 3406cd9b6..d63616e13 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -87,19 +87,6 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e "--cookie-secret=$(COOKIE_SECRET)", "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", }, - Env: []corev1.EnvVar{ - { - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", - }, - Key: "cookie_secret", - }, - }, - }, - }, VolumeMounts: []corev1.VolumeMount{ { Name: "proxy-tls-secret", @@ -112,6 +99,23 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e // Adding the new OAuth sidecar container raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + cookieSecret := corev1.EnvVar{ + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + } + + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, + cookieSecret, + ) + tlsSecretVolume := corev1.Volume{ Name: "proxy-tls-secret", VolumeSource: corev1.VolumeSource{ From 06efef73f1377d1b6898dabcb86bbdeeda09c256 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 16:47:51 +0200 Subject: [PATCH 159/369] Refactor RayCluster mutating webhook --- PROJECT | 2 +- config/default/kustomization.yaml | 5 - config/openshift/kustomization.yaml | 22 +++ .../manager_webhook_patch.yaml | 0 .../webhookcainjection_patch.yaml | 0 main.go | 3 +- pkg/controllers/raycluster_webhook.go | 159 +++++++++--------- pkg/controllers/support.go | 7 - 8 files changed, 102 insertions(+), 96 deletions(-) create mode 100644 config/openshift/kustomization.yaml rename config/{default => openshift}/manager_webhook_patch.yaml (100%) rename config/{default => openshift}/webhookcainjection_patch.yaml (100%) diff --git a/PROJECT b/PROJECT index 3e821aa02..0fc5562e2 100644 --- a/PROJECT +++ b/PROJECT @@ -12,7 +12,7 @@ projectName: codeflare-operator repo: github.com/project-codeflare/codeflare-operator resources: - controller: true - domain: codeflare.dev + domain: ray.io group: ray kind: RayCluster path: github.com/project-codeflare/codeflare-operator/pkg/controllers diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index fccc7164e..42fa71796 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -16,13 +16,8 @@ commonLabels: bases: - ../rbac - ../manager - - ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: - metrics_service.yaml - -patches: -- path: manager_webhook_patch.yaml -- path: webhookcainjection_patch.yaml diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml new file mode 100644 index 000000000..22e168162 --- /dev/null +++ b/config/openshift/kustomization.yaml @@ -0,0 +1,22 @@ +# Adds namespace to all resources. +namespace: openshift-operators + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: codeflare-operator- + +# Labels to add to all resources and selectors. +commonLabels: + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare + +bases: + - ../default + - ../webhook + +patches: +- path: manager_webhook_patch.yaml +- path: webhookcainjection_patch.yaml diff --git a/config/default/manager_webhook_patch.yaml b/config/openshift/manager_webhook_patch.yaml similarity index 100% rename from config/default/manager_webhook_patch.yaml rename to config/openshift/manager_webhook_patch.yaml diff --git a/config/default/webhookcainjection_patch.yaml b/config/openshift/webhookcainjection_patch.yaml similarity index 100% rename from config/default/webhookcainjection_patch.yaml rename to config/openshift/webhookcainjection_patch.yaml diff --git a/main.go b/main.go index 40f916a50..1ed191bd5 100644 --- a/main.go +++ b/main.go @@ -147,8 +147,7 @@ func main() { }) exitOnError(err, "unable to start manager") - rayClusterDefaulter := &controllers.RayClusterDefaulter{} - exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook") + exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if ok { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index d63616e13..0cd635040 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -23,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,105 +34,101 @@ import ( // log is for logging in this package. var rayclusterlog = logf.Log.WithName("raycluster-resource") -func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { +func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&RayClusterDefaulter{ - Config: r.Config, - rayDashboardOauthEnabled: r.isRayDashboardOAuthEnabledWebhook(), + WithDefaulter(&rayClusterDefaulter{ + Config: cfg, }). Complete() } -//+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 -type RayClusterDefaulter struct { - Config *config.KubeRayConfiguration - rayDashboardOauthEnabled bool +type rayClusterDefaulter struct { + Config *config.KubeRayConfiguration } -var _ webhook.CustomDefaulter = &RayClusterDefaulter{} +var _ webhook.CustomDefaulter = &rayClusterDefaulter{} // Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { +func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) - if r.rayDashboardOauthEnabled { - rayclusterlog.Info("default", "name", raycluster.Name) - // Check and add OAuth proxy if it does not exist. - alreadyExists := false - for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { - if container.Name == "oauth-proxy" { - rayclusterlog.Info("OAuth sidecar already exists, no patch needed") - alreadyExists = true - break // exits the for loop - } + if !pointer.BoolDeref(r.Config.RayDashboardOAuthEnabled, true) { + return nil + } + + // Check and add OAuth proxy if it does not exist + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.V(2).Info("OAuth sidecar already exists, no patch needed") + return nil } + } - if !alreadyExists { - rayclusterlog.Info("Adding OAuth sidecar container") - // definition of the new container - newOAuthSidecar := corev1.Container{ - Name: "oauth-proxy", - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", - Ports: []corev1.ContainerPort{ - {ContainerPort: 8443, Name: "oauth-proxy"}, - }, - Args: []string{ - "--https-address=:8443", - "--provider=openshift", - "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", - "--upstream=http://localhost:8265", - "--tls-cert=/etc/tls/private/tls.crt", - "--tls-key=/etc/tls/private/tls.key", - "--cookie-secret=$(COOKIE_SECRET)", - "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "proxy-tls-secret", - MountPath: "/etc/tls/private", - ReadOnly: true, - }, - }, - } - - // Adding the new OAuth sidecar container - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - - cookieSecret := corev1.EnvVar{ - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", - }, - Key: "cookie_secret", - }, - }, - } - - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, - cookieSecret, - ) - - tlsSecretVolume := corev1.Volume{ - Name: "proxy-tls-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: raycluster.Name + "-proxy-tls-secret", - }, + rayclusterlog.V(2).Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + } + + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + + cookieSecret := corev1.EnvVar{ + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", }, - } + Key: "cookie_secret", + }, + }, + } - raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, + cookieSecret, + ) + + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, + }, + } - // Ensure the service account is set - if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" - } - } + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" } + return nil } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 698bd8da3..344e6215f 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -155,10 +155,3 @@ func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { } return true } - -func (r *RayClusterDefaulter) isRayDashboardOAuthEnabledWebhook() bool { - if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { - return *r.Config.RayDashboardOAuthEnabled - } - return true -} From 28a9c28d69182d4f62f7ce13aa35ca7eced26f22 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 17:30:54 +0200 Subject: [PATCH 160/369] Start RayCluster webhook on OpenShift only --- main.go | 35 ++++++++++++++++++--- pkg/controllers/raycluster_controller.go | 18 ++++------- pkg/controllers/support.go | 39 ------------------------ 3 files changed, 37 insertions(+), 55 deletions(-) diff --git a/main.go b/main.go index 1ed191bd5..7eff5fc40 100644 --- a/main.go +++ b/main.go @@ -147,11 +147,21 @@ func main() { }) exitOnError(err, "unable to start manager") - exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") + OpenShift := isOpenShift(ctx, kubeClient.DiscoveryClient) - ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + if OpenShift { + // TODO: setup the RayCluster webhook on vanilla Kubernetes + exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") + } + + ok, err := hasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if ok { - rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay} + rayClusterController := controllers.RayClusterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Config: cfg.KubeRay, + IsOpenShift: OpenShift, + } exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") @@ -207,7 +217,7 @@ func createConfigMap(ctx context.Context, client kubernetes.Interface, ns, name return err } -func HasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) { +func hasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) { gv, kind := gvk.ToAPIVersionAndKind() if resources, err := dc.ServerResourcesForGroupVersion(gv); err != nil { if apierrors.IsNotFound(err) { @@ -247,3 +257,20 @@ func exitOnError(err error, msg string) { os.Exit(1) } } + +func isOpenShift(ctx context.Context, dc discovery.DiscoveryInterface) bool { + logger := ctrl.LoggerFrom(ctx) + apiGroupList, err := dc.ServerGroups() + if err != nil { + logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") + return false + } + for i := 0; i < len(apiGroupList.Groups); i++ { + if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { + logger.Info("We detected being on OpenShift!") + return true + } + } + logger.Info("We detected being on Vanilla Kubernetes!") + return false +} diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 551367e49..461e3d01f 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -48,13 +48,12 @@ import ( // RayClusterReconciler reconciles a RayCluster object type RayClusterReconciler struct { client.Client - kubeClient *kubernetes.Clientset - routeClient *routev1client.RouteV1Client - Scheme *runtime.Scheme - CookieSalt string - Config *config.KubeRayConfiguration - IsOpenShift bool - IsOpenShiftInitialized bool + kubeClient *kubernetes.Clientset + routeClient *routev1client.RouteV1Client + Scheme *runtime.Scheme + CookieSalt string + Config *config.KubeRayConfiguration + IsOpenShift bool } const ( @@ -106,11 +105,6 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } - if !r.IsOpenShiftInitialized { - r.IsOpenShift = isOpenShift(ctx, r.kubeClient, &cluster) - r.IsOpenShiftInitialized = true - } - if cluster.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 344e6215f..5a47993be 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "strings" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -12,10 +11,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" v1 "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" - "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" ) @@ -103,41 +99,6 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ) } -// getDiscoveryClient returns a discovery client for the current reconciler -func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) { - return discovery.NewDiscoveryClientForConfig(config) -} - -// Check where we are running. We are trying to distinguish here whether -// this is vanilla kubernetes cluster or Openshift -func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster) bool { - // The discovery package is used to discover APIs supported by a Kubernetes API server. - logger := ctrl.LoggerFrom(ctx) - config, err := ctrl.GetConfig() - if err != nil && config == nil { - logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes") - return false - } - dclient, err := getDiscoveryClient(config) - if err != nil && dclient == nil { - logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes") - return false - } - apiGroupList, err := dclient.ServerGroups() - if err != nil { - logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") - return false - } - for i := 0; i < len(apiGroupList.Groups); i++ { - if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { - logger.Info("We detected being on OpenShift!") - return true - } - } - logger.Info("We detected being on Vanilla Kubernetes!") - return false -} - // getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { ingressDomain := "" From 4a7cb60bce2381e6b735af87ab6fb1d784b41606 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 17:58:25 +0200 Subject: [PATCH 161/369] Refactor RayCluster controller --- pkg/controllers/raycluster_controller.go | 65 +++++++++++++++--------- pkg/controllers/support.go | 22 -------- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 461e3d01f..ba6ca84fe 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "crypto/sha1" "encoding/base64" + "fmt" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -96,9 +97,9 @@ var ( func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := ctrl.LoggerFrom(ctx) - var cluster rayv1.RayCluster + cluster := &rayv1.RayCluster{} - if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { + if err := r.Get(ctx, req.NamespacedName, cluster); err != nil { if !errors.IsNotFound(err) { logger.Error(err, "Error getting RayCluster resource") } @@ -106,21 +107,21 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if cluster.ObjectMeta.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { + if !controllerutil.ContainsFinalizer(cluster, oAuthFinalizer) { logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) - controllerutil.AddFinalizer(&cluster, oAuthFinalizer) - if err := r.Update(ctx, &cluster); err != nil { + controllerutil.AddFinalizer(cluster, oAuthFinalizer) + if err := r.Update(ctx, cluster); err != nil { // this log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } } - } else if controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { + } else if controllerutil.ContainsFinalizer(cluster, oAuthFinalizer) { err := client.IgnoreNotFound(r.Client.Delete( ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: crbNameFromCluster(&cluster), + Name: crbNameFromCluster(cluster), }, }, &deleteOptions, @@ -129,8 +130,8 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - controllerutil.RemoveFinalizer(&cluster, oAuthFinalizer) - if err := r.Update(ctx, &cluster); err != nil { + controllerutil.RemoveFinalizer(cluster, oAuthFinalizer) + if err := r.Update(ctx, cluster); err != nil { logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } @@ -138,66 +139,66 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && r.IsOpenShift { + if cluster.Status.State != "suspended" && isRayDashboardOAuthEnabled(r.Config) && r.IsOpenShift { logger.Info("Creating OAuth Objects") - _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth Route") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to create OAuth Secret") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth Service") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth ServiceAccount") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth ClusterRoleBinding") return ctrl.Result{RequeueAfter: requeueTime}, err } logger.Info("Creating RayClient Route") - _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update RayClient Route") return ctrl.Result{RequeueAfter: requeueTime}, err } - } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { + } else if cluster.Status.State != "suspended" && !isRayDashboardOAuthEnabled(r.Config) && !r.IsOpenShift { logger.Info("We detected being on Vanilla Kubernetes!") logger.Info("Creating Dashboard Ingress") - dashboardName := dashboardNameFromCluster(&cluster) - dashboardIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName) + dashboardName := dashboardNameFromCluster(cluster) + dashboardIngressHost, err := getIngressHost(r.Config, cluster, dashboardName) if err != nil { return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, dashboardIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(cluster, dashboardIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } logger.Info("Creating RayClient Ingress") - rayClientName := rayClientNameFromCluster(&cluster) - rayClientIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName) + rayClientName := rayClientNameFromCluster(cluster) + rayClientIngressHost, err := getIngressHost(r.Config, cluster, rayClientName) if err != nil { return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, rayClientIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(cluster, rayClientIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update RayClient Ingress") return ctrl.Result{RequeueAfter: requeueTime}, err @@ -207,6 +208,24 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } +// getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. +func getIngressHost(cfg *config.KubeRayConfiguration, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { + ingressDomain := "" + if cfg != nil && cfg.IngressDomain != "" { + ingressDomain = cfg.IngressDomain + } else { + return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") + } + return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil +} + +func isRayDashboardOAuthEnabled(cfg *config.KubeRayConfiguration) bool { + if cfg != nil && cfg.RayDashboardOAuthEnabled != nil { + return *cfg.RayDashboardOAuthEnabled + } + return true +} + func crbNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 5a47993be..673d59419 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -1,9 +1,6 @@ package controllers import ( - "context" - "fmt" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" networkingv1 "k8s.io/api/networking/v1" @@ -11,7 +8,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" v1 "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" - "k8s.io/client-go/kubernetes" routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" ) @@ -98,21 +94,3 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ), ) } - -// getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. -func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { - ingressDomain := "" - if r.Config != nil && r.Config.IngressDomain != "" { - ingressDomain = r.Config.IngressDomain - } else { - return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") - } - return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil -} - -func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { - if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { - return *r.Config.RayDashboardOAuthEnabled - } - return true -} From 9f5bde6b52a2810a7cb6922a2c25100e4641c399 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 9 Feb 2024 13:47:58 +0100 Subject: [PATCH 162/369] Adjust env tests for RayCluster controller --- .github/workflows/component_tests.yaml | 43 ++++ Makefile | 10 + pkg/controllers/raycluster_controller_test.go | 206 ++++++++---------- pkg/controllers/suite_test.go | 32 ++- 4 files changed, 168 insertions(+), 123 deletions(-) create mode 100644 .github/workflows/component_tests.yaml diff --git a/.github/workflows/component_tests.yaml b/.github/workflows/component_tests.yaml new file mode 100644 index 000000000..815025b27 --- /dev/null +++ b/.github/workflows/component_tests.yaml @@ -0,0 +1,43 @@ +name: Component tests + +on: + pull_request: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + push: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + +concurrency: + group: ${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kubernetes-component: + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set Go + uses: actions/setup-go@v5 + with: + go-version: v1.20 + + - name: Run component tests + run: | + make test-component diff --git a/Makefile b/Makefile index e1b77a8e1..cb8cd533c 100644 --- a/Makefile +++ b/Makefile @@ -205,6 +205,7 @@ $(LOCALBIN): KUSTOMIZE ?= $(LOCALBIN)/kustomize YQ ?= $(LOCALBIN)/yq CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +GINKGO ?= $(LOCALBIN)/ginkgo ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk @@ -241,6 +242,11 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +.PHONY: ginkgo +ginkgo: $(GINKGO) ## Download ginkgo locally if necessary. +$(GINKGO): $(LOCALBIN) + test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo + .PHONY: install-yq install-yq: $(YQ) ## Download yq locally if necessary $(YQ): $(LOCALBIN) @@ -350,6 +356,10 @@ catalog-push: ## Push a catalog image. test-unit: manifests fmt vet envtest ## Run unit tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out +.PHONY: test-component +test-component: envtest ginkgo ## Run component tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -v ./pkg/controllers/ + .PHONY: test-e2e test-e2e: manifests fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e diff --git a/pkg/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go index 6592aa292..94958b334 100644 --- a/pkg/controllers/raycluster_controller_test.go +++ b/pkg/controllers/raycluster_controller_test.go @@ -17,8 +17,6 @@ limitations under the License. package controllers import ( - "context" - "math/rand" "time" . "github.com/onsi/ginkgo/v2" @@ -29,158 +27,144 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" routev1 "github.com/openshift/api/route/v1" ) -func stringInList(l []string, s string) bool { - for _, i := range l { - if i == s { - return true - } - } - return false -} - -var letters = []rune("abcdefghijklmnopqrstuvwxyz") -var r = rand.New(rand.NewSource(time.Now().UnixNano())) - -func randSeq(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[r.Intn(len(letters))] - } - return string(b) -} - var _ = Describe("RayCluster controller", func() { Context("RayCluster controller test", func() { var rayClusterName = "test-raycluster" - var typeNamespaceName types.NamespacedName - ctx := context.Background() - BeforeEach(func() { - By("Generate random number so each run is creating unique") - rString := randSeq(10) - rayClusterName = rayClusterName + "-" + rString - typeNamespaceName = types.NamespacedName{Name: rayClusterName, Namespace: rayClusterName} + var namespaceName string + BeforeEach(func(ctx SpecContext) { By("Creating a namespace for running the tests.") namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: rayClusterName, + GenerateName: "test-", }, } - var err error - err = k8sClient.Create(ctx, namespace) - Expect(err).To(Not(HaveOccurred())) + namespace, err := k8sClient.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func(ctx SpecContext) { + err := k8sClient.CoreV1().Namespaces().Delete(ctx, namespace.Name, metav1.DeleteOptions{}) + Expect(err).To(Not(HaveOccurred())) + }) + namespaceName = namespace.Name By("creating a basic instance of the RayCluster CR") raycluster := &rayv1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Name: rayClusterName, - Namespace: rayClusterName, + Namespace: namespace.Name, }, Spec: rayv1.RayClusterSpec{ HeadGroupSpec: rayv1.HeadGroupSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{}, - }, + Containers: []corev1.Container{}, }, }, RayStartParams: map[string]string{}, }, }, } - err = k8sClient.Get(ctx, typeNamespaceName, &rayv1.RayCluster{}) - Expect(errors.IsNotFound(err)).To(Equal(true)) - err = k8sClient.Create(ctx, raycluster) + _, err = rayClient.RayV1().RayClusters(namespace.Name).Create(ctx, raycluster, metav1.CreateOptions{}) Expect(err).To(Not(HaveOccurred())) }) - AfterEach(func() { - By("removing the instance of the RayCluster used") - // err := clientSet.CoreV1().Namespaces().Delete(ctx, RayClusterName, metav1.DeleteOptions{}) - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - if err != nil { - Expect(errors.IsNotFound(err)).To(Equal(true)) - } else { + AfterEach(func(ctx SpecContext) { + By("removing instances of the RayClusters used") + rayClusters, err := rayClient.RayV1().RayClusters(namespaceName).List(ctx, metav1.ListOptions{}) + Expect(err).To(Not(HaveOccurred())) + + for _, rayCluster := range rayClusters.Items { + err = rayClient.RayV1().RayClusters(namespaceName).Delete(ctx, rayCluster.Name, metav1.DeleteOptions{}) Expect(err).To(Not(HaveOccurred())) - _ = k8sClient.Delete(ctx, &foundRayCluster) } - Eventually(func() bool { - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - return errors.IsNotFound(err) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - }) - It("should have oauth finalizer set", func() { - foundRayCluster := rayv1.RayCluster{} - Eventually(func() bool { - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - return stringInList(foundRayCluster.Finalizers, oAuthFinalizer) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() ([]rayv1.RayCluster, error) { + rayClusters, err := rayClient.RayV1().RayClusters(namespaceName).List(ctx, metav1.ListOptions{}) + return rayClusters.Items, err + }).WithTimeout(time.Second * 10).Should(BeEmpty()) }) - It("should create all oauth resources", func() { - Eventually(func() error { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) - if err != nil { - return err - } - return nil - }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) - }) + It("should have oauth finalizer set", func(ctx SpecContext) { + Eventually(func() ([]string, error) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) + return foundRayCluster.Finalizers, err + }).WithTimeout(time.Second * 10).Should(ContainElement(oAuthFinalizer)) + }, SpecTimeout(time.Second*10)) - It("should set owner references for all resources", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).ToNot(HaveOccurred()) - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + It("should create all oauth resources", func(ctx SpecContext) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + + Eventually(func() (*corev1.Secret, error) { + return k8sClient.CoreV1().Secrets(namespaceName).Get(ctx, oauthSecretNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*corev1.Service, error) { + return k8sClient.CoreV1().Services(namespaceName).Get(ctx, oauthServiceNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*rbacv1.ClusterRoleBinding, error) { + return k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, crbNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*routev1.Route, error) { + return routeClient.RouteV1().Routes(namespaceName).Get(ctx, dashboardNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + }) + + It("should set owner references for all resources", func(ctx SpecContext) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) + + Eventually(func() (*corev1.Secret, error) { + return k8sClient.CoreV1().Secrets(namespaceName).Get(ctx, oauthSecretNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*corev1.Secret, error) { + return k8sClient.CoreV1().Secrets(namespaceName).Get(ctx, oauthSecretNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) + Eventually(func() (*corev1.Service, error) { + return k8sClient.CoreV1().Services(namespaceName).Get(ctx, oauthServiceNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*corev1.Service, error) { + return k8sClient.CoreV1().Services(namespaceName).Get(ctx, oauthServiceNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) + Eventually(func() (*routev1.Route, error) { + return routeClient.RouteV1().Routes(namespaceName).Get(ctx, dashboardNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*routev1.Route, error) { + return routeClient.RouteV1().Routes(namespaceName).Get(ctx, dashboardNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) }) - It("should remove CRB when the RayCluster is deleted", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + It("should remove CRB when the RayCluster is deleted", func(ctx SpecContext) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Delete(ctx, &foundRayCluster) + + err = rayClient.RayV1().RayClusters(namespaceName).Delete(ctx, foundRayCluster.Name, metav1.DeleteOptions{}) Expect(err).To(Not(HaveOccurred())) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) + + Eventually(func() error { + _, err := k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, crbNameFromCluster(foundRayCluster), metav1.GetOptions{}) + return err + }).WithTimeout(time.Second * 10).Should(Satisfy(errors.IsNotFound)) }) + }) }) + +func OwnerReferenceKind(meta metav1.Object) string { + return meta.GetOwnerReferences()[0].Kind +} + +func OwnerReferenceName(meta metav1.Object) string { + return meta.GetOwnerReferences()[0].Name +} diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index 0110179f7..6828bdada 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -27,17 +27,18 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" routev1 "github.com/openshift/api/route/v1" + routeclient "github.com/openshift/client-go/route/clientset/versioned" //+kubebuilder:scaffold:imports ) @@ -45,8 +46,12 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var cfg *rest.Config -var k8sClient client.Client +var k8sClient *kubernetes.Clientset +var rayClient *rayclient.Clientset +var routeClient *routeclient.Clientset var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) @@ -56,10 +61,11 @@ func TestAPIs(t *testing.T) { const ( RayClusterCRDFileDownload = "https://raw.githubusercontent.com/ray-project/kuberay/master/ray-operator/config/crd/bases/ray.io_rayclusters.yaml" - RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/route.crd.yaml" + RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/zz_generated.crd-manifests/routes-Default.crd.yaml" ) var _ = BeforeSuite(func() { + ctx, cancel = context.WithCancel(context.Background()) logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) var err error @@ -86,7 +92,6 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ - filepath.Join("..", "config", "crd"), filepath.Join(".", "test-crds"), }, ErrorIfCRDPathMissing: true, @@ -99,29 +104,31 @@ var _ = BeforeSuite(func() { //+kubebuilder:scaffold:scheme - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + k8sClient, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - clientSet, err := kubernetes.NewForConfig(cfg) + rayClient, err = rayclient.NewForConfig(cfg) + Expect(err).To(Not(HaveOccurred())) + routeClient, err = routeclient.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) err = rayv1.AddToScheme(scheme.Scheme) Expect(err).To(Not(HaveOccurred())) err = routev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, }) Expect(err).NotTo(HaveOccurred()) err = (&RayClusterReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - kubeClient: clientSet, - CookieSalt: "foo", + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + CookieSalt: "foo", + IsOpenShift: true, }).SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) go func() { defer GinkgoRecover() - err = k8sManager.Start(context.Background()) + err = k8sManager.Start(ctx) Expect(err).ToNot(HaveOccurred(), "failed to run manager") }() }) @@ -130,6 +137,7 @@ var _ = AfterSuite(func() { By("tearing down the test environment") err := os.RemoveAll("./test-crds") Expect(err).NotTo(HaveOccurred()) + cancel() err = testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) From 0873fc4058245c59495915804fbcbbea791439c0 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 17 Apr 2024 18:01:35 +0100 Subject: [PATCH 163/369] Create RayCluster ValidatingWebhook --- PROJECT | 1 + config/openshift/kustomization.yaml | 4 +- ...ch.yaml => webhookcainjection_mpatch.yaml} | 0 .../openshift/webhookcainjection_vpatch.yaml | 7 ++++ config/webhook/kustomization.yaml | 2 + config/webhook/manifests.yaml | 27 +++++++++++++ pkg/controllers/raycluster_webhook.go | 39 +++++++++++++++++++ 7 files changed, 78 insertions(+), 2 deletions(-) rename config/openshift/{webhookcainjection_patch.yaml => webhookcainjection_mpatch.yaml} (100%) create mode 100644 config/openshift/webhookcainjection_vpatch.yaml diff --git a/PROJECT b/PROJECT index 0fc5562e2..965a62fd1 100644 --- a/PROJECT +++ b/PROJECT @@ -19,5 +19,6 @@ resources: version: v1 webhooks: defaulting: true + validation: true webhookVersion: v1 version: "3" diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml index 22e168162..b39da0736 100644 --- a/config/openshift/kustomization.yaml +++ b/config/openshift/kustomization.yaml @@ -6,7 +6,6 @@ namespace: openshift-operators # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. -namePrefix: codeflare-operator- # Labels to add to all resources and selectors. commonLabels: @@ -19,4 +18,5 @@ bases: patches: - path: manager_webhook_patch.yaml -- path: webhookcainjection_patch.yaml +- path: webhookcainjection_mpatch.yaml +- path: webhookcainjection_vpatch.yaml diff --git a/config/openshift/webhookcainjection_patch.yaml b/config/openshift/webhookcainjection_mpatch.yaml similarity index 100% rename from config/openshift/webhookcainjection_patch.yaml rename to config/openshift/webhookcainjection_mpatch.yaml diff --git a/config/openshift/webhookcainjection_vpatch.yaml b/config/openshift/webhookcainjection_vpatch.yaml new file mode 100644 index 000000000..91f741a05 --- /dev/null +++ b/config/openshift/webhookcainjection_vpatch.yaml @@ -0,0 +1,7 @@ +# This patch add annotation to admission webhook config +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 9cf26134e..412cee110 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,3 +1,5 @@ +namePrefix: codeflare-operator- + resources: - manifests.yaml - service.yaml diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index faf91696b..c14976cfe 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -25,3 +25,30 @@ webhooks: resources: - rayclusters sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-ray-io-v1-raycluster + failurePolicy: Fail + name: vraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 0cd635040..d0c046941 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -23,10 +23,12 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/project-codeflare/codeflare-operator/pkg/config" ) @@ -40,16 +42,24 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf WithDefaulter(&rayClusterDefaulter{ Config: cfg, }). + WithValidator(&rayClusterWebhook{ + Config: cfg, + }). Complete() } // +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 type rayClusterDefaulter struct { Config *config.KubeRayConfiguration } +type rayClusterWebhook struct { + Config *config.KubeRayConfiguration +} var _ webhook.CustomDefaulter = &rayClusterDefaulter{} +var _ webhook.CustomValidator = &rayClusterWebhook{} // Default implements webhook.Defaulter so a webhook will be registered for the type func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { @@ -132,3 +142,32 @@ func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e return nil } + +func (v *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + raycluster := obj.(*rayv1.RayCluster) + var warnings admission.Warnings + var allErrors field.ErrorList + specPath := field.NewPath("spec") + + if pointer.BoolDeref(raycluster.Spec.HeadGroupSpec.EnableIngress, false) { + rayclusterlog.Info("Creating RayCluster resources with EnableIngress set to true or unspecified is not allowed") + allErrors = append(allErrors, field.Invalid(specPath.Child("headGroupSpec").Child("enableIngress"), raycluster.Spec.HeadGroupSpec.EnableIngress, "creating RayCluster resources with EnableIngress set to true or unspecified is not allowed")) + } + + return warnings, allErrors.ToAggregate() +} + +func (v *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + newRayCluster := newObj.(*rayv1.RayCluster) + if !newRayCluster.DeletionTimestamp.IsZero() { + // Object is being deleted, skip validations + return nil, nil + } + warnings, err := v.ValidateCreate(ctx, newRayCluster) + return warnings, err +} + +func (v *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // Optional: Add delete validation logic here + return nil, nil +} From b4464d1c1355d475e465469b7bc8afa5c6f64f66 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 18 Apr 2024 10:16:15 +0100 Subject: [PATCH 164/369] Remove Defaulter type in webhook --- pkg/controllers/raycluster_webhook.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index d0c046941..bc003d4d2 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -27,7 +27,6 @@ import ( "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -39,7 +38,7 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&rayClusterDefaulter{ + WithDefaulter(&rayClusterWebhook{ Config: cfg, }). WithValidator(&rayClusterWebhook{ @@ -51,21 +50,15 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf // +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 // +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 -type rayClusterDefaulter struct { - Config *config.KubeRayConfiguration -} type rayClusterWebhook struct { Config *config.KubeRayConfiguration } -var _ webhook.CustomDefaulter = &rayClusterDefaulter{} -var _ webhook.CustomValidator = &rayClusterWebhook{} - // Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { +func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) - if !pointer.BoolDeref(r.Config.RayDashboardOAuthEnabled, true) { + if !pointer.BoolDeref(w.Config.RayDashboardOAuthEnabled, true) { return nil } @@ -143,7 +136,7 @@ func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e return nil } -func (v *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { raycluster := obj.(*rayv1.RayCluster) var warnings admission.Warnings var allErrors field.ErrorList @@ -157,17 +150,17 @@ func (v *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Obje return warnings, allErrors.ToAggregate() } -func (v *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { +func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newRayCluster := newObj.(*rayv1.RayCluster) if !newRayCluster.DeletionTimestamp.IsZero() { // Object is being deleted, skip validations return nil, nil } - warnings, err := v.ValidateCreate(ctx, newRayCluster) + warnings, err := w.ValidateCreate(ctx, newRayCluster) return warnings, err } -func (v *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { // Optional: Add delete validation logic here return nil, nil } From 2b885ebf330b29d64deb50829c13eaa95b1cef1f Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 18 Apr 2024 10:32:44 +0100 Subject: [PATCH 165/369] Use single rayClusterWebhook instance --- pkg/controllers/raycluster_webhook.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index bc003d4d2..8d06eebec 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -27,6 +27,7 @@ import ( "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -36,14 +37,13 @@ import ( var rayclusterlog = logf.Log.WithName("raycluster-resource") func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration) error { + rayClusterWebhookInstance := &rayClusterWebhook{ + Config: cfg, + } return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&rayClusterWebhook{ - Config: cfg, - }). - WithValidator(&rayClusterWebhook{ - Config: cfg, - }). + WithDefaulter(rayClusterWebhookInstance). + WithValidator(rayClusterWebhookInstance). Complete() } @@ -54,6 +54,9 @@ type rayClusterWebhook struct { Config *config.KubeRayConfiguration } +var _ webhook.CustomDefaulter = &rayClusterWebhook{} +var _ webhook.CustomValidator = &rayClusterWebhook{} + // Default implements webhook.Defaulter so a webhook will be registered for the type func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) From a51f336b2f4a8271d519bb46ccf89a459061dd58 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 12:09:24 +0200 Subject: [PATCH 166/369] Fix oauth-proxy cookie Secret --- pkg/controllers/raycluster_webhook.go | 49 ++++++++++++--------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 8d06eebec..d465fd2e4 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -59,14 +59,14 @@ var _ webhook.CustomValidator = &rayClusterWebhook{} // Default implements webhook.Defaulter so a webhook will be registered for the type func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { - raycluster := obj.(*rayv1.RayCluster) + rayCluster := obj.(*rayv1.RayCluster) if !pointer.BoolDeref(w.Config.RayDashboardOAuthEnabled, true) { return nil } // Check and add OAuth proxy if it does not exist - for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + for _, container := range rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { if container.Name == "oauth-proxy" { rayclusterlog.V(2).Info("OAuth sidecar already exists, no patch needed") return nil @@ -74,17 +74,30 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } rayclusterlog.V(2).Info("Adding OAuth sidecar container") - // definition of the new container + newOAuthSidecar := corev1.Container{ Name: "oauth-proxy", Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", Ports: []corev1.ContainerPort{ {ContainerPort: 8443, Name: "oauth-proxy"}, }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: rayCluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + }, + }, Args: []string{ "--https-address=:8443", "--provider=openshift", - "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--openshift-service-account=" + rayCluster.Name + "-oauth-proxy", "--upstream=http://localhost:8265", "--tls-cert=/etc/tls/private/tls.crt", "--tls-key=/etc/tls/private/tls.key", @@ -100,40 +113,22 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err }, } - // Adding the new OAuth sidecar container - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - - cookieSecret := corev1.EnvVar{ - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", - }, - Key: "cookie_secret", - }, - }, - } - - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, - cookieSecret, - ) + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) tlsSecretVolume := corev1.Volume{ Name: "proxy-tls-secret", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: raycluster.Name + "-proxy-tls-secret", + SecretName: rayCluster.Name + "-proxy-tls-secret", }, }, } - raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) // Ensure the service account is set - if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" } return nil From 1c85429226d05c74d3857c384f1ee37c8f9bd673 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 14:57:03 +0200 Subject: [PATCH 167/369] Use deep semantic comparison in RayCluster validation webhook --- config/webhook/manifests.yaml | 1 - pkg/controllers/raycluster_webhook.go | 160 +++++++++++++++++--------- pkg/controllers/support.go | 56 ++++++++- 3 files changed, 158 insertions(+), 59 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index c14976cfe..ee680a180 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -21,7 +21,6 @@ webhooks: - v1 operations: - CREATE - - UPDATE resources: - rayclusters sideEffects: None diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index d465fd2e4..4ac908b59 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -33,6 +33,11 @@ import ( "github.com/project-codeflare/codeflare-operator/pkg/config" ) +const ( + oauthProxyContainerName = "oauth-proxy" + oauthProxyVolumeName = "proxy-tls-secret" +) + // log is for logging in this package. var rayclusterlog = logf.Log.WithName("raycluster-resource") @@ -47,7 +52,7 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf Complete() } -// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 // +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 type rayClusterWebhook struct { @@ -65,18 +70,105 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err return nil } - // Check and add OAuth proxy if it does not exist - for _, container := range rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { - if container.Name == "oauth-proxy" { - rayclusterlog.V(2).Info("OAuth sidecar already exists, no patch needed") - return nil - } + rayclusterlog.V(2).Info("Adding OAuth sidecar container") + + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName)) + + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName)) + + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" + + return nil +} + +func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + rayCluster := obj.(*rayv1.RayCluster) + + var warnings admission.Warnings + var allErrors field.ErrorList + + allErrors = append(allErrors, validateIngress(rayCluster)...) + + return warnings, allErrors.ToAggregate() +} + +func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + rayCluster := newObj.(*rayv1.RayCluster) + + var warnings admission.Warnings + var allErrors field.ErrorList + + if !rayCluster.DeletionTimestamp.IsZero() { + // Object is being deleted, skip validations + return nil, nil + } + + allErrors = append(allErrors, validateIngress(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) + allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + + return warnings, allErrors.ToAggregate() +} + +func (w *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // Optional: Add delete validation logic here + return nil, nil +} + +func validateOAuthProxyContainer(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), byContainerName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "containers"), + "OAuth Proxy container is immutable"); err != nil { + allErrors = append(allErrors, err) } - rayclusterlog.V(2).Info("Adding OAuth sidecar container") + return allErrors +} + +func validateOAuthProxyVolume(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList - newOAuthSidecar := corev1.Container{ - Name: "oauth-proxy", + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), byVolumeName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "volumes"), + "OAuth Proxy TLS Secret volume is immutable"); err != nil { + allErrors = append(allErrors, err) + } + + return allErrors +} + +func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if pointer.BoolDeref(rayCluster.Spec.HeadGroupSpec.EnableIngress, false) { + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "headGroupSpec", "enableIngress"), + rayCluster.Spec.HeadGroupSpec.EnableIngress, + "RayCluster resources with EnableIngress set to true or unspecified is not allowed")) + } + + return allErrors +} + +func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != rayCluster.Name+"-oauth-proxy" { + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "headGroupSpec", "template", "spec", "serviceAccountName"), + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName, + "RayCluster head group service account is immutable")) + } + + return allErrors +} + +func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container { + return corev1.Container{ + Name: oauthProxyContainerName, Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", Ports: []corev1.ContainerPort{ {ContainerPort: 8443, Name: "oauth-proxy"}, @@ -106,59 +198,21 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err }, VolumeMounts: []corev1.VolumeMount{ { - Name: "proxy-tls-secret", + Name: oauthProxyVolumeName, MountPath: "/etc/tls/private", ReadOnly: true, }, }, } +} - rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - - tlsSecretVolume := corev1.Volume{ - Name: "proxy-tls-secret", +func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume { + return corev1.Volume{ + Name: oauthProxyVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: rayCluster.Name + "-proxy-tls-secret", }, }, } - - rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) - - // Ensure the service account is set - if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" - } - - return nil -} - -func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { - raycluster := obj.(*rayv1.RayCluster) - var warnings admission.Warnings - var allErrors field.ErrorList - specPath := field.NewPath("spec") - - if pointer.BoolDeref(raycluster.Spec.HeadGroupSpec.EnableIngress, false) { - rayclusterlog.Info("Creating RayCluster resources with EnableIngress set to true or unspecified is not allowed") - allErrors = append(allErrors, field.Invalid(specPath.Child("headGroupSpec").Child("enableIngress"), raycluster.Spec.HeadGroupSpec.EnableIngress, "creating RayCluster resources with EnableIngress set to true or unspecified is not allowed")) - } - - return warnings, allErrors.ToAggregate() -} - -func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { - newRayCluster := newObj.(*rayv1.RayCluster) - if !newRayCluster.DeletionTimestamp.IsZero() { - // Object is being deleted, skip validations - return nil, nil - } - warnings, err := w.ValidateCreate(ctx, newRayCluster) - return warnings, err -} - -func (w *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { - // Optional: Add delete validation logic here - return nil, nil } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 673d59419..0efbc7475 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -3,9 +3,11 @@ package controllers import ( rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation/field" v1 "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" @@ -29,7 +31,6 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf ) } -// Create an Ingress object for the RayCluster func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). @@ -42,7 +43,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net WithAPIVersion(cluster.APIVersion). WithKind(cluster.Kind). WithName(cluster.Name). - WithUID(types.UID(cluster.UID))). + WithUID(cluster.UID)). WithSpec(networkingv1ac.IngressSpec(). WithIngressClassName("nginx"). WithRules(networkingv1ac.IngressRule(). @@ -65,7 +66,6 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net ) } -// Create an Ingress object for the RayCluster func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). @@ -73,7 +73,7 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo WithAPIVersion(cluster.APIVersion). WithKind(cluster.Kind). WithName(cluster.Name). - WithUID(types.UID(cluster.UID))). + WithUID(cluster.UID)). WithSpec(networkingv1ac.IngressSpec(). WithRules(networkingv1ac.IngressRule(). WithHost(ingressHost). // Full Hostname @@ -94,3 +94,49 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ), ) } + +type compare[T any] func(T, T) bool + +func upsert[T any](items []T, item T, predicate compare[T]) []T { + for i, t := range items { + if predicate(t, item) { + items[i] = item + return items + } + } + return append(items, item) +} + +func contains[T any](items []T, item T, predicate compare[T], path *field.Path, msg string) *field.Error { + for _, t := range items { + if predicate(t, item) { + if equality.Semantic.DeepDerivative(item, t) { + return nil + } + return field.Invalid(path, t, msg) + } + } + return field.Required(path, msg) +} + +var byContainerName = compare[corev1.Container]( + func(c1, c2 corev1.Container) bool { + return c1.Name == c2.Name + }) + +func withContainerName(name string) compare[corev1.Container] { + return func(c1, c2 corev1.Container) bool { + return c1.Name == name + } +} + +var byVolumeName = compare[corev1.Volume]( + func(v1, v2 corev1.Volume) bool { + return v1.Name == v2.Name + }) + +func withVolumeName(name string) compare[corev1.Volume] { + return func(v1, v2 corev1.Volume) bool { + return v1.Name == name + } +} From 1518c686e7f7b55f3803c3ca457ef335bb197c68 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 15:19:42 +0200 Subject: [PATCH 168/369] Rename RayCluster webhooks --- config/webhook/manifests.yaml | 4 ++-- pkg/controllers/raycluster_webhook.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index ee680a180..5e98107c8 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -13,7 +13,7 @@ webhooks: namespace: system path: /mutate-ray-io-v1-raycluster failurePolicy: Fail - name: mraycluster.kb.io + name: mraycluster.ray.openshift.ai rules: - apiGroups: - ray.io @@ -39,7 +39,7 @@ webhooks: namespace: system path: /validate-ray-io-v1-raycluster failurePolicy: Fail - name: vraycluster.kb.io + name: vraycluster.ray.openshift.ai rules: - apiGroups: - ray.io diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 4ac908b59..e7d9a081f 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -52,8 +52,8 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf Complete() } -// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 -// +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create,versions=v1,name=mraycluster.ray.openshift.ai,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.ray.openshift.ai,admissionReviewVersions=v1 type rayClusterWebhook struct { Config *config.KubeRayConfiguration From 7c644080f71e7cd5a7a87ff7863220fd3b645aad Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 15:24:32 +0200 Subject: [PATCH 169/369] Better message for RayCluster enableIngress field validation --- pkg/controllers/raycluster_webhook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index e7d9a081f..27a4b56b5 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -147,7 +147,7 @@ func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "headGroupSpec", "enableIngress"), rayCluster.Spec.HeadGroupSpec.EnableIngress, - "RayCluster resources with EnableIngress set to true or unspecified is not allowed")) + "RayCluster with enableIngress set to true is not allowed")) } return allErrors From b90451c16b693a1159ad846ad71129fe07002f0e Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 18 Apr 2024 18:11:07 -0400 Subject: [PATCH 170/369] Get go version for setup-go action from go.mod file --- .github/workflows/component_tests.yaml | 2 +- .github/workflows/e2e_tests.yaml | 2 +- .github/workflows/mnist-job-test-image.yml | 2 +- .github/workflows/olm_tests.yaml | 2 +- .github/workflows/operator-image.yml | 2 +- .github/workflows/precommit.yml | 2 +- .github/workflows/tag-and-build.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- .github/workflows/verify_generated_files.yml | 4 ++-- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/component_tests.yaml b/.github/workflows/component_tests.yaml index 815025b27..8f5bf7a63 100644 --- a/.github/workflows/component_tests.yaml +++ b/.github/workflows/component_tests.yaml @@ -36,7 +36,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Run component tests run: | diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 30ef1ea99..0a0ad3977 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -45,7 +45,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/mnist-job-test-image.yml b/.github/workflows/mnist-job-test-image.yml index 4323bd5dd..02c4b5c7e 100644 --- a/.github/workflows/mnist-job-test-image.yml +++ b/.github/workflows/mnist-job-test-image.yml @@ -19,7 +19,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Login to Quay.io id: podman-login-quay diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 472eda20f..abce29d3d 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -44,7 +44,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index 8c30a8c17..0713d36c7 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -22,7 +22,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Login to Quay.io id: podman-login-quay diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 2ca5869c6..267d537a9 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -26,7 +26,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Activate cache uses: actions/cache@v4 diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index a60dceb7b..dc4a07a20 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -56,7 +56,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Verify that release doesn't exist yet shell: bash {0} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 747cee03d..b8e39f08e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -27,7 +27,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Activate cache uses: actions/cache@v4 diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index 9027c1aad..a5f6eb38a 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -24,7 +24,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Verify that imports are organized run: make verify-imports @@ -35,6 +35,6 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated run: make manifests && git diff --exit-code From abdebf594e980d76d9846644636aa56adb7f156f Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 18 Apr 2024 18:21:06 -0400 Subject: [PATCH 171/369] Remove dangling MCAD and InstaScale references Cleanup a few MCAD/InstaScale references missed in c771aae4. --- CONTRIBUTING.md | 10 ---------- README.md | 8 ++------ .../codeflare-operator.clusterserviceversion.yaml | 2 -- main.go | 12 ++++-------- 4 files changed, 6 insertions(+), 26 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8613b92fe..3d7bf8d35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,15 +14,6 @@ The following should be installed in your working environment: ## Basic Overview The main entrypoint for the operator is `main.go` -The MCAD and InstaScale custom resources are defined under the `api` dir: - - See `mcad_types.go` and `instascale_types.go` - -The MCAD and InstaScale resource templates can be found under `config/internal`: - - Sorted under `mcad` and `instascale` subdirs - -The code for MCAD/InstaScale resource reconcilliation can be found in the `controllers` dir: - - See `mcad_controller.go` and `instascale_controller.go` - ## Building and Deployment If changes are made in the `api` dir, run: `make manifests` - This will generate new CRDs and associated files @@ -53,7 +44,6 @@ The CodeFlare Operator currently has unit tests and pre-commit checks - Note that both are required for CI to pass on pull requests To write and inspect unit tests: - - MCAD and InstaScale unit tests under `mcad_controller_test.go` and `instascale_controller_test.go` in the `controllers` dir - Unit test functions are defined in `suite_test.go` (with utils in `util/util.go`) in the `controllers dir` - Test cases defined under `controllers/testdata` diff --git a/README.md b/README.md index ef2b44d5a..423fb9745 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # codeflare-operator -Operator for installation and lifecycle management of CodeFlare distributed workload stack, starting with MCAD and InstaScale +Operator for installation and lifecycle management of CodeFlare distributed workload stack. @@ -9,9 +9,7 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.3.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.1) | -| Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | | CodeFlare-SDK | [v0.15.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.1) | -| InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | | KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | @@ -90,7 +88,7 @@ The e2e tests can be executed locally by running the following commands: To properly run e2e tests on disconnected cluster user has to provide additional environment variables to properly configure testing environment: -- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job using MCAD +- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job - `CODEFLARE_TEST_RAY_IMAGE` - image tag for Ray cluster image - `MNIST_DATASET_URL` - URL where MNIST dataset is available - `PIP_INDEX_URL` - URL where PyPI server with needed dependencies is running @@ -118,8 +116,6 @@ For ODH tests additional environment variables are needed: There may be instances in which a new CodeFlare stack release requires releases of only a subset of the stack components. Examples could be hotfixes for a specific component. In these instances: 1. Build updated components as needed: - - Build and release [MCAD](https://github.com/project-codeflare/multi-cluster-app-dispatcher) - - Build and release [InstaScale](https://github.com/project-codeflare/instascale) - Build and release [CodeFlare-SDK](https://github.com/project-codeflare/codeflare-sdk) 2. Invoke [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, this action will create a repository tag, build and push operator image. diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 04a0b301d..f47cdd1a2 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -72,9 +72,7 @@ spec: keywords: - Pipelines - Scaling - - MCAD - App - - InstaScale - Jobs links: - name: Codeflare Operator diff --git a/main.go b/main.go index 7eff5fc40..b1aa0a91c 100644 --- a/main.go +++ b/main.go @@ -56,12 +56,10 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - OperatorVersion = "UNKNOWN" - McadVersion = "UNKNOWN" - InstaScaleVersion = "UNKNOWN" - BuildDate = "UNKNOWN" + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + OperatorVersion = "UNKNOWN" + BuildDate = "UNKNOWN" ) func init() { @@ -90,8 +88,6 @@ func main() { setupLog.Info("Build info", "operatorVersion", OperatorVersion, - "mcadVersion", McadVersion, - "instaScaleVersion", InstaScaleVersion, "date", BuildDate, ) From a0edab0385ea96f484878f01292b4c059033f587 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 19:52:21 +0200 Subject: [PATCH 172/369] Use cert-controller to generate webhook certificates --- config/default/kustomization.yaml | 4 + .../manager_webhook_patch.yaml | 2 +- config/openshift/kustomization.yaml | 22 ---- .../openshift/webhookcainjection_mpatch.yaml | 7 -- .../openshift/webhookcainjection_vpatch.yaml | 7 -- config/rbac/role.yaml | 27 +++++ config/webhook/kustomization.yaml | 3 +- config/webhook/secret.yaml | 5 + config/webhook/service.yaml | 3 - go.mod | 35 +++--- go.sum | 105 +++++++----------- main.go | 86 ++++++++++---- pkg/controllers/raycluster_webhook.go | 6 +- 13 files changed, 167 insertions(+), 145 deletions(-) rename config/{openshift => default}/manager_webhook_patch.yaml (87%) delete mode 100644 config/openshift/kustomization.yaml delete mode 100644 config/openshift/webhookcainjection_mpatch.yaml delete mode 100644 config/openshift/webhookcainjection_vpatch.yaml create mode 100644 config/webhook/secret.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 42fa71796..30b1273cd 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -16,8 +16,12 @@ commonLabels: bases: - ../rbac - ../manager + - ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: - metrics_service.yaml + +patches: + - path: manager_webhook_patch.yaml diff --git a/config/openshift/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml similarity index 87% rename from config/openshift/manager_webhook_patch.yaml rename to config/default/manager_webhook_patch.yaml index 5d6b541dc..ab553c007 100644 --- a/config/openshift/manager_webhook_patch.yaml +++ b/config/default/manager_webhook_patch.yaml @@ -20,4 +20,4 @@ spec: - name: cert secret: defaultMode: 420 - secretName: codeflare-operator-raycluster-webhook-cert + secretName: webhook-server-cert diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml deleted file mode 100644 index b39da0736..000000000 --- a/config/openshift/kustomization.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Adds namespace to all resources. -namespace: openshift-operators - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. - -# Labels to add to all resources and selectors. -commonLabels: - app.kubernetes.io/name: codeflare-operator - app.kubernetes.io/part-of: codeflare - -bases: - - ../default - - ../webhook - -patches: -- path: manager_webhook_patch.yaml -- path: webhookcainjection_mpatch.yaml -- path: webhookcainjection_vpatch.yaml diff --git a/config/openshift/webhookcainjection_mpatch.yaml b/config/openshift/webhookcainjection_mpatch.yaml deleted file mode 100644 index 39db86347..000000000 --- a/config/openshift/webhookcainjection_mpatch.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# This patch add annotation to admission webhook config -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/openshift/webhookcainjection_vpatch.yaml b/config/openshift/webhookcainjection_vpatch.yaml deleted file mode 100644 index 91f741a05..000000000 --- a/config/openshift/webhookcainjection_vpatch.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# This patch add annotation to admission webhook config -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 70a6a861d..24ac70871 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,33 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch - apiGroups: - authentication.k8s.io resources: diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 412cee110..297cf8a86 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,8 +1,7 @@ -namePrefix: codeflare-operator- - resources: - manifests.yaml - service.yaml +- secret.yaml configurations: - kustomizeconfig.yaml diff --git a/config/webhook/secret.yaml b/config/webhook/secret.yaml new file mode 100644 index 000000000..af960e6c0 --- /dev/null +++ b/config/webhook/secret.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: system diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index ada0e3281..01a9f23df 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -2,9 +2,6 @@ apiVersion: v1 kind: Service metadata: name: webhook-service - namespace: openshift-operators - annotations: - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert spec: ports: - port: 443 diff --git a/go.mod b/go.mod index 80f3a52b8..06797c212 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,19 @@ go 1.20 require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 + github.com/open-policy-agent/cert-controller v0.10.1 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 - k8s.io/api v0.27.8 - k8s.io/apimachinery v0.27.8 - k8s.io/client-go v0.27.8 - k8s.io/component-base v0.27.8 - k8s.io/klog/v2 v2.90.1 - k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 - sigs.k8s.io/controller-runtime v0.15.3 + k8s.io/api v0.28.1 + k8s.io/apimachinery v0.28.1 + k8s.io/client-go v0.28.1 + k8s.io/component-base v0.28.1 + k8s.io/klog/v2 v2.100.1 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b + sigs.k8s.io/controller-runtime v0.16.1 sigs.k8s.io/yaml v1.3.0 ) @@ -28,22 +29,22 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.6.9 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect @@ -66,7 +67,9 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect - go.uber.org/multierr v1.10.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.16.0 // indirect @@ -74,14 +77,14 @@ require ( golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.27.7 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/apiextensions-apiserver v0.28.1 // indirect + k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 51ec31f47..8a496b7e9 100644 --- a/go.sum +++ b/go.sum @@ -34,13 +34,11 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -48,12 +46,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -63,8 +59,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -73,25 +67,20 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -108,8 +97,8 @@ github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -121,8 +110,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= -github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= @@ -159,8 +148,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= -github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -188,14 +177,12 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -275,7 +262,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -326,6 +312,9 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= +github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= github.com/openshift-online/ocm-sdk-go v0.1.368 h1:qP+gkChV8WDwwpkUw1xUyjTXKdvrwyd70Gff2GMUSeU= github.com/openshift-online/ocm-sdk-go v0.1.368/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= @@ -370,9 +359,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -383,10 +371,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -403,9 +389,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -418,20 +401,21 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -463,6 +447,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -523,7 +509,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -618,7 +603,6 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -686,8 +670,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= -gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -735,14 +719,12 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -755,9 +737,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -770,7 +749,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -787,7 +765,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -795,7 +772,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -806,27 +782,28 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.27.8 h1:Ja93gbyII5c3TJzWefEwGhlqC5SZksWhzRS+OYHIocU= -k8s.io/api v0.27.8/go.mod h1:2HuWJC6gpx4UScY+ezziNzv6j6Jqd2q0rGgobYSSjcs= -k8s.io/apiextensions-apiserver v0.27.7 h1:YqIOwZAUokzxJIjunmUd4zS1v3JhK34EPXn+pP0/bsU= -k8s.io/apiextensions-apiserver v0.27.7/go.mod h1:x0p+b5a955lfPz9gaDeBy43obM12s+N9dNHK6+dUL+g= -k8s.io/apimachinery v0.27.8 h1:Xg+ogjDm8s7KmV3vZGf7uOZ0jrC6FPy2Lk/h7BIRmvg= -k8s.io/apimachinery v0.27.8/go.mod h1:EIXLxLt/b1muPITiF5zlrcm7I+YnXsIgM+0GdnPTQvA= -k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= -k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= -k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= -k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= +k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= +k8s.io/apiextensions-apiserver v0.28.1/go.mod h1:sVvrI+P4vxh2YBBcm8n2ThjNyzU4BQGilCQ/JAY5kGs= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= +k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= +k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= +k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs= +k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D1/LqPzj0bLjxLYWbdUtV0= +k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= -sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= +sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= +sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/main.go b/main.go index b1aa0a91c..4228d1b4a 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "strings" "time" + cert "github.com/open-policy-agent/cert-controller/pkg/rotator" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" @@ -32,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" @@ -40,10 +42,11 @@ import ( "k8s.io/client-go/rest" configv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/klog/v2" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/yaml" routev1 "github.com/openshift/api/route/v1" @@ -95,8 +98,8 @@ func main() { cfg := &config.CodeFlareOperatorConfiguration{ ClientConnection: &config.ClientConnection{ - QPS: pointer.Float32(50), - Burst: pointer.Int32(100), + QPS: ptr.To(float32(50)), + Burst: ptr.To(int32(100)), }, ControllerManager: config.ControllerManager{ Metrics: config.MetricsConfiguration{ @@ -110,7 +113,7 @@ func main() { LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, KubeRay: &config.KubeRayConfiguration{ - RayDashboardOAuthEnabled: pointer.Bool(true), + RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", }, } @@ -123,17 +126,21 @@ func main() { kubeClient, err := kubernetes.NewForConfig(kubeConfig) exitOnError(err, "unable to create Kubernetes client") - exitOnError(loadIntoOrCreate(ctx, kubeClient, namespaceOrDie(), configMapName, cfg), "unable to initialise configuration") + namespace := namespaceOrDie() - kubeConfig.Burst = int(pointer.Int32Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst))) - kubeConfig.QPS = pointer.Float32Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) + exitOnError(loadIntoOrCreate(ctx, kubeClient, namespace, configMapName, cfg), "unable to initialise configuration") + + kubeConfig.Burst = int(ptr.Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst))) + kubeConfig.QPS = ptr.Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) setupLog.V(2).Info("REST client", "qps", kubeConfig.QPS, "burst", kubeConfig.Burst) mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: cfg.Metrics.BindAddress, + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: cfg.Metrics.BindAddress, + }, HealthProbeBindAddress: cfg.Health.BindAddress, - LeaderElection: pointer.BoolDeref(cfg.LeaderElection.LeaderElect, false), + LeaderElection: ptr.Deref(cfg.LeaderElection.LeaderElect, false), LeaderElectionID: cfg.LeaderElection.ResourceName, LeaderElectionNamespace: cfg.LeaderElection.ResourceNamespace, LeaderElectionResourceLock: cfg.LeaderElection.ResourceLock, @@ -141,16 +148,30 @@ func main() { RetryPeriod: &cfg.LeaderElection.RetryPeriod.Duration, RenewDeadline: &cfg.LeaderElection.RenewDeadline.Duration, }) - exitOnError(err, "unable to start manager") + exitOnError(err, "unable to create manager") + + certsReady := make(chan struct{}) + exitOnError(setupCertManagement(mgr, namespace, certsReady), "unable to setup cert-controller") OpenShift := isOpenShift(ctx, kubeClient.DiscoveryClient) - if OpenShift { - // TODO: setup the RayCluster webhook on vanilla Kubernetes - exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") - } + go setupControllers(mgr, kubeClient, cfg, OpenShift, certsReady) + + exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") + exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") + + setupLog.Info("starting manager") + exitOnError(mgr.Start(ctx), "error running manager") +} + +func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration, OpenShift bool, certsReady chan struct{}) { + setupLog.Info("Waiting for certificate generation to complete") + <-certsReady + setupLog.Info("Certs ready") - ok, err := hasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") + + ok, err := hasAPIResourceForGVK(dc, rayv1.GroupVersion.WithKind("RayCluster")) if ok { rayClusterController := controllers.RayClusterReconciler{ Client: mgr.GetClient(), @@ -162,12 +183,37 @@ func main() { } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") } +} - exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") - exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;update - setupLog.Info("starting manager") - exitOnError(mgr.Start(ctx), "error running manager") +func setupCertManagement(mgr ctrl.Manager, namespace string, certsReady chan struct{}) error { + return cert.AddRotator(mgr, &cert.CertRotator{ + SecretKey: types.NamespacedName{ + Namespace: namespace, + Name: "codeflare-operator-webhook-server-cert", + }, + CertDir: "/tmp/k8s-webhook-server/serving-certs", + CAName: "codeflare", + CAOrganization: "openshift.ai", + DNSName: fmt.Sprintf("%s.%s.svc", "codeflare-operator-webhook-service", namespace), + IsReady: certsReady, + Webhooks: []cert.WebhookInfo{ + { + Type: cert.Validating, + Name: "codeflare-operator-validating-webhook-configuration", + }, + { + Type: cert.Mutating, + Name: "codeflare-operator-mutating-webhook-configuration", + }, + }, + // When the controller is running in the leader election mode, + // we expect webhook server will run in primary and secondary instance + RequireLeaderElection: false, + }) } func loadIntoOrCreate(ctx context.Context, client kubernetes.Interface, ns, name string, cfg *config.CodeFlareOperatorConfiguration) error { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 27a4b56b5..b4078119d 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -24,7 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -66,7 +66,7 @@ var _ webhook.CustomValidator = &rayClusterWebhook{} func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { rayCluster := obj.(*rayv1.RayCluster) - if !pointer.BoolDeref(w.Config.RayDashboardOAuthEnabled, true) { + if !ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { return nil } @@ -143,7 +143,7 @@ func validateOAuthProxyVolume(rayCluster *rayv1.RayCluster) field.ErrorList { func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList { var allErrors field.ErrorList - if pointer.BoolDeref(rayCluster.Spec.HeadGroupSpec.EnableIngress, false) { + if ptr.Deref(rayCluster.Spec.HeadGroupSpec.EnableIngress, false) { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "headGroupSpec", "enableIngress"), rayCluster.Spec.HeadGroupSpec.EnableIngress, From ffc555aa51615e7a3468e4797f578b4f73918747 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 09:55:45 +0200 Subject: [PATCH 173/369] Skip OAuth proxy validation when it's disabled --- pkg/controllers/raycluster_webhook.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index b4078119d..20ebe3fe2 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -104,9 +104,12 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r } allErrors = append(allErrors, validateIngress(rayCluster)...) - allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) - allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) - allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + + if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { + allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) + allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + } return warnings, allErrors.ToAggregate() } From cdf154a10b310859accf5508840dc6a68ddb83f4 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 10:13:48 +0200 Subject: [PATCH 174/369] Add missing validations on RayCluster creation --- pkg/controllers/raycluster_webhook.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 20ebe3fe2..b29794ee9 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -89,6 +89,12 @@ func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Obje allErrors = append(allErrors, validateIngress(rayCluster)...) + if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { + allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) + allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + } + return warnings, allErrors.ToAggregate() } From 2fe0a5257cbadb22210e20629e23eab1d774eb18 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 10:15:18 +0200 Subject: [PATCH 175/369] Delay webhook readiness until certicates are set up --- main.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 4228d1b4a..700f12e8f 100644 --- a/main.go +++ b/main.go @@ -18,8 +18,10 @@ package main import ( "context" + "errors" "flag" "fmt" + "net/http" "os" "strings" "time" @@ -153,18 +155,16 @@ func main() { certsReady := make(chan struct{}) exitOnError(setupCertManagement(mgr, namespace, certsReady), "unable to setup cert-controller") - OpenShift := isOpenShift(ctx, kubeClient.DiscoveryClient) + go setupControllers(mgr, kubeClient, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) - go setupControllers(mgr, kubeClient, cfg, OpenShift, certsReady) - - exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") - exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") + setupLog.Info("setting up health endpoints") + exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") setupLog.Info("starting manager") exitOnError(mgr.Start(ctx), "error running manager") } -func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration, OpenShift bool, certsReady chan struct{}) { +func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) { setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Certs ready") @@ -177,7 +177,7 @@ func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *co Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay, - IsOpenShift: OpenShift, + IsOpenShift: isOpenShift, } exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { @@ -216,6 +216,22 @@ func setupCertManagement(mgr ctrl.Manager, namespace string, certsReady chan str }) } +func setupProbeEndpoints(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { + err := mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping) + if err != nil { + return err + } + + return mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, func(req *http.Request) error { + select { + case <-certsReady: + return mgr.GetWebhookServer().StartedChecker()(req) + default: + return errors.New("certificates are not ready") + } + }) +} + func loadIntoOrCreate(ctx context.Context, client kubernetes.Interface, ns, name string, cfg *config.CodeFlareOperatorConfiguration) error { configMap, err := client.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { From 64de673a8a69d9703d2e928fa0b1b6001344738f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 11:14:14 +0200 Subject: [PATCH 176/369] Update OWNERS --- OWNERS | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/OWNERS b/OWNERS index c962aaaf2..955704b3e 100644 --- a/OWNERS +++ b/OWNERS @@ -1,20 +1,19 @@ approvers: - anishasthana - astefanutti + - ChristianZaccaria - jbusche - kpostoffice - - maxusmusti - sutaakar - tedhtchang reviewers: - anishasthana - astefanutti + - Bobbins228 - ChristianZaccaria - dimakis - Fiona-Waters - jbusche - kpostoffice - - maxusmusti - - MichaelClifford - sutaakar - tedhtchang From 9b99614c1ca72cb3200592362d2dda787f9677e2 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 16 Apr 2024 17:44:02 +0100 Subject: [PATCH 177/369] CVE fixes of High priority --- go.mod | 12 +- go.sum | 575 ++------------------------------------------------------- 2 files changed, 23 insertions(+), 564 deletions(-) diff --git a/go.mod b/go.mod index 06797c212..b49f20139 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,8 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 +replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 + require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -58,22 +60,22 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect + github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.4 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect diff --git a/go.sum b/go.sum index 8a496b7e9..0b63f5730 100644 --- a/go.sum +++ b/go.sum @@ -1,95 +1,27 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -101,227 +33,85 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= -github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= -github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.0 h1:/RvQ24k3TnNdfBSW0ou9EOi5jx2cX7zfE8n2nLKuiP0= -github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= -github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= -github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.0 h1:4k1tROTJctHotannFYzu77dY3bgtMRymQP7tXQjqpPk= -github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= -github.com/openshift-online/ocm-sdk-go v0.1.368 h1:qP+gkChV8WDwwpkUw1xUyjTXKdvrwyd70Gff2GMUSeU= -github.com/openshift-online/ocm-sdk-go v0.1.368/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= +github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= +github.com/openshift-online/ocm-sdk-go v0.1.411/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -331,457 +121,127 @@ github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= @@ -799,9 +259,6 @@ k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From 7caa6c750d464f448dade235a3f6c144af8a057c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 16 Apr 2024 18:01:17 +0100 Subject: [PATCH 178/369] Replace pgx/v4 for pgx/v5 --- go.mod | 3 +++ go.sum | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b49f20139..47e28213b 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apise replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 +replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 + require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -53,6 +55,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 0b63f5730..8d177bc09 100644 --- a/go.sum +++ b/go.sum @@ -66,14 +66,12 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= -github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -187,6 +185,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 07098c01cc3397573f19c5b8219397d0806d5a9d Mon Sep 17 00:00:00 2001 From: Kevin Postlethwait Date: Fri, 19 Apr 2024 10:20:10 -0400 Subject: [PATCH 179/369] add additional function which creates network policy (#498) * add additional function which creates network policy Signed-off-by: Kevin * Locate KubeRay deployment from DSCInitialization application namespace * tidy go mod files Signed-off-by: Kevin * err undefined Signed-off-by: Kevin * add dsci to schema instead of odh Signed-off-by: Kevin * add roles for creating nwp Signed-off-by: Kevin * add rule for metrics project Signed-off-by: Kevin * add missing dsci import Signed-off-by: Kevin --------- Signed-off-by: Kevin Co-authored-by: Antonin Stefanutti --- config/rbac/role.yaml | 18 ++ go.mod | 34 +-- go.sum | 253 +++++++++++++++++++---- main.go | 3 + pkg/controllers/raycluster_controller.go | 74 +++++++ 5 files changed, 329 insertions(+), 53 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 24ac70871..137098390 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -73,6 +73,14 @@ rules: - get - patch - update +- apiGroups: + - dscinitialization.opendatahub.io + resources: + - dscinitializations + verbs: + - get + - list + - watch - apiGroups: - networking.k8s.io resources: @@ -83,6 +91,16 @@ rules: - get - patch - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - patch + - update - apiGroups: - ray.io resources: diff --git a/go.mod b/go.mod index 47e28213b..57c47c462 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,27 @@ module github.com/project-codeflare/codeflare-operator go 1.20 require ( - github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/ginkgo/v2 v2.12.1 github.com/onsi/gomega v1.27.10 github.com/open-policy-agent/cert-controller v0.10.1 - github.com/openshift/api v0.0.0-20230213134911-7ba313770556 + github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 + github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 - k8s.io/api v0.28.1 - k8s.io/apimachinery v0.28.1 - k8s.io/client-go v0.28.1 - k8s.io/component-base v0.28.1 + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/client-go v11.0.0+incompatible + k8s.io/component-base v0.28.3 k8s.io/klog/v2 v2.100.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/controller-runtime v0.16.1 sigs.k8s.io/yaml v1.3.0 ) +replace k8s.io/client-go => k8s.io/client-go v0.28.3 + replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 @@ -37,8 +40,8 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -52,9 +55,9 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -64,6 +67,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect + github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect @@ -74,22 +78,22 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.13.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.1 // indirect + k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) diff --git a/go.sum b/go.sum index 8d177bc09..d67f55ff8 100644 --- a/go.sum +++ b/go.sum @@ -1,69 +1,124 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -75,41 +130,69 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= +github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= +github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= github.com/openshift-online/ocm-sdk-go v0.1.411/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y= -github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= -github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= +github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 h1:e3zIxk67/kiABxGFfFVECqJ4FcQRG5DPF8lgDV9f+MM= +github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= +github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= +github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -121,6 +204,7 @@ github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= @@ -129,15 +213,18 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -147,122 +234,212 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= -k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= -k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= -k8s.io/apiextensions-apiserver v0.28.1/go.mod h1:sVvrI+P4vxh2YBBcm8n2ThjNyzU4BQGilCQ/JAY5kGs= -k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= -k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= -k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= -k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= -k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= -k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= +k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= +k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs= +k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D1/LqPzj0bLjxLYWbdUtV0= k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/main.go b/main.go index 700f12e8f..134e71533 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "time" cert "github.com/open-policy-agent/cert-controller/pkg/rotator" + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" @@ -73,6 +74,8 @@ func init() { utilruntime.Must(rayv1.AddToScheme(scheme)) // OpenShift Route utilruntime.Must(routev1.Install(scheme)) + // ODH + utilruntime.Must(dsciv1.AddToScheme(scheme)) } func main() { diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index ba6ca84fe..4b08b209e 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -23,6 +23,7 @@ import ( "encoding/base64" "fmt" + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" @@ -32,7 +33,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" coreapply "k8s.io/client-go/applyconfigurations/core/v1" + metav1apply "k8s.io/client-go/applyconfigurations/meta/v1" v1 "k8s.io/client-go/applyconfigurations/meta/v1" + networkingapply "k8s.io/client-go/applyconfigurations/networking/v1" rbacapply "k8s.io/client-go/applyconfigurations/rbac/v1" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -83,6 +86,8 @@ var ( // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create; // +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create; +// +kubebuilder:rbac:groups=dscinitialization.opendatahub.io,resources=dscinitializations,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -205,6 +210,25 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } + // Locate the KubeRay operator deployment: + // - First try to get the ODH / RHOAI application namespace from the DSCInitialization + // - Or fallback to the well-known defaults + var kubeRayNamespaces []string + dsci := &dsciv1.DSCInitialization{} + err := r.Client.Get(ctx, client.ObjectKey{Name: "default-dsci"}, dsci) + if errors.IsNotFound(err) { + kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} + } else if err != nil { + return ctrl.Result{}, err + } else { + kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} + } + + _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredNetworkPolicy(cluster, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update NetworkPolicy") + } + return ctrl.Result{}, nil } @@ -337,6 +361,56 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, r *RayClusterReconciler) *cor // Create a Kubernetes secret to store the cookie secret } +func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingapply.NetworkPolicyApplyConfiguration { + return networkingapply.NetworkPolicy(cluster.Name, cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSpec(networkingapply.NetworkPolicySpec(). + WithPodSelector(metav1apply.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). + WithIngress( + networkingapply.NetworkPolicyIngressRule(). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(6379)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + ).WithFrom( + networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector()), + ), + networkingapply.NetworkPolicyIngressRule(). + WithFrom( + networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector(). + WithMatchLabels(map[string]string{"app.kubernetes.io/component": "kuberay-operator"})). + WithNamespaceSelector(metav1apply.LabelSelector(). + WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + WithKey(corev1.LabelMetadataName). + WithOperator(metav1.LabelSelectorOpIn). + WithValues(kubeRayNamespaces...)))). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + ), + networkingapply.NetworkPolicyIngressRule(). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + ). + WithFrom( + networkingapply.NetworkPolicyPeer().WithNamespaceSelector(metav1apply.LabelSelector(). + WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + WithKey(corev1.LabelMetadataName). + WithOperator(metav1.LabelSelectorOpIn). + WithValues("openshift-monitoring"))), + ), + networkingapply.NetworkPolicyIngressRule(). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + ), + ), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + // SetupWithManager sets up the controller with the Manager. func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { r.kubeClient = kubernetes.NewForConfigOrDie(mgr.GetConfig()) From e47db52752041f373f3a6536e4c0abacd257f080 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 17 Apr 2024 17:27:24 +0100 Subject: [PATCH 180/369] Added Mtls patch (cherry picked from commit de2de96fc88022df783b637ccb145d1d73ba66ff) --- config/rbac/role.yaml | 6 + main.go | 25 +++ pkg/config/config.go | 2 + pkg/controllers/raycluster_webhook.go | 217 +++++++++++++++++++++++++- pkg/controllers/support.go | 11 ++ 5 files changed, 254 insertions(+), 7 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 137098390..899e49157 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -44,6 +44,12 @@ rules: - subjectaccessreviews verbs: - create +- apiGroups: + - config.openshift.io + resources: + - ingresses + verbs: + - get - apiGroups: - "" resources: diff --git a/main.go b/main.go index 134e71533..95b0d5905 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,7 @@ import ( "sigs.k8s.io/yaml" routev1 "github.com/openshift/api/route/v1" + clientset "github.com/openshift/client-go/config/clientset/versioned" "github.com/project-codeflare/codeflare-operator/pkg/config" "github.com/project-codeflare/codeflare-operator/pkg/controllers" @@ -78,6 +79,8 @@ func init() { utilruntime.Must(dsciv1.AddToScheme(scheme)) } +// +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get; + func main() { var configMapName string flag.StringVar(&configMapName, "config", "codeflare-operator-config", @@ -120,6 +123,7 @@ func main() { KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", + MTLSEnabled: ptr.To(true), }, } @@ -158,6 +162,13 @@ func main() { certsReady := make(chan struct{}) exitOnError(setupCertManagement(mgr, namespace, certsReady), "unable to setup cert-controller") + if cfg.KubeRay.IngressDomain == "" { + configClient, err := clientset.NewForConfig(kubeConfig) + exitOnError(err, "unable to create Route Client Set") + cfg.KubeRay.IngressDomain, err = getClusterDomain(ctx, configClient) + exitOnError(err, cfg.KubeRay.IngressDomain) + } + go setupControllers(mgr, kubeClient, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) setupLog.Info("setting up health endpoints") @@ -335,3 +346,17 @@ func isOpenShift(ctx context.Context, dc discovery.DiscoveryInterface) bool { logger.Info("We detected being on Vanilla Kubernetes!") return false } + +func getClusterDomain(ctx context.Context, configClient *clientset.Clientset) (string, error) { + ingress, err := configClient.ConfigV1().Ingresses().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get Ingress object: %v", err) + } + + domain := ingress.Spec.Domain + if domain == "" { + return "", fmt.Errorf("domain is not set in the Ingress object") + } + + return domain, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 08e2579b9..af3bc3491 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -35,6 +35,8 @@ type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` IngressDomain string `json:"ingressDomain"` + + MTLSEnabled *bool `json:"mTLSEnabled,omitempty"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index b29794ee9..1e5b0b870 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "strconv" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -36,6 +37,7 @@ import ( const ( oauthProxyContainerName = "oauth-proxy" oauthProxyVolumeName = "proxy-tls-secret" + initContainerName = "create-cert" ) // log is for logging in this package. @@ -66,17 +68,44 @@ var _ webhook.CustomValidator = &rayClusterWebhook{} func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { rayCluster := obj.(*rayv1.RayCluster) - if !ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { - return nil - } + if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { + rayclusterlog.V(2).Info("Adding OAuth sidecar container") + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName)) - rayclusterlog.V(2).Info("Adding OAuth sidecar container") + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName)) - rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName)) + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" + } - rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName)) + if ptr.Deref(w.Config.MTLSEnabled, true) { + rayclusterlog.V(2).Info("Adding create-cert Init Containers") + // HeadGroupSpec // + // Append the list of environment variables for the ray-head container + for _, envVar := range envVarList() { + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) + } + + // Append the create-cert Init Container + rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain), withContainerName(initContainerName)) + + // Append the CA volumes + for _, caVol := range caVolumes(rayCluster) { + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) + } + // WorkerGroupSpec // + // Append the list of environment variables for the worker container + for _, envVar := range envVarList() { + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) + } + + // Append the CA volumes + for _, caVol := range caVolumes(rayCluster) { + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) + } + // Append the create-cert Init Container + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), withContainerName(initContainerName)) - rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" + } return nil } @@ -117,6 +146,14 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) } + // Init Container related errors + if ptr.Deref(w.Config.MTLSEnabled, true) { + allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w)...) + allErrors = append(allErrors, validateWorkerInitContainer(rayCluster)...) + allErrors = append(allErrors, validateHeadEnvVars(rayCluster)...) + allErrors = append(allErrors, validateWorkerEnvVars(rayCluster)...) + allErrors = append(allErrors, validateCaVolumes(rayCluster)...) + } return warnings, allErrors.ToAggregate() } @@ -225,3 +262,169 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume { }, } } + +func initCaVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "ca-vol", + MountPath: "/home/ray/workspace/ca", + ReadOnly: true, + }, + { + Name: "server-cert", + MountPath: "/home/ray/workspace/tls", + ReadOnly: false, + }, + } +} + +func envVarList() []corev1.EnvVar { + return []corev1.EnvVar{ + { + Name: "MY_POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: "RAY_USE_TLS", + Value: "1", + }, + { + Name: "RAY_TLS_SERVER_CERT", + Value: "/home/ray/workspace/tls/server.crt", + }, + { + Name: "RAY_TLS_SERVER_KEY", + Value: "/home/ray/workspace/tls/server.key", + }, + { + Name: "RAY_TLS_CA_CERT", + Value: "/home/ray/workspace/tls/ca.crt", + }, + } +} + +func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume { + return []corev1.Volume{ + { + Name: "ca-vol", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: `ca-secret-` + rayCluster.Name, + }, + }, + }, + { + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } +} + +func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) corev1.Container { + rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + domain + // Service name for basic interactive + svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc" + + initContainerHead := corev1.Container{ + Name: "create-cert", + Image: "quay.io/project-codeflare/ray:latest-py39-cu118", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: initCaVolumeMounts(), + } + return initContainerHead +} + +func rayWorkerInitContainer() corev1.Container { + initContainerWorker := corev1.Container{ + Name: "create-cert", + Image: "quay.io/project-codeflare/ray:latest-py39-cu118", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: initCaVolumeMounts(), + } + return initContainerWorker +} + +func validateHeadInitContainer(rayCluster *rayv1.RayCluster, w *rayClusterWebhook) field.ErrorList { + var allErrors field.ErrorList + + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain), byContainerName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "initContainers"), + "create-cert Init Container is immutable"); err != nil { + allErrors = append(allErrors, err) + } + + return allErrors +} + +func validateWorkerInitContainer(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), byContainerName, + field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "initContainers"), + "create-cert Init Container is immutable"); err != nil { + allErrors = append(allErrors, err) + } + + return allErrors +} + +func validateCaVolumes(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + for _, caVol := range caVolumes(rayCluster) { + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVol, byVolumeName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "volumes"), + "ca-vol and server-cert Secret volumes are immutable"); err != nil { + allErrors = append(allErrors, err) + } + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, byVolumeName, + field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "volumes"), + "ca-vol and server-cert Secret volumes are immutable"); err != nil { + allErrors = append(allErrors, err) + } + } + + return allErrors +} + +func validateHeadEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + for _, envVar := range envVarList() { + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, envVar, byEnvVarName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "containers", strconv.Itoa(0), "env"), + "RAY_TLS related environment variables are immutable"); err != nil { + allErrors = append(allErrors, err) + } + } + + return allErrors +} + +func validateWorkerEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + for _, envVar := range envVarList() { + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, byEnvVarName, + field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "containers", strconv.Itoa(0), "env"), + "RAY_TLS related environment variables are immutable"); err != nil { + allErrors = append(allErrors, err) + } + } + + return allErrors +} diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 0efbc7475..0ea8f424d 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -140,3 +140,14 @@ func withVolumeName(name string) compare[corev1.Volume] { return v1.Name == name } } + +var byEnvVarName = compare[corev1.EnvVar]( + func(e1, e2 corev1.EnvVar) bool { + return e1.Name == e2.Name + }) + +func withEnvVarName(name string) compare[corev1.EnvVar] { + return func(e1, e2 corev1.EnvVar) bool { + return e1.Name == name + } +} From f5be3340d7fde1e4ebdf96b527607d94cd47efb9 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Fri, 19 Apr 2024 11:24:02 +0100 Subject: [PATCH 181/369] Added Ingress domain to validateHeadInitContainer --- pkg/controllers/raycluster_webhook.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 1e5b0b870..9362afe56 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -148,7 +148,7 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r // Init Container related errors if ptr.Deref(w.Config.MTLSEnabled, true) { - allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w)...) + allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w.Config.IngressDomain)...) allErrors = append(allErrors, validateWorkerInitContainer(rayCluster)...) allErrors = append(allErrors, validateHeadEnvVars(rayCluster)...) allErrors = append(allErrors, validateWorkerEnvVars(rayCluster)...) @@ -358,10 +358,10 @@ func rayWorkerInitContainer() corev1.Container { return initContainerWorker } -func validateHeadInitContainer(rayCluster *rayv1.RayCluster, w *rayClusterWebhook) field.ErrorList { +func validateHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) field.ErrorList { var allErrors field.ErrorList - if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain), byContainerName, + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, domain), byContainerName, field.NewPath("spec", "headGroupSpec", "template", "spec", "initContainers"), "create-cert Init Container is immutable"); err != nil { allErrors = append(allErrors, err) From 0bf2a4fa1c17b081400a25ff0dc115d69d79d673 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 14:48:58 +0200 Subject: [PATCH 182/369] Generate RayCluster CA certificate Secret --- main.go | 2 +- pkg/controllers/raycluster_controller.go | 197 ++++++++++++++++------- 2 files changed, 140 insertions(+), 59 deletions(-) diff --git a/main.go b/main.go index 95b0d5905..31afbcbbe 100644 --- a/main.go +++ b/main.go @@ -79,7 +79,7 @@ func init() { utilruntime.Must(dsciv1.AddToScheme(scheme)) } -// +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get; +// +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get func main() { var configMapName string diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 4b08b209e..b30287553 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -19,9 +19,16 @@ package controllers import ( "context" "crypto/rand" + "crypto/rsa" "crypto/sha1" + "crypto/x509" + "crypto/x509/pkix" "encoding/base64" + "encoding/pem" "fmt" + "math/big" + rand2 "math/rand" + "time" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -32,18 +39,18 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - coreapply "k8s.io/client-go/applyconfigurations/core/v1" - metav1apply "k8s.io/client-go/applyconfigurations/meta/v1" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" - networkingapply "k8s.io/client-go/applyconfigurations/networking/v1" - rbacapply "k8s.io/client-go/applyconfigurations/rbac/v1" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + rbacv1ac "k8s.io/client-go/applyconfigurations/rbac/v1" "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" routev1 "github.com/openshift/api/route/v1" - routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" + routev1ac "github.com/openshift/client-go/route/applyconfigurations/route/v1" routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -144,6 +151,26 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + if isMTLSEnabled(r.Config) { + caSecretName := caSecretNameFromCluster(cluster) + _, err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Get(ctx, caSecretName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + key, cert, err := generateCACertificate() + if err != nil { + logger.Error(err, "Failed to generate CA certificate") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to create CA Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } else if err != nil { + logger.Error(err, "Failed to get CA Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } + if cluster.Status.State != "suspended" && isRayDashboardOAuthEnabled(r.Config) && r.IsOpenShift { logger.Info("Creating OAuth Objects") _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) @@ -152,7 +179,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(cluster, r.CookieSalt), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to create OAuth Secret") return ctrl.Result{RequeueAfter: requeueTime}, err @@ -244,28 +271,29 @@ func getIngressHost(cfg *config.KubeRayConfiguration, cluster *rayv1.RayCluster, } func isRayDashboardOAuthEnabled(cfg *config.KubeRayConfiguration) bool { - if cfg != nil && cfg.RayDashboardOAuthEnabled != nil { - return *cfg.RayDashboardOAuthEnabled - } - return true + return cfg == nil || ptr.Deref(cfg.RayDashboardOAuthEnabled, true) +} + +func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool { + return cfg == nil || ptr.Deref(cfg.MTLSEnabled, true) } func crbNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} } -func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.ClusterRoleBindingApplyConfiguration { - return rbacapply.ClusterRoleBinding( +func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration { + return rbacv1ac.ClusterRoleBinding( crbNameFromCluster(cluster)). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSubjects( - rbacapply.Subject(). + rbacv1ac.Subject(). WithKind("ServiceAccount"). WithName(oauthServiceAccountNameFromCluster(cluster)). WithNamespace(cluster.Namespace), ). WithRoleRef( - rbacapply.RoleRef(). + rbacv1ac.RoleRef(). WithAPIGroup("rbac.authorization.k8s.io"). WithKind("ClusterRole"). WithName("system:auth-delegator"), @@ -276,8 +304,8 @@ func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-oauth-proxy" } -func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountApplyConfiguration { - return coreapply.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace). +func desiredServiceAccount(cluster *rayv1.RayCluster) *corev1ac.ServiceAccountApplyConfiguration { + return corev1ac.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithAnnotations(map[string]string{ "serviceaccounts.openshift.io/oauth-redirectreference.first": "" + @@ -285,7 +313,7 @@ func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountA `"reference":{"kind":"Route","name":"` + dashboardNameFromCluster(cluster) + `"}}`, }). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } @@ -297,19 +325,19 @@ func rayClientNameFromCluster(cluster *rayv1.RayCluster) string { return "rayclient-" + cluster.Name } -func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { - return routeapply.Route(dashboardNameFromCluster(cluster), cluster.Namespace). +func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfiguration { + return routev1ac.Route(dashboardNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). - WithSpec(routeapply.RouteSpec(). - WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). - WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))). - WithTLS(routeapply.TLSConfig(). + WithSpec(routev1ac.RouteSpec(). + WithTo(routev1ac.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). + WithPort(routev1ac.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))). + WithTLS(routev1ac.TLSConfig(). WithInsecureEdgeTerminationPolicy(routev1.InsecureEdgeTerminationPolicyRedirect). WithTermination(routev1.TLSTerminationReencrypt), ), ). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } @@ -321,14 +349,14 @@ func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string { return cluster.Name + "-proxy-tls-secret" } -func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfiguration { - return coreapply.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace). +func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfiguration { + return corev1ac.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithAnnotations(map[string]string{"service.beta.openshift.io/serving-cert-secret-name": oauthServiceTLSSecretName(cluster)}). WithSpec( - coreapply.ServiceSpec(). + corev1ac.ServiceSpec(). WithPorts( - coreapply.ServicePort(). + corev1ac.ServicePort(). WithName(oAuthServicePortName). WithPort(oAuthServicePort). WithTargetPort(intstr.FromString(oAuthServicePortName)). @@ -337,7 +365,7 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfi WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), ). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } @@ -346,68 +374,121 @@ func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string { } // desiredOAuthSecret defines the desired OAuth secret object -func desiredOAuthSecret(cluster *rayv1.RayCluster, r *RayClusterReconciler) *coreapply.SecretApplyConfiguration { +func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac.SecretApplyConfiguration { // Generate the cookie secret for the OAuth proxy hasher := sha1.New() // REVIEW is SHA1 okay here? - hasher.Write([]byte(cluster.Name + r.CookieSalt)) + hasher.Write([]byte(cluster.Name + cookieSalt)) cookieSecret := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - return coreapply.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). + return corev1ac.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithStringData(map[string]string{"cookie_secret": cookieSecret}). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) - // Create a Kubernetes secret to store the cookie secret } -func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingapply.NetworkPolicyApplyConfiguration { - return networkingapply.NetworkPolicy(cluster.Name, cluster.Namespace). +func caSecretNameFromCluster(cluster *rayv1.RayCluster) string { + return "ca-secret-" + cluster.Name +} + +func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.SecretApplyConfiguration { + return corev1ac.Secret(caSecretNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithData(map[string][]byte{ + corev1.TLSPrivateKeyKey: key, + corev1.TLSCertKey: cert, + }). + WithOwnerReferences(metav1ac.OwnerReference(). + WithUID(cluster.UID). + WithName(cluster.Name). + WithKind(cluster.Kind). + WithAPIVersion(cluster.APIVersion)) +} + +func generateCACertificate() ([]byte, []byte, error) { + serialNumber := big.NewInt(rand2.Int63()) + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"OpenShift AI"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + certPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + privateKeyBytes := x509.MarshalPKCS1PrivateKey(certPrivateKey) + privateKeyPem := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + }, + ) + certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivateKey.PublicKey, certPrivateKey) + certPem := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + return privateKeyPem, certPem, nil +} + +func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { + return networkingv1ac.NetworkPolicy(cluster.Name, cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). - WithSpec(networkingapply.NetworkPolicySpec(). - WithPodSelector(metav1apply.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). + WithSpec(networkingv1ac.NetworkPolicySpec(). + WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). WithIngress( - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(6379)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(6379)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), ).WithFrom( - networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector()), + networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector()), ), - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithFrom( - networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector(). + networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector(). WithMatchLabels(map[string]string{"app.kubernetes.io/component": "kuberay-operator"})). - WithNamespaceSelector(metav1apply.LabelSelector(). - WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + WithNamespaceSelector(metav1ac.LabelSelector(). + WithMatchExpressions(metav1ac.LabelSelectorRequirement(). WithKey(corev1.LabelMetadataName). WithOperator(metav1.LabelSelectorOpIn). WithValues(kubeRayNamespaces...)))). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), ), - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), ). WithFrom( - networkingapply.NetworkPolicyPeer().WithNamespaceSelector(metav1apply.LabelSelector(). - WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + networkingv1ac.NetworkPolicyPeer().WithNamespaceSelector(metav1ac.LabelSelector(). + WithMatchExpressions(metav1ac.LabelSelectorRequirement(). WithKey(corev1.LabelMetadataName). WithOperator(metav1.LabelSelectorOpIn). WithValues("openshift-monitoring"))), ), - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), ), ), ). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } From 70a72b95f68d6999bea4c190caf9aeaa57079e4b Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 15:13:12 +0200 Subject: [PATCH 183/369] Update RayCluster CA Secret if it exists --- pkg/controllers/raycluster_controller.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index b30287553..c7ec3b3e7 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -153,7 +153,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) if isMTLSEnabled(r.Config) { caSecretName := caSecretNameFromCluster(cluster) - _, err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Get(ctx, caSecretName, metav1.GetOptions{}) + caSecret, err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Get(ctx, caSecretName, metav1.GetOptions{}) if errors.IsNotFound(err) { key, cert, err := generateCACertificate() if err != nil { @@ -162,12 +162,20 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { - logger.Error(err, "Failed to create CA Secret") + logger.Error(err, "Failed to apply CA Secret") return ctrl.Result{RequeueAfter: requeueTime}, err } } else if err != nil { logger.Error(err, "Failed to get CA Secret") return ctrl.Result{RequeueAfter: requeueTime}, err + } else { + key := caSecret.Data[corev1.TLSPrivateKeyKey] + cert := caSecret.Data[corev1.TLSCertKey] + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to apply CA Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } } } @@ -434,6 +442,10 @@ func generateCACertificate() ([]byte, []byte, error) { }, ) certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivateKey.PublicKey, certPrivateKey) + if err != nil { + return nil, nil, err + } + certPem := pem.EncodeToMemory(&pem.Block{ Type: "CERTIFICATE", Bytes: certBytes, From 46d4bab81f04cd14f0b210b080fa8310e25be8ee Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 15:17:02 +0200 Subject: [PATCH 184/369] Fix RayCluster CA Secret keys --- pkg/controllers/raycluster_controller.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index c7ec3b3e7..a3033c84a 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -75,6 +75,9 @@ const ( oAuthServicePortName = "oauth-proxy" ingressServicePortName = "dashboard" logRequeueing = "requeueing" + + CAPrivateKeyKey = "ca.key" + CACertKey = "ca.crt" ) var ( @@ -169,8 +172,8 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "Failed to get CA Secret") return ctrl.Result{RequeueAfter: requeueTime}, err } else { - key := caSecret.Data[corev1.TLSPrivateKeyKey] - cert := caSecret.Data[corev1.TLSCertKey] + key := caSecret.Data[CAPrivateKeyKey] + cert := caSecret.Data[CACertKey] _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to apply CA Secret") @@ -404,8 +407,8 @@ func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.Secr return corev1ac.Secret(caSecretNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithData(map[string][]byte{ - corev1.TLSPrivateKeyKey: key, - corev1.TLSCertKey: cert, + CAPrivateKeyKey: key, + CACertKey: cert, }). WithOwnerReferences(metav1ac.OwnerReference(). WithUID(cluster.UID). From 0f7519806273774dfd6b5e3b4f402f42c8812aff Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 16:33:05 +0200 Subject: [PATCH 185/369] Update RayCluster CA certificate common and issuer names --- pkg/controllers/raycluster_controller.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index a3033c84a..a60cda5ee 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -422,7 +422,10 @@ func generateCACertificate() ([]byte, []byte, error) { cert := &x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ - Organization: []string{"OpenShift AI"}, + CommonName: "root-ca", + }, + Issuer: pkix.Name{ + CommonName: "root-ca", }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(1, 0, 0), From 1a0141e5d0db20597800f5326949d9ab488f4108 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 16:34:05 +0200 Subject: [PATCH 186/369] Fix missing certificate volume mounts to RayCluster --- pkg/controllers/raycluster_webhook.go | 25 +++++++++++++++++++------ pkg/controllers/support.go | 5 +++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 9362afe56..eb530697e 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -79,7 +79,8 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err if ptr.Deref(w.Config.MTLSEnabled, true) { rayclusterlog.V(2).Info("Adding create-cert Init Containers") - // HeadGroupSpec // + // HeadGroupSpec + // Append the list of environment variables for the ray-head container for _, envVar := range envVarList() { rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) @@ -92,7 +93,14 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err for _, caVol := range caVolumes(rayCluster) { rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) } - // WorkerGroupSpec // + + // Append the certificate volume mounts + for _, mount := range certVolumeMounts() { + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].VolumeMounts = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].VolumeMounts, mount, byVolumeMountName) + } + + // WorkerGroupSpec + // Append the list of environment variables for the worker container for _, envVar := range envVarList() { rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) @@ -102,9 +110,14 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err for _, caVol := range caVolumes(rayCluster) { rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) } + + // Append the certificate volume mounts + for _, mount := range certVolumeMounts() { + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].VolumeMounts = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].VolumeMounts, mount, byVolumeMountName) + } + // Append the create-cert Init Container rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), withContainerName(initContainerName)) - } return nil @@ -263,7 +276,7 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume { } } -func initCaVolumeMounts() []corev1.VolumeMount { +func certVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "ca-vol", @@ -339,7 +352,7 @@ func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) corev1.Co "-c", `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, }, - VolumeMounts: initCaVolumeMounts(), + VolumeMounts: certVolumeMounts(), } return initContainerHead } @@ -353,7 +366,7 @@ func rayWorkerInitContainer() corev1.Container { "-c", `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, }, - VolumeMounts: initCaVolumeMounts(), + VolumeMounts: certVolumeMounts(), } return initContainerWorker } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 0ea8f424d..d208b52d3 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -141,6 +141,11 @@ func withVolumeName(name string) compare[corev1.Volume] { } } +var byVolumeMountName = compare[corev1.VolumeMount]( + func(v1, v2 corev1.VolumeMount) bool { + return v1.Name == v2.Name + }) + var byEnvVarName = compare[corev1.EnvVar]( func(e1, e2 corev1.EnvVar) bool { return e1.Name == e2.Name From 1c8a64d78b48458aed7a01d989e1aa35bc50c5ff Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 19 Apr 2024 11:02:57 -0400 Subject: [PATCH 187/369] add entry for rayclient to allow all connections Signed-off-by: Kevin --- pkg/controllers/raycluster_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index a60cda5ee..6377e27b9 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -502,6 +502,7 @@ func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) networkingv1ac.NetworkPolicyIngressRule(). WithPorts( networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), ), ), ). From 0c2aa476226627af731ebc584b667e0868bd00df Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 19 Apr 2024 11:34:16 -0400 Subject: [PATCH 188/369] add cfg check for mtls before enabling access to client Signed-off-by: Kevin --- pkg/controllers/raycluster_controller.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 6377e27b9..41464af20 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -262,7 +262,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} } - _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredNetworkPolicy(cluster, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update NetworkPolicy") } @@ -460,7 +460,13 @@ func generateCACertificate() ([]byte, []byte, error) { return privateKeyPem, certPem, nil } -func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { +func desiredNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { + allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{ + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + } + if ptr.Deref(cfg.MTLSEnabled, true) { + allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001))) + } return networkingv1ac.NetworkPolicy(cluster.Name, cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(networkingv1ac.NetworkPolicySpec(). @@ -501,8 +507,7 @@ func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) ), networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + allSecuredPorts..., ), ), ). From fa700faab260065328d571e5fdf4c3e60d58eeb8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Fri, 19 Apr 2024 18:28:32 +0200 Subject: [PATCH 189/369] Update dependency versions for release v1.4.0 (#540) Co-authored-by: codeflare-machine-account --- Makefile | 2 +- README.md | 6 +++--- config/manager/params.env | 2 +- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index cb8cd533c..99e633247 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.0.0 +KUBERAY_VERSION ?= v1.1.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/README.md b/README.md index 423fb9745..7531cfdd1 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.3.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.1) | -| CodeFlare-SDK | [v0.15.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.1) | -| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | +| CodeFlare Operator | [v1.4.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.0) | +| CodeFlare-SDK | [v0.16.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.0) | +| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 8843b5e31..6ac22e106 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.1 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.0 namespace=opendatahub diff --git a/go.mod b/go.mod index 57c47c462..f8a0da50a 100644 --- a/go.mod +++ b/go.mod @@ -10,15 +10,15 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a - github.com/ray-project/kuberay/ray-operator v1.0.0 + github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 - k8s.io/api v0.28.3 - k8s.io/apimachinery v0.28.3 + k8s.io/api v0.28.4 + k8s.io/apimachinery v0.28.4 k8s.io/client-go v11.0.0+incompatible - k8s.io/component-base v0.28.3 + k8s.io/component-base v0.28.4 k8s.io/klog/v2 v2.100.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b - sigs.k8s.io/controller-runtime v0.16.1 + sigs.k8s.io/controller-runtime v0.16.3 sigs.k8s.io/yaml v1.3.0 ) @@ -92,7 +92,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.2 // indirect + k8s.io/apiextensions-apiserver v0.28.4 // indirect k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index d67f55ff8..10d47e582 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqSc github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= -github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= +github.com/ray-project/kuberay/ray-operator v1.1.0 h1:kCbPZG6SDv11A1F1VR0oG1oCTjrKUuZiMDsyJqmNr3g= +github.com/ray-project/kuberay/ray-operator v1.1.0/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= @@ -402,18 +402,18 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= -k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= -k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= -k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= +k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= +k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= +k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= +k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= -k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= +k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= -k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo= +k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -431,8 +431,8 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= -sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= From 1e761579cbb663c33e2febfcecea77f3a21453a8 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 22 Apr 2024 10:22:23 +0200 Subject: [PATCH 190/369] Fix OLM PR check by creating ConfigMap disabling oauth --- .github/workflows/olm_tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index abce29d3d..b13f2f894 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -75,6 +75,9 @@ jobs: - name: Deploy latest released CodeFlare operator from OLM id: deploy run: | + echo Create the CodeFlare operator ConfigMap + kubectl apply -n '${{ env.SUBSCRIPTION_NAMESPACE }}' -f config/e2e/config.yaml + echo Deploying CodeFlare operator using Subscription envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml From 12b72cbde4938cb4039f073cb01d7243121dc25c Mon Sep 17 00:00:00 2001 From: Mark Campbell Date: Wed, 24 Apr 2024 10:52:08 +0100 Subject: [PATCH 191/369] Altered network policy to allow all trafic for head & worker pods (#544) * Altered network policy to allow all trafic for head & worker pods * Added Worker Network Policy * Review changes * Re-added client port to NWP * Added Dashboard port & renaming function * Added empty label selector * Update pkg/controllers/raycluster_controller.go Co-authored-by: Antonin Stefanutti --------- Co-authored-by: Antonin Stefanutti --- pkg/controllers/raycluster_controller.go | 34 +++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 41464af20..bf58426bc 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -262,7 +262,12 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} } - _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredHeadNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update NetworkPolicy") + } + + _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredWorkersNetworkPolicy(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update NetworkPolicy") } @@ -459,24 +464,41 @@ func generateCACertificate() ([]byte, []byte, error) { return privateKeyPem, certPem, nil } - -func desiredNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { +func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration { + return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSpec(networkingv1ac.NetworkPolicySpec(). + WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})). + WithIngress( + networkingv1ac.NetworkPolicyIngressRule(). + WithFrom( + networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name})), + ), + ), + ). + WithOwnerReferences( + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} +func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{ networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), } if ptr.Deref(cfg.MTLSEnabled, true) { allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001))) } - return networkingv1ac.NetworkPolicy(cluster.Name, cluster.Namespace). + return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(networkingv1ac.NetworkPolicySpec(). WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). WithIngress( + networkingv1ac.NetworkPolicyIngressRule(). + WithFrom( + networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name})), + ), networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(6379)), networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), ).WithFrom( networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector()), From 460626933270d6c8c63cab177d2b802ff1615b94 Mon Sep 17 00:00:00 2001 From: oksanabaza Date: Wed, 17 Apr 2024 12:43:44 +0100 Subject: [PATCH 192/369] Fix Unknown build info in CodeFlare operator log --- Dockerfile | 3 ++- Makefile | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f5c51c3f5..2d28eb026 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum +COPY ./Makefile ./Makefile RUN go mod download # Copy the Go sources @@ -13,7 +14,7 @@ COPY pkg/ pkg/ # Build USER root -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -tags strictfipsruntime -a -o manager main.go +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 make go-build-for-image FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / diff --git a/Makefile b/Makefile index 99e633247..dbaa9a208 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,15 @@ build: fmt vet ## Build manager binary. " \ -o bin/manager main.go +.PHONY: go-build-for-image +go-build-for-image: fmt vet ## Build manager binary. + go build \ + -ldflags " \ + -X 'main.OperatorVersion=$(BUILD_VERSION)' \ + -X 'main.BuildDate=$(BUILD_DATE)' \ + " \ + -tags strictfipsruntime -a -o manager main.go + .PHONY: run run: manifests fmt vet ## Run a controller from your host. go run ./main.go From 0f4ef72f1655e415bd35764c0b858e0d896093c8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 26 Apr 2024 18:04:50 +0200 Subject: [PATCH 193/369] Watch for RayCluster CRD then start RayCluster controller --- go.mod | 4 +-- main.go | 103 ++++++++++++++++++++++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index f8a0da50a..047755ad9 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,9 @@ require ( github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 k8s.io/api v0.28.4 + k8s.io/apiextensions-apiserver v0.28.4 k8s.io/apimachinery v0.28.4 k8s.io/client-go v11.0.0+incompatible k8s.io/component-base v0.28.4 @@ -78,7 +80,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.18.0 // indirect @@ -92,7 +93,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.4 // indirect k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect diff --git a/main.go b/main.go index 31afbcbbe..ed27cda2c 100644 --- a/main.go +++ b/main.go @@ -30,19 +30,24 @@ import ( dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" + "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + retrywatch "k8s.io/client-go/tools/watch" configv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/klog/v2" "k8s.io/utils/ptr" @@ -169,34 +174,83 @@ func main() { exitOnError(err, cfg.KubeRay.IngressDomain) } - go setupControllers(mgr, kubeClient, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) - setupLog.Info("setting up health endpoints") exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") + setupLog.Info("setting up RayCluster controller") + exitOnError(waitForRayClusterAPIandSetupController(ctx, mgr, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady), "unable to setup RayCluster controller") + setupLog.Info("starting manager") exitOnError(mgr.Start(ctx), "error running manager") } -func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) { +func setupRayClusterController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) error { setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Certs ready") - exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") + err := controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay) + if err != nil { + return err + } - ok, err := hasAPIResourceForGVK(dc, rayv1.GroupVersion.WithKind("RayCluster")) - if ok { - rayClusterController := controllers.RayClusterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Config: cfg.KubeRay, - IsOpenShift: isOpenShift, - } - exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") - } else if err != nil { - exitOnError(err, "Could not determine if RayCluster CR present on cluster.") + rayClusterController := controllers.RayClusterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Config: cfg.KubeRay, + IsOpenShift: isOpenShift, + } + return rayClusterController.SetupWithManager(mgr) +} + +func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) error { + crdClient, err := apiextensionsclientset.NewForConfig(mgr.GetConfig()) + if err != nil { + return err + } + crdList, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + if slices.ContainsFunc(crdList.Items, func(crd apiextensionsv1.CustomResourceDefinition) bool { + return crd.Name == "rayclusters.ray.io" + }) { + return setupRayClusterController(mgr, cfg, isOpenShift, certsReady) + } + + retryWatcher, err := retrywatch.NewRetryWatcher(crdList.ResourceVersion, &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return crdClient.ApiextensionsV1().CustomResourceDefinitions().Watch(ctx, metav1.ListOptions{}) + }, + }) + if err != nil { + return err } + + go func() { + defer retryWatcher.Stop() + for { + select { + case <-ctx.Done(): + return + case event := <-retryWatcher.ResultChan(): + switch event.Type { + case watch.Error: + exitOnError(apierrors.FromObject(event.Object), "error watching for RayCluster API") + + case watch.Added: + setupLog.Info("RayCluster API installed, setting up controller") + exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") + return + } + } + } + }() + + return nil } // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update @@ -289,23 +343,6 @@ func createConfigMap(ctx context.Context, client kubernetes.Interface, ns, name return err } -func hasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) { - gv, kind := gvk.ToAPIVersionAndKind() - if resources, err := dc.ServerResourcesForGroupVersion(gv); err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return false, err - } else { - for _, res := range resources.APIResources { - if res.Kind == kind { - return true, nil - } - } - } - return false, nil -} - func namespaceOrDie() string { // This way assumes you've set the NAMESPACE environment variable either manually, when running // the operator standalone, or using the downward API, when running the operator in-cluster. From 14a55860dcc80a7b32202f2ec4335a18870a3180 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 26 Apr 2024 18:35:42 +0200 Subject: [PATCH 194/369] Add required RBAC to list and watch CRDs --- config/rbac/role.yaml | 8 ++++++++ main.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 899e49157..bb419783c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -32,6 +32,14 @@ rules: - list - update - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch - apiGroups: - authentication.k8s.io resources: diff --git a/main.go b/main.go index ed27cda2c..8a1c86269 100644 --- a/main.go +++ b/main.go @@ -203,6 +203,8 @@ func setupRayClusterController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorCo return rayClusterController.SetupWithManager(mgr) } +// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch + func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) error { crdClient, err := apiextensionsclientset.NewForConfig(mgr.GetConfig()) if err != nil { From 54a7a9004827f242380c16fc70d18dbe4af73fb3 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 26 Apr 2024 18:49:55 +0200 Subject: [PATCH 195/369] Always setup RayCluster controller in a routine --- main.go | 52 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/main.go b/main.go index 8a1c86269..c71fa5b6b 100644 --- a/main.go +++ b/main.go @@ -178,7 +178,7 @@ func main() { exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") setupLog.Info("setting up RayCluster controller") - exitOnError(waitForRayClusterAPIandSetupController(ctx, mgr, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady), "unable to setup RayCluster controller") + go waitForRayClusterAPIandSetupController(ctx, mgr, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) setupLog.Info("starting manager") exitOnError(mgr.Start(ctx), "error running manager") @@ -205,19 +205,17 @@ func setupRayClusterController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorCo // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch -func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) error { +func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) { crdClient, err := apiextensionsclientset.NewForConfig(mgr.GetConfig()) - if err != nil { - return err - } + exitOnError(err, "unable to create CRD client") + crdList, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) - if err != nil { - return err - } + exitOnError(err, "unable to list CRDs") + if slices.ContainsFunc(crdList.Items, func(crd apiextensionsv1.CustomResourceDefinition) bool { return crd.Name == "rayclusters.ray.io" }) { - return setupRayClusterController(mgr, cfg, isOpenShift, certsReady) + exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") } retryWatcher, err := retrywatch.NewRetryWatcher(crdList.ResourceVersion, &cache.ListWatch{ @@ -228,31 +226,25 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage return crdClient.ApiextensionsV1().CustomResourceDefinitions().Watch(ctx, metav1.ListOptions{}) }, }) - if err != nil { - return err - } + exitOnError(err, "unable to create retry watcher") - go func() { - defer retryWatcher.Stop() - for { - select { - case <-ctx.Done(): + defer retryWatcher.Stop() + for { + select { + case <-ctx.Done(): + return + case event := <-retryWatcher.ResultChan(): + switch event.Type { + case watch.Error: + exitOnError(apierrors.FromObject(event.Object), "error watching for RayCluster API") + + case watch.Added: + setupLog.Info("RayCluster API installed, setting up controller") + exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") return - case event := <-retryWatcher.ResultChan(): - switch event.Type { - case watch.Error: - exitOnError(apierrors.FromObject(event.Object), "error watching for RayCluster API") - - case watch.Added: - setupLog.Info("RayCluster API installed, setting up controller") - exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") - return - } } } - }() - - return nil + } } // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update From 146b88dfe61663e4841ee5dd7f2146050e14572c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 26 Apr 2024 19:31:41 +0200 Subject: [PATCH 196/369] Start RayCluster controller only after CRD is established --- main.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index c71fa5b6b..aa6c4dff7 100644 --- a/main.go +++ b/main.go @@ -238,10 +238,15 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage case watch.Error: exitOnError(apierrors.FromObject(event.Object), "error watching for RayCluster API") - case watch.Added: - setupLog.Info("RayCluster API installed, setting up controller") - exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") - return + case watch.Added, watch.Modified: + if crd := event.Object.(*apiextensionsv1.CustomResourceDefinition); crd.Name == "rayclusters.ray.io" && + slices.ContainsFunc(crd.Status.Conditions, func(condition apiextensionsv1.CustomResourceDefinitionCondition) bool { + return condition.Type == apiextensionsv1.Established && condition.Status == apiextensionsv1.ConditionTrue + }) { + setupLog.Info("RayCluster API installed, setting up controller") + exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") + return + } } } } From a7619510bca4d70ed4362846703735f027106f10 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 2 May 2024 08:11:34 -0400 Subject: [PATCH 197/369] use namespace from ray cluster Signed-off-by: Kevin --- pkg/controllers/raycluster_webhook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index eb530697e..987b7a611 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -253,7 +253,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container { "--tls-cert=/etc/tls/private/tls.crt", "--tls-key=/etc/tls/private/tls.key", "--cookie-secret=$(COOKIE_SECRET)", - "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"" + rayCluster.Namespace + "\",\"verb\":\"get\"}}", }, VolumeMounts: []corev1.VolumeMount{ { From 830b619ebeecd728abb29bf1fbf609ba6358f299 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 29 Apr 2024 23:13:17 +0100 Subject: [PATCH 198/369] allow for custom CertGeneratorImage --- config/e2e/config.yaml | 1 + main.go | 1 + pkg/config/config.go | 2 ++ pkg/controllers/raycluster_webhook.go | 24 ++++++++++++------------ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 6b8f55a0c..ba15737e2 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -7,3 +7,4 @@ data: kuberay: rayDashboardOAuthEnabled: false ingressDomain: "kind" + certGeneratorImage: "quay.io/project-codeflare/ray:latest-py39-cu118" diff --git a/main.go b/main.go index aa6c4dff7..813f25b44 100644 --- a/main.go +++ b/main.go @@ -129,6 +129,7 @@ func main() { RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", MTLSEnabled: ptr.To(true), + CertGeneratorImage: "quay.io/project-codeflare/ray:latest-py39-cu118", }, } diff --git a/pkg/config/config.go b/pkg/config/config.go index af3bc3491..8c48147cb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -37,6 +37,8 @@ type KubeRayConfiguration struct { IngressDomain string `json:"ingressDomain"` MTLSEnabled *bool `json:"mTLSEnabled,omitempty"` + + CertGeneratorImage string `json:"certGeneratorImage"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 987b7a611..2f8d46d1a 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -87,7 +87,7 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } // Append the create-cert Init Container - rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain), withContainerName(initContainerName)) + rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain, w.Config.CertGeneratorImage), withContainerName(initContainerName)) // Append the CA volumes for _, caVol := range caVolumes(rayCluster) { @@ -117,7 +117,7 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } // Append the create-cert Init Container - rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), withContainerName(initContainerName)) + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(w.Config.CertGeneratorImage), withContainerName(initContainerName)) } return nil @@ -161,8 +161,8 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r // Init Container related errors if ptr.Deref(w.Config.MTLSEnabled, true) { - allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w.Config.IngressDomain)...) - allErrors = append(allErrors, validateWorkerInitContainer(rayCluster)...) + allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w.Config.IngressDomain, w.Config.CertGeneratorImage)...) + allErrors = append(allErrors, validateWorkerInitContainer(rayCluster, w.Config.CertGeneratorImage)...) allErrors = append(allErrors, validateHeadEnvVars(rayCluster)...) allErrors = append(allErrors, validateWorkerEnvVars(rayCluster)...) allErrors = append(allErrors, validateCaVolumes(rayCluster)...) @@ -339,14 +339,14 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume { } } -func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) corev1.Container { +func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string, certGeneratorImage string) corev1.Container { rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + domain // Service name for basic interactive svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc" initContainerHead := corev1.Container{ Name: "create-cert", - Image: "quay.io/project-codeflare/ray:latest-py39-cu118", + Image: certGeneratorImage, Command: []string{ "sh", "-c", @@ -357,10 +357,10 @@ func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) corev1.Co return initContainerHead } -func rayWorkerInitContainer() corev1.Container { +func rayWorkerInitContainer(certGeneratorImage string) corev1.Container { initContainerWorker := corev1.Container{ Name: "create-cert", - Image: "quay.io/project-codeflare/ray:latest-py39-cu118", + Image: certGeneratorImage, Command: []string{ "sh", "-c", @@ -371,10 +371,10 @@ func rayWorkerInitContainer() corev1.Container { return initContainerWorker } -func validateHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) field.ErrorList { +func validateHeadInitContainer(rayCluster *rayv1.RayCluster, domain string, certGeneratorImage string) field.ErrorList { var allErrors field.ErrorList - if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, domain), byContainerName, + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, domain, certGeneratorImage), byContainerName, field.NewPath("spec", "headGroupSpec", "template", "spec", "initContainers"), "create-cert Init Container is immutable"); err != nil { allErrors = append(allErrors, err) @@ -383,10 +383,10 @@ func validateHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) fiel return allErrors } -func validateWorkerInitContainer(rayCluster *rayv1.RayCluster) field.ErrorList { +func validateWorkerInitContainer(rayCluster *rayv1.RayCluster, certGeneratorImage string) field.ErrorList { var allErrors field.ErrorList - if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), byContainerName, + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(certGeneratorImage), byContainerName, field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "initContainers"), "create-cert Init Container is immutable"); err != nil { allErrors = append(allErrors, err) From 2c2b0a28ed44cc42ab88b4c95a3ae90a268d1ac5 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 8 May 2024 17:09:48 +0100 Subject: [PATCH 199/369] addressing comments --- config/e2e/config.yaml | 2 +- main.go | 2 +- pkg/controllers/raycluster_webhook.go | 26 +++++++++++++------------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index ba15737e2..68a37c833 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -7,4 +7,4 @@ data: kuberay: rayDashboardOAuthEnabled: false ingressDomain: "kind" - certGeneratorImage: "quay.io/project-codeflare/ray:latest-py39-cu118" + mTLSEnabled: false diff --git a/main.go b/main.go index 813f25b44..72d2fdc2c 100644 --- a/main.go +++ b/main.go @@ -129,7 +129,7 @@ func main() { RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", MTLSEnabled: ptr.To(true), - CertGeneratorImage: "quay.io/project-codeflare/ray:latest-py39-cu118", + CertGeneratorImage: "registry.access.redhat.com/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", }, } diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 2f8d46d1a..3e0b29624 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -87,7 +87,7 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } // Append the create-cert Init Container - rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain, w.Config.CertGeneratorImage), withContainerName(initContainerName)) + rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config), withContainerName(initContainerName)) // Append the CA volumes for _, caVol := range caVolumes(rayCluster) { @@ -117,7 +117,7 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } // Append the create-cert Init Container - rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(w.Config.CertGeneratorImage), withContainerName(initContainerName)) + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(w.Config), withContainerName(initContainerName)) } return nil @@ -161,8 +161,8 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r // Init Container related errors if ptr.Deref(w.Config.MTLSEnabled, true) { - allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w.Config.IngressDomain, w.Config.CertGeneratorImage)...) - allErrors = append(allErrors, validateWorkerInitContainer(rayCluster, w.Config.CertGeneratorImage)...) + allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w.Config)...) + allErrors = append(allErrors, validateWorkerInitContainer(rayCluster, w.Config)...) allErrors = append(allErrors, validateHeadEnvVars(rayCluster)...) allErrors = append(allErrors, validateWorkerEnvVars(rayCluster)...) allErrors = append(allErrors, validateCaVolumes(rayCluster)...) @@ -339,14 +339,14 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume { } } -func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string, certGeneratorImage string) corev1.Container { - rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + domain +func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) corev1.Container { + rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + config.IngressDomain // Service name for basic interactive svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc" initContainerHead := corev1.Container{ Name: "create-cert", - Image: certGeneratorImage, + Image: config.CertGeneratorImage, Command: []string{ "sh", "-c", @@ -357,10 +357,10 @@ func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string, certGener return initContainerHead } -func rayWorkerInitContainer(certGeneratorImage string) corev1.Container { +func rayWorkerInitContainer(config *config.KubeRayConfiguration) corev1.Container { initContainerWorker := corev1.Container{ Name: "create-cert", - Image: certGeneratorImage, + Image: config.CertGeneratorImage, Command: []string{ "sh", "-c", @@ -371,10 +371,10 @@ func rayWorkerInitContainer(certGeneratorImage string) corev1.Container { return initContainerWorker } -func validateHeadInitContainer(rayCluster *rayv1.RayCluster, domain string, certGeneratorImage string) field.ErrorList { +func validateHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) field.ErrorList { var allErrors field.ErrorList - if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, domain, certGeneratorImage), byContainerName, + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, config), byContainerName, field.NewPath("spec", "headGroupSpec", "template", "spec", "initContainers"), "create-cert Init Container is immutable"); err != nil { allErrors = append(allErrors, err) @@ -383,10 +383,10 @@ func validateHeadInitContainer(rayCluster *rayv1.RayCluster, domain string, cert return allErrors } -func validateWorkerInitContainer(rayCluster *rayv1.RayCluster, certGeneratorImage string) field.ErrorList { +func validateWorkerInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) field.ErrorList { var allErrors field.ErrorList - if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(certGeneratorImage), byContainerName, + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(config), byContainerName, field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "initContainers"), "create-cert Init Container is immutable"); err != nil { allErrors = append(allErrors, err) From ebab80a3c2c9ae292ef58c223c06f9365afa630c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 13 May 2024 11:55:41 +0100 Subject: [PATCH 200/369] Comment out install/uninstall make targets --- Makefile | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index dbaa9a208..cbe476a56 100644 --- a/Makefile +++ b/Makefile @@ -183,14 +183,18 @@ ifndef ignore-not-found endif .PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl apply -f - - git restore config/* - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - git restore config/* +install: + @echo "No CRDs required to be installed" +# Uncomment install and uninstall make targets once AppWrapper CRDs are required. +# .PHONY: install +# install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. +# $(KUSTOMIZE) build config/crd | kubectl apply -f - +# git restore config/* + +# .PHONY: uninstall +# uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. +# $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - +# git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. From 29db9a7e21e42640a92a8e16751ceade465dc235 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Mon, 13 May 2024 16:52:52 +0200 Subject: [PATCH 201/369] Update dependency versions for release v1.4.1 (#554) Co-authored-by: codeflare-machine-account --- README.md | 4 ++-- config/manager/params.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7531cfdd1..8ea39c52e 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.4.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.0) | -| CodeFlare-SDK | [v0.16.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.0) | +| CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | +| CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | diff --git a/config/manager/params.env b/config/manager/params.env index 6ac22e106..e3c1fd4c7 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.1 namespace=opendatahub From ab129df453a5593d1097e46a7ec8624f65ddfc7b Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 14 May 2024 15:35:21 +0100 Subject: [PATCH 202/369] Added kueue setup make target --- Makefile | 5 +++ scripts/setup-kueue-resources.sh | 77 ++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100755 scripts/setup-kueue-resources.sh diff --git a/Makefile b/Makefile index cbe476a56..8f2a9cddc 100644 --- a/Makefile +++ b/Makefile @@ -400,3 +400,8 @@ image-mnist-job-test-build: ## Build container image with the MNIST job. .PHONY: image-mnist-job-test-push image-mnist-job-test-push: image-mnist-job-test-build ## Push container image with the MNIST job. podman push ${MNIST_JOB_TEST_IMG} + +# Make target for generating kueue related resources +.PHONY: kueue-setup +kueue-setup: + bash scripts/setup-kueue-resources.sh diff --git a/scripts/setup-kueue-resources.sh b/scripts/setup-kueue-resources.sh new file mode 100755 index 000000000..ad5736d28 --- /dev/null +++ b/scripts/setup-kueue-resources.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Function to prompt the user for input with a default value +prompt_for_input() { + local prompt_message=$1 + local default_value=$2 + local user_input + + read -p "$prompt_message [$default_value] ): " user_input + echo "${user_input:-$default_value}" +} + +echo "ClusterQueue Configuration" + +# Using function to get user inputs +name=$(prompt_for_input "Enter your Cluster Queue name, (default : " "cluster-queue") +cpu=$(prompt_for_input "Enter your Cluster Queue CPU (The amount of CPUs available: (default: " "16") +memory=$(prompt_for_input "Enter your Cluster Queue Memory (The amount of memory available) (default: " "60Gi") +gpu=$(prompt_for_input "Enter your Cluster Queue GPU count (The amount of GPUs available) (default: " "0") +pods=$(prompt_for_input "Enter your Cluster Queue Pods limit (The maximum amount of pods that can be created) (default:" "10") +flavour=$(prompt_for_input "Enter your Cluster Queue Resource Flavour (default: " "default-flavor") + +# Applying ClusterQueue configuration +kubectl apply --server-side -f - < Date: Fri, 19 Apr 2024 13:41:30 -0400 Subject: [PATCH 203/369] go mod tidy -go 1.21 --- go.mod | 2 +- go.sum | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 047755ad9..2d8f951b6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/project-codeflare/codeflare-operator -go 1.20 +go 1.21 require ( github.com/onsi/ginkgo/v2 v2.12.1 diff --git a/go.sum b/go.sum index 10d47e582..d5fdfbeb3 100644 --- a/go.sum +++ b/go.sum @@ -120,13 +120,19 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= +github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= +github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -139,6 +145,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -183,6 +190,7 @@ github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3ev github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= @@ -214,6 +222,7 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/ray-project/kuberay/ray-operator v1.1.0 h1:kCbPZG6SDv11A1F1VR0oG1oCTjrKUuZiMDsyJqmNr3g= github.com/ray-project/kuberay/ray-operator v1.1.0/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -231,6 +240,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -239,6 +249,7 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -247,6 +258,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= @@ -258,6 +270,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -290,6 +303,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -423,6 +437,7 @@ k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= +k8s.io/kube-aggregator v0.28.3/go.mod h1:5DyLevbRTcWnT1f9b+lB3BfbXC1w7gDa/OtB6kKInCw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D1/LqPzj0bLjxLYWbdUtV0= From 2cb14bbf66f6e469177f180c48d25c6b4c6d7807 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 19 Apr 2024 14:00:56 -0400 Subject: [PATCH 204/369] adapt Dockerfile changes for go 1.21 from #491 --- Dockerfile | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2d28eb026..b2bf71dcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,26 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.20.10 as builder + +# BEGIN -- workaround lack of go-toolset for golang 1.21 + +ARG GOLANG_IMAGE=golang:1.21 + +ARG GOARCH=amd64 + +FROM ${GOLANG_IMAGE} AS golang +FROM registry.access.redhat.com/ubi8/ubi:8.8 AS builder + +# Install system dependencies +RUN dnf upgrade -y && dnf install -y \ + gcc \ + make \ + openssl-devel \ + && dnf clean all && rm -rf /var/cache/yum + +# Install Go +ENV PATH=/usr/local/go/bin:$PATH +COPY --from=golang /usr/local/go /usr/local/go + +# END -- workaround lack of go-toolset for golang 1.21 WORKDIR /workspace # Copy the Go Modules manifests @@ -14,7 +35,7 @@ COPY pkg/ pkg/ # Build USER root -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 make go-build-for-image +RUN CGO_ENABLED=1 GOOS=linux GOARCH=${GOARCH} make go-build-for-image FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / From dac0c239da1d2bab107e1c2ba8d0478679b869a4 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 22 Apr 2024 11:01:46 -0400 Subject: [PATCH 205/369] Add AppWrapper CRDs and controller --- Makefile | 18 +- README.md | 7 +- config/crd/appwrapper/kustomization.yaml | 4 + config/crd/crd-appwrapper.yml | 258 ++++++++++++++++++ config/crd/kustomization.yaml | 16 ++ config/crd/kustomizeconfig.yaml | 19 ++ .../crd/patches/webhook_in_appwrappers.yaml | 16 ++ config/default/kustomization.yaml | 1 + config/e2e/config.yaml | 2 + config/rbac/role.yaml | 170 ++++++++++++ go.mod | 55 ++-- go.sum | 197 +++++++++---- main.go | 57 +++- pkg/config/config.go | 11 + pkg/controllers/appwrapper_controller.go | 46 ++++ test/e2e/kueue.sh | 31 +++ test/e2e/setup.sh | 26 ++ test/odh/{mcad_ray_test.go => ray_test.go} | 2 +- 18 files changed, 846 insertions(+), 90 deletions(-) create mode 100644 config/crd/appwrapper/kustomization.yaml create mode 100644 config/crd/crd-appwrapper.yml create mode 100644 config/crd/kustomization.yaml create mode 100644 config/crd/kustomizeconfig.yaml create mode 100644 config/crd/patches/webhook_in_appwrappers.yaml create mode 100644 pkg/controllers/appwrapper_controller.go create mode 100755 test/e2e/kueue.sh rename test/odh/{mcad_ray_test.go => ray_test.go} (98%) diff --git a/Makefile b/Makefile index 8f2a9cddc..878e1306b 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,15 @@ PREVIOUS_VERSION ?= v0.0.0-dev VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) +# APPWRAPPER_VERSION defines the default version of the AppWrapper controller + APPWRAPPER_VERSION ?= v0.8.1 + APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper + # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) + APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} + + # KUEUE_VERSION defines the default version of Kueue (used for testing) + KUEUE_VERSION ?= v0.6.2 + # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v1.1.0 @@ -127,8 +136,11 @@ help: ## Display this help. # this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests -manifests: controller-gen kustomize install-yq ## Generate RBAC objects. +manifests: controller-gen kustomize install-yq ## Generate RBAC objects and import upstream CRDs. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." + $(SED) -i -E "s|(- )\${APPWRAPPER_REPO}.*|\1\${APPWRAPPER_CRD}|" config/crd/appwrapper/kustomization.yaml + $(KUSTOMIZE) build config/crd/appwrapper | $(YQ) -s '"crd-" + .spec.names.singular' --no-doc + mv crd-*.yml config/crd .PHONY: fmt fmt: ## Run go fmt against code. @@ -381,6 +393,10 @@ test-e2e: manifests fmt vet ## Run e2e tests. kind-e2e: ## Set up e2e KinD cluster test/e2e/kind.sh +.PHONY: kueue-e2e +kueue-e2e: ## Deploy Kueue for e2e tests + KUEUE_VERSION=$(KUEUE_VERSION) test/e2e/kueue.sh + .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh diff --git a/README.md b/README.md index 8ea39c52e..f2c4a4200 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | -| CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | -| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | +| CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | +| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | +| AppWrapper | [v0.8.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.8.1) | ## Development @@ -32,6 +33,8 @@ The e2e tests can be executed locally by running the following commands: ```bash # Create a KinD cluster make kind-e2e + # Deploy Kueue + make kueue-e2e # Install the CRDs make install ``` diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml new file mode 100644 index 000000000..e0be56c97 --- /dev/null +++ b/config/crd/appwrapper/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.8.1 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml new file mode 100644 index 000000000..ad9cfc34b --- /dev/null +++ b/config/crd/crd-appwrapper.yml @@ -0,0 +1,258 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: appwrappers.workload.codeflare.dev +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: webhook-service + namespace: system + path: /convert + conversionReviewVersions: + - v1 + group: workload.codeflare.dev + names: + kind: AppWrapper + listKind: AppWrapperList + plural: appwrappers + singular: appwrapper + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Status + type: string + - jsonPath: .status.conditions[?(@.type=="QuotaReserved")].status + name: Quota Reserved + type: string + - jsonPath: .status.conditions[?(@.type=="ResourcesDeployed")].status + name: Resources Deployed + type: string + - jsonPath: .status.conditions[?(@.type=="Unhealthy")].status + name: Unhealthy + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: AppWrapper is the Schema for the appwrappers API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: AppWrapperSpec defines the desired state of the AppWrapper + properties: + components: + description: Components lists the components contained in the AppWrapper + items: + description: AppWrapperComponent describes a single wrapped Kubernetes resource + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map that may be used to store and retrieve + arbitrary metadata about the Component to customize its treatment by the AppWrapper controller. + type: object + podSetInfos: + description: PodSetInfos assigned to the Component's PodSets by Kueue + items: + description: AppWrapperPodSetInfo contains the data that Kueue wants to inject into an admitted PodSpecTemplate + properties: + annotations: + additionalProperties: + type: string + description: Annotations to be added to the PodSpecTemplate + type: object + labels: + additionalProperties: + type: string + description: Labels to be added to the PodSepcTemplate + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelectors to be added to the PodSpecTemplate + type: object + tolerations: + description: Tolerations to be added to the PodSpecTemplate + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + type: array + podSets: + description: PodSets contained in the Component + items: + description: AppWrapperPodSet describes an homogeneous set of pods + properties: + podPath: + description: PodPath is the path Component.Template to the PodTemplateSpec for this PodSet + type: string + replicaPath: + description: ReplicaPath is the path within Component.Template to the replica count for this PodSet + type: string + replicas: + description: Replicas is the number of pods in this PodSet + format: int32 + type: integer + required: + - podPath + type: object + type: array + template: + description: Template defines the Kubernetes resource for the Component + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + required: + - template + type: object + type: array + suspend: + description: Suspend suspends the AppWrapper when set to true + type: boolean + required: + - components + type: object + status: + description: AppWrapperStatus defines the observed state of the appwrapper + properties: + conditions: + description: |- + Conditions hold the latest available observations of the AppWrapper current state. + + + The type of the condition could be: + + + - QuotaReserved: The AppWrapper was admitted by Kueue and has quota allocated to it + - ResourcesDeployed: The contained resources are deployed (or being deployed) on the cluster + - PodsReady: All pods of the contained resources are in the Ready or Succeeded state + - Unhealthy: One or more of the contained resources is unhealthy + - DeletingResources: The contained resources are in the process of being deleted from the cluster + items: + description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + phase: + description: Phase of the AppWrapper object + type: string + resettingCount: + description: Retries counts the number of times the AppWrapper has entered the Resetting Phase + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 000000000..042f878f3 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,16 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- crd-appwrapper.yml + +patches: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +- path: patches/webhook_in_appwrappers.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [WEBHOOK] To enable webhook, uncomment the following section +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 000000000..ec5c150a9 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/crd/patches/webhook_in_appwrappers.yaml b/config/crd/patches/webhook_in_appwrappers.yaml new file mode 100644 index 000000000..bc147c122 --- /dev/null +++ b/config/crd/patches/webhook_in_appwrappers.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: appwrappers.workload.codeflare.dev +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 30b1273cd..e4f0687af 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -14,6 +14,7 @@ commonLabels: app.kubernetes.io/part-of: codeflare bases: + - ../crd - ../rbac - ../manager - ../webhook diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 68a37c833..e216371fe 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -8,3 +8,5 @@ data: rayDashboardOAuthEnabled: false ingressDomain: "kind" mTLSEnabled: false + appwrapper: + enabled: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index bb419783c..4b2111c33 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,28 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -39,6 +61,18 @@ rules: verbs: - get - list +- apiGroups: + - apps + resources: + - deployments + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update - watch - apiGroups: - authentication.k8s.io @@ -52,6 +86,30 @@ rules: - subjectaccessreviews verbs: - create +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cluster.ray.io + resources: + - rayclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - config.openshift.io resources: @@ -95,6 +153,60 @@ rules: - get - list - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloadpriorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/finalizers + verbs: + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get + - patch + - update - apiGroups: - networking.k8s.io resources: @@ -162,3 +274,61 @@ rules: - get - patch - update +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - scheduling.sigs.k8s.io + resources: + - podgroups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - scheduling.x-k8s.io + resources: + - podgroups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/finalizers + verbs: + - update +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/status + verbs: + - get + - patch + - update diff --git a/go.mod b/go.mod index 2d8f951b6..970598150 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,30 @@ module github.com/project-codeflare/codeflare-operator go 1.21 require ( - github.com/onsi/ginkgo/v2 v2.12.1 - github.com/onsi/gomega v1.27.10 + github.com/onsi/ginkgo/v2 v2.16.0 + github.com/onsi/gomega v1.31.1 github.com/open-policy-agent/cert-controller v0.10.1 github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a + github.com/project-codeflare/appwrapper v0.8.1 + github.com/project-codeflare/codeflare-common v0.0.0-20240419154738-88e961b89d29 github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - k8s.io/api v0.28.4 - k8s.io/apiextensions-apiserver v0.28.4 - k8s.io/apimachinery v0.28.4 + k8s.io/api v0.29.2 + k8s.io/apiextensions-apiserver v0.29.2 + k8s.io/apimachinery v0.29.2 k8s.io/client-go v11.0.0+incompatible - k8s.io/component-base v0.28.4 - k8s.io/klog/v2 v2.100.1 + k8s.io/component-base v0.29.2 + k8s.io/klog/v2 v2.110.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b - sigs.k8s.io/controller-runtime v0.16.3 - sigs.k8s.io/yaml v1.3.0 + sigs.k8s.io/controller-runtime v0.17.0 + sigs.k8s.io/kueue v0.6.2 + sigs.k8s.io/yaml v1.4.0 ) -replace k8s.io/client-go => k8s.io/client-go v0.28.3 +replace k8s.io/client-go => k8s.io/client-go v0.29.2 replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 @@ -35,18 +37,19 @@ replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -54,15 +57,16 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kubeflow/training-operator v1.7.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -71,11 +75,11 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect @@ -86,14 +90,17 @@ require ( golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect + k8s.io/apiextensions-apiserver v0.29.0 // indirect + k8s.io/apiserver v0.29.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + sigs.k8s.io/jobset v0.3.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index d5fdfbeb3..6e67c85d9 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,21 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 h1:X8MJ0fnN5FPdcGF5Ij2/OW+HgiJrRg3AfHAx1PJtIzM= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -17,6 +25,10 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -33,26 +45,32 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= @@ -61,8 +79,9 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= @@ -77,7 +96,6 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -90,6 +108,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= +github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -97,15 +117,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -115,10 +136,17 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= @@ -133,7 +161,6 @@ github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -150,6 +177,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubeflow/common v0.4.7 h1:zz6QS4k2u2FY838M/FjOtwjJq39MRZVZcvPahRYL97M= +github.com/kubeflow/common v0.4.7/go.mod h1:43MAof/uhpJA2C0urynqatE3oKFQc7m2HLmJty7waqY= +github.com/kubeflow/mpi-operator v0.4.0 h1:PS4jLoMuRyrk/DHuYkI0D46sQQYpQt375HjOV4KVMFs= +github.com/kubeflow/mpi-operator v0.4.0/go.mod h1:/A4mTy/RYh2UIgaGUiXUaW70eThjsogu80WbbcZpuMg= +github.com/kubeflow/training-operator v1.7.0 h1:Zh61GlOWrlRi4UFOtJeV+/5REo/OndhwQ25KYd0llzc= +github.com/kubeflow/training-operator v1.7.0/go.mod h1:BZCLX1h06wY3YSeSZZcGYAqI9/nVi7isVCRkfgZe9nE= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -178,15 +211,15 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= -github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= +github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= +github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= @@ -201,15 +234,14 @@ github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a h1:Yk9J5qXjp+yfSRCzS0EElrhpTgfYJ+S+W/z84cmlmX4= -github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/appwrapper v0.8.1 h1:o3nbd5L7Zn7sorXQ3rOR2EW9ESq4XoAQlp+kDLPQ0Hc= +github.com/project-codeflare/appwrapper v0.8.1/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= +github.com/project-codeflare/codeflare-common v0.0.0-20240419154738-88e961b89d29 h1:Hb98JXMOA15+kefHxH1V0EnvRqi7j7PL3vF9xzL7+4M= +github.com/project-codeflare/codeflare-common v0.0.0-20240419154738-88e961b89d29/go.mod h1:LoZMDO3zgkl0OoPz0bDVI4WFC3wQ8xE23J2ncsvjy+g= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -223,12 +255,18 @@ github.com/ray-project/kuberay/ray-operator v1.1.0 h1:kCbPZG6SDv11A1F1VR0oG1oCTj github.com/ray-project/kuberay/ray-operator v1.1.0/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -246,10 +284,35 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= +go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= +go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= +go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= +go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -257,6 +320,7 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -269,20 +333,19 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -291,6 +354,7 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -302,8 +366,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -327,7 +392,9 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -340,6 +407,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -357,8 +425,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -367,15 +436,23 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -410,51 +487,61 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= -k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= -k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= -k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= -k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= -k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= -k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/apiserver v0.29.1 h1:e2wwHUfEmMsa8+cuft8MT56+16EONIEK8A/gpBSco+g= +k8s.io/apiserver v0.29.1/go.mod h1:V0EpkTRrJymyVT3M49we8uh2RvXf7fWC5XLB0P3SwRw= +k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= +k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo= -k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU= +k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= +k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= k8s.io/kube-aggregator v0.28.3/go.mod h1:5DyLevbRTcWnT1f9b+lB3BfbXC1w7gDa/OtB6kKInCw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D1/LqPzj0bLjxLYWbdUtV0= -k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/metrics v0.29.1 h1:qutc3aIPMCniMuEApuLaeYX47rdCn8eycVDx7R6wMlQ= +k8s.io/metrics v0.29.1/go.mod h1:JrbV2U71+v7d/9qb90UVKL8r0uJ6Z2Hy4V7mDm05cKs= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= +sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= +sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/jobset v0.3.1 h1:GGUH6CNDgsVOip+oz6uiMYg5Tt50wHTmIgmeaTolQ2g= +sigs.k8s.io/jobset v0.3.1/go.mod h1:x3pw53wKIxq5fFz7/WacV1Exd9fTHa530szVXBdo9OE= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kueue v0.6.2 h1:wSX6DY/BNCIaza9R8TyhRxwzSrY02EkyxfXXeHbV758= +sigs.k8s.io/kueue v0.6.2/go.mod h1:TQ1lIc+2rdvObH7SszaxdCGTMYbp9XupKnCj0ZZDgEw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/main.go b/main.go index 72d2fdc2c..6b77ce738 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,9 @@ import ( cert "github.com/open-policy-agent/cert-controller/pkg/rotator" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + awv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + awconfig "github.com/project-codeflare/appwrapper/pkg/config" + awctrl "github.com/project-codeflare/appwrapper/pkg/controller" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" "golang.org/x/exp/slices" @@ -55,6 +58,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/yaml" routev1 "github.com/openshift/api/route/v1" @@ -62,16 +66,15 @@ import ( "github.com/project-codeflare/codeflare-operator/pkg/config" "github.com/project-codeflare/codeflare-operator/pkg/controllers" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - OperatorVersion = "UNKNOWN" - BuildDate = "UNKNOWN" + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + OperatorVersion = "UNKNOWN" + BuildDate = "UNKNOWN" + AppWrapperVersion = "UNKNOWN" ) func init() { @@ -82,6 +85,10 @@ func init() { utilruntime.Must(routev1.Install(scheme)) // ODH utilruntime.Must(dsciv1.AddToScheme(scheme)) + // AppWrapper + utilruntime.Must(awv1beta2.AddToScheme(scheme)) + // Kueue + utilruntime.Must(kueue.AddToScheme(scheme)) } // +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get @@ -104,6 +111,7 @@ func main() { setupLog.Info("Build info", "operatorVersion", OperatorVersion, + "appwrapperVersion", AppWrapperVersion, "date", BuildDate, ) @@ -131,6 +139,10 @@ func main() { MTLSEnabled: ptr.To(true), CertGeneratorImage: "registry.access.redhat.com/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", }, + AppWrapper: &config.AppWrapperConfiguration{ + Enabled: ptr.To(false), + Config: awconfig.NewAppWrapperConfig(), + }, } kubeConfig, err := ctrl.GetConfig() @@ -175,7 +187,10 @@ func main() { exitOnError(err, cfg.KubeRay.IngressDomain) } - setupLog.Info("setting up health endpoints") + setupLog.Info("setting up indexers") + exitOnError(setupIndexers(ctx, mgr, kubeClient, cfg), "unable to setup indexers") + + setupLog.Info("setting up health endpoints") exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") setupLog.Info("setting up RayCluster controller") @@ -251,6 +266,34 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage } } } + + if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) { + hasAW, errAW := hasAPIResourceForGVK(dc, awv1beta2.GroupVersion.WithKind("AppWrapper")) + hasWL, errWL := hasAPIResourceForGVK(dc, kueue.GroupVersion.WithKind("Workload")) + if hasAW && hasWL { + exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper webhook") + exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper controller") + } else if errAW != nil || errWL != nil { + exitOnError(err, "Could not determine if Workload and AppWrapper CRDs present on cluster.") + } else { + setupLog.Info("AppWrapper controller disabled", "Workload CRD present", hasWL, + "AppWrapper CRD present", hasAW, "Config flag value", *cfg.AppWrapper.Enabled) + } + } +} + +func setupIndexers(ctx context.Context, mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration) error { + if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) { + hasWL, errWL := hasAPIResourceForGVK(dc, kueue.GroupVersion.WithKind("Workload")) + if hasWL { + if err := awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config); err != nil { + return fmt.Errorf("workload indexer: %w", err) + } + } else if errWL != nil { + return fmt.Errorf("checking Workload CR: %w", errWL) + } + } + return nil } // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update diff --git a/pkg/config/config.go b/pkg/config/config.go index 8c48147cb..0df71d451 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,6 +17,8 @@ limitations under the License. package config import ( + awconfig "github.com/project-codeflare/appwrapper/pkg/config" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" ) @@ -29,6 +31,15 @@ type CodeFlareOperatorConfiguration struct { ControllerManager `json:",inline"` KubeRay *KubeRayConfiguration `json:"kuberay,omitempty"` + + AppWrapper *AppWrapperConfiguration `json:"appwrapper,omitempty"` +} + +type AppWrapperConfiguration struct { + // Enabled controls whether or not the AppWrapper Controller is enabled + Enabled *bool `json:"enabled,omitempty"` + // AppWrapper contains the AppWrapper controller configuration + Config *awconfig.AppWrapperConfig `json:"config,omitempty"` } type KubeRayConfiguration struct { diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go new file mode 100644 index 000000000..27884be90 --- /dev/null +++ b/pkg/controllers/appwrapper_controller.go @@ -0,0 +1,46 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +// RBAC generation comments copied from the appwrapper controllers + +// permission to fully control appwrappers +//+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/finalizers,verbs=update + +// permission to edit wrapped resources: pods, services, jobs, podgroups, pytorchjobs, rayclusters +//+kubebuilder:rbac:groups="",resources=pods;services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=scheduling.sigs.k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=scheduling.x-k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=kubeflow.org,resources=pytorchjobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=cluster.ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete + +// permissions needed by Kueue's generic reconciller +// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=list;get;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update;patch +// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/finalizers,verbs=update +// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch +// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch + +// permissions needed by Webhook to enable SubjectAccessReview +//+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=list diff --git a/test/e2e/kueue.sh b/test/e2e/kueue.sh new file mode 100755 index 000000000..e057a023e --- /dev/null +++ b/test/e2e/kueue.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2024 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Installs a Kueue release on a cluster + +set -euo pipefail +: "${KUEUE_VERSION}" + +echo "Deploying Kueue version $KUEUE_VERSION" +kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml + +# Sleep until the kueue manager is running +echo "Waiting for pods in the kueue-system namespace to become ready" +while [[ $(kubectl get pods -n kueue-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}' | tr ' ' '\n' | sort -u) != "True" ]] +do + echo -n "." && sleep 1; +done +echo "" diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index d5c99cbf8..620d93f63 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -56,3 +56,29 @@ roleRef: kind: ClusterRole name: mcad-controller-rayclusters EOF + +echo Creating Kueue ResourceFlavor and ClusterQueue +cat < Date: Mon, 22 Apr 2024 17:31:03 -0400 Subject: [PATCH 206/369] test cleanup: use RayClusters helper from codeflare-common --- go.mod | 2 +- go.sum | 4 ++-- test/odh/ray_test.go | 24 +++--------------------- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 970598150..6ec37ea5f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.8.1 - github.com/project-codeflare/codeflare-common v0.0.0-20240419154738-88e961b89d29 + github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 diff --git a/go.sum b/go.sum index 6e67c85d9..1dc258d2c 100644 --- a/go.sum +++ b/go.sum @@ -240,8 +240,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.8.1 h1:o3nbd5L7Zn7sorXQ3rOR2EW9ESq4XoAQlp+kDLPQ0Hc= github.com/project-codeflare/appwrapper v0.8.1/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= -github.com/project-codeflare/codeflare-common v0.0.0-20240419154738-88e961b89d29 h1:Hb98JXMOA15+kefHxH1V0EnvRqi7j7PL3vF9xzL7+4M= -github.com/project-codeflare/codeflare-common v0.0.0-20240419154738-88e961b89d29/go.mod h1:LoZMDO3zgkl0OoPz0bDVI4WFC3wQ8xE23J2ncsvjy+g= +github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f h1:9Uron4ej4Tt5ULX5CMzjmPqIZu3q/m07d4jhbNSwdPY= +github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/test/odh/ray_test.go b/test/odh/ray_test.go index 3aab33d58..14856919d 100644 --- a/test/odh/ray_test.go +++ b/test/odh/ray_test.go @@ -20,16 +20,13 @@ import ( "testing" . "github.com/onsi/gomega" - gomega "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" - corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestMCADRay(t *testing.T) { +func TestRay(t *testing.T) { test := With(t) // Create a namespace @@ -85,7 +82,7 @@ func TestMCADRay(t *testing.T) { createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) // Make sure the RayCluster is created and running - test.Eventually(rayClusters(test, namespace), TestTimeoutLong). + test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong). Should( And( HaveLen(1), @@ -94,7 +91,7 @@ func TestMCADRay(t *testing.T) { ) // Make sure the RayCluster finishes and is deleted - test.Eventually(rayClusters(test, namespace), TestTimeoutLong). + test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong). Should(HaveLen(0)) } @@ -130,18 +127,3 @@ func readMnistPy(test Test) []byte { return ParseTemplate(test, template, props) } - -// TODO: This belongs on codeflare-common/support/ray.go -func rayClusters(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []*rayv1.RayCluster { - return func(g gomega.Gomega) []*rayv1.RayCluster { - rcs, err := t.Client().Ray().RayV1().RayClusters(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - rcsp := []*rayv1.RayCluster{} - for _, v := range rcs.Items { - rcsp = append(rcsp, &v) - } - - return rcsp - } -} From d464e1cf340dcea499593f91c2a518647b56b957 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 22 Apr 2024 17:33:14 -0400 Subject: [PATCH 207/369] rename: mcad- to e2e- --- test/e2e/setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 620d93f63..29ad8926d 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -24,7 +24,7 @@ cat < Date: Mon, 22 Apr 2024 17:59:26 -0400 Subject: [PATCH 208/369] appwrapper adjustments to GitHub Actions --- .github/workflows/e2e_tests.yaml | 11 ++++++++++ .../workflows/project-codeflare-release.yml | 22 +++++++++++++++++-- .github/workflows/tag-and-build.yml | 9 ++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0a0ad3977..96e5d08f7 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -55,6 +55,10 @@ jobs: - name: Setup and start KinD cluster uses: ./common/github-actions/kind + - name: Deploy Kueue + run: | + make kueue-e2e + - name: Deploy CodeFlare stack id: deploy run: | @@ -86,6 +90,13 @@ jobs: echo "Printing CodeFlare operator logs" kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log + - name: Print Kueue operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing Kueue operator logs" + KUEUE_CONTROLLER_POD=$(kubectl get pods -n kueue-system | grep kueue-controller | awk '{print $1}') + kubectl logs -n kueue-system --tail -1 ${KUEUE_CONTROLLER_POD} | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kueue.log + - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 017c7b023..ae2dec938 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -13,6 +13,9 @@ on: codeflare-sdk-version: description: 'Version of CodeFlare-SDK to be released (for example: v0.0.0)' required: true + appwrapper-version: + description: 'Tested version of AppWrapper (for example: v0.0.0)' + required: true kuberay-version: description: 'Tested version of KubeRay (for example: v0.0.0)' required: true @@ -43,6 +46,7 @@ jobs: echo "Operator Version: ${{ github.event.inputs.operator-version }}" echo "Replaces: ${{ github.event.inputs.replaces }}" echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}" + echo "Tested AppWrapper Version: ${{ github.event.inputs.appwrapper-version }}" echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}" echo "Is Stable: ${{ github.event.inputs.is-stable }}" echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" @@ -50,7 +54,7 @@ jobs: echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" release-codeflare-sdk: - needs: check-kuberay-version + needs: [check-appwrapper-version, check-kuberay-version] runs-on: ubuntu-latest steps: @@ -90,7 +94,7 @@ jobs: steps: - name: Release CodeFlare operator run: | - gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash @@ -105,6 +109,20 @@ jobs: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + check-appwrapper-version: + runs-on: ubuntu-latest + + steps: + - name: Check if AppWrapper release exists + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/appwrapper/releases/tag/${{ github.event.inputs.appwrapper-version }}) + if [[ "$status_code" == "200" ]]; then + echo "AppWrapper release with version ${{ github.event.inputs.appwrapper-version }} exists and available to use." + else + echo "AppWrapper release with version ${{ github.event.inputs.appwrapper-version }} does not exist. Please select an existing version." + exit 1 + fi + check-kuberay-version: runs-on: ubuntu-latest diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index dc4a07a20..c4e33b312 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -16,6 +16,10 @@ on: description: 'Published version of CodeFlare-SDK' required: true default: 'v0.0.0-dev' + appwrapper-version: + description: 'Tested version of AppWrapper' + required: true + default: 'v0.0.0-dev' kuberay-version: description: 'Tested version of KubeRay' required: true @@ -83,8 +87,13 @@ jobs: run: | sed -i -E "s|(.*CodeFlare Operator.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.version }}\2${{ github.event.inputs.version }}\3|" README.md sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md + sed -i -E "s|(.*AppWrapper.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.appwrapper-version }}\2${{ github.event.inputs.appwrapper-version }}\3|" README.md sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md + - name: Adjust AppWrapper dependency in the code + run: | + sed -i -E "s/(.*APPWRAPPER_VERSION \?= ).*/\1${{ github.event.inputs.appwrapper-version }}/" Makefile + - name: Adjust Kuberay dependency in the code run: | sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile From 24a0a90d10209d1da00da4739728fb9c115b5028 Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 23 Apr 2024 15:45:19 -0400 Subject: [PATCH 209/369] upgrade appwrapper to v0.9.0 --- Makefile | 2 +- README.md | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 9 +++------ go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 878e1306b..f2e767926 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller - APPWRAPPER_VERSION ?= v0.8.1 + APPWRAPPER_VERSION ?= v0.9.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/README.md b/README.md index f2c4a4200..b21f3643f 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ CodeFlare Stack Compatibility Matrix |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | | CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | +| AppWrapper | [v0.9.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.9.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | -| AppWrapper | [v0.8.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.8.1) | ## Development diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index e0be56c97..8c79725d7 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.8.1 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.9.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index ad9cfc34b..c2e60db1b 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -139,18 +139,15 @@ spec: items: description: AppWrapperPodSet describes an homogeneous set of pods properties: - podPath: - description: PodPath is the path Component.Template to the PodTemplateSpec for this PodSet - type: string - replicaPath: - description: ReplicaPath is the path within Component.Template to the replica count for this PodSet + path: + description: Path is the path Component.Template to the PodTemplateSpec for this PodSet type: string replicas: description: Replicas is the number of pods in this PodSet format: int32 type: integer required: - - podPath + - path type: object type: array template: diff --git a/go.mod b/go.mod index 6ec37ea5f..cc4e2951b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.8.1 + github.com/project-codeflare/appwrapper v0.9.0 github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index 1dc258d2c..66ddc0dc6 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.8.1 h1:o3nbd5L7Zn7sorXQ3rOR2EW9ESq4XoAQlp+kDLPQ0Hc= -github.com/project-codeflare/appwrapper v0.8.1/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= +github.com/project-codeflare/appwrapper v0.9.0 h1:iQq91Fp3FWYsLgr4XvSTp2asfrItG/BAZ5Zfh50CzyQ= +github.com/project-codeflare/appwrapper v0.9.0/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f h1:9Uron4ej4Tt5ULX5CMzjmPqIZu3q/m07d4jhbNSwdPY= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From c196f35e623f7917fbad6514585f8884ac424452 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 26 Apr 2024 16:43:13 -0400 Subject: [PATCH 210/369] adjust appwrapper controller startup --- .github/workflows/olm_tests.yaml | 4 ++++ config/rbac/role.yaml | 1 + go.mod | 3 +-- go.sum | 8 +++---- main.go | 39 +++++++++++++++----------------- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index b13f2f894..b14bc1a0c 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -54,6 +54,10 @@ jobs: - name: Setup and start KinD cluster uses: ./common/github-actions/kind + - name: Deploy Kueue + run: | + make kueue-e2e + - name: Install OLM run: | kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4b2111c33..454881679 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -61,6 +61,7 @@ rules: verbs: - get - list + - watch - apiGroups: - apps resources: diff --git a/go.mod b/go.mod index cc4e2951b..a7ed5e03b 100644 --- a/go.mod +++ b/go.mod @@ -97,8 +97,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.0 // indirect - k8s.io/apiserver v0.29.1 // indirect + k8s.io/apiserver v0.29.2 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/jobset v0.3.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 66ddc0dc6..d7a45fb7a 100644 --- a/go.sum +++ b/go.sum @@ -494,13 +494,13 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= -k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= -k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= +k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/apiserver v0.29.1 h1:e2wwHUfEmMsa8+cuft8MT56+16EONIEK8A/gpBSco+g= -k8s.io/apiserver v0.29.1/go.mod h1:V0EpkTRrJymyVT3M49we8uh2RvXf7fWC5XLB0P3SwRw= +k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ= +k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ= k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= diff --git a/main.go b/main.go index 6b77ce738..d96974718 100644 --- a/main.go +++ b/main.go @@ -188,14 +188,17 @@ func main() { } setupLog.Info("setting up indexers") - exitOnError(setupIndexers(ctx, mgr, kubeClient, cfg), "unable to setup indexers") + exitOnError(setupIndexers(ctx, mgr, cfg), "unable to setup indexers") - setupLog.Info("setting up health endpoints") + setupLog.Info("setting up health endpoints") exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") setupLog.Info("setting up RayCluster controller") go waitForRayClusterAPIandSetupController(ctx, mgr, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) + setupLog.Info("setting up AppWrapper controller") + go setupAppWrapperController(mgr, cfg, certsReady) + setupLog.Info("starting manager") exitOnError(mgr.Start(ctx), "error running manager") } @@ -266,31 +269,25 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage } } } +} + +func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { + setupLog.Info("Waiting for certificate generation to complete") + <-certsReady + setupLog.Info("Certs ready") if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) { - hasAW, errAW := hasAPIResourceForGVK(dc, awv1beta2.GroupVersion.WithKind("AppWrapper")) - hasWL, errWL := hasAPIResourceForGVK(dc, kueue.GroupVersion.WithKind("Workload")) - if hasAW && hasWL { - exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper webhook") - exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper controller") - } else if errAW != nil || errWL != nil { - exitOnError(err, "Could not determine if Workload and AppWrapper CRDs present on cluster.") - } else { - setupLog.Info("AppWrapper controller disabled", "Workload CRD present", hasWL, - "AppWrapper CRD present", hasAW, "Config flag value", *cfg.AppWrapper.Enabled) - } + exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper webhook") + exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper controller") + } else { + setupLog.Info("AppWrapper controller disabled", "Config flag value", *cfg.AppWrapper.Enabled) } } -func setupIndexers(ctx context.Context, mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration) error { +func setupIndexers(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration) error { if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) { - hasWL, errWL := hasAPIResourceForGVK(dc, kueue.GroupVersion.WithKind("Workload")) - if hasWL { - if err := awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config); err != nil { - return fmt.Errorf("workload indexer: %w", err) - } - } else if errWL != nil { - return fmt.Errorf("checking Workload CR: %w", errWL) + if err := awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config); err != nil { + return fmt.Errorf("workload indexer: %w", err) } } return nil From a42b2b3dabfd189583247d9967a64fb6541baaf8 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 29 Apr 2024 11:34:00 -0400 Subject: [PATCH 211/369] graceful handling of Kueue not being installed before the codeflare operator --- .github/workflows/olm_tests.yaml | 4 - main.go | 171 ++++++++++++++++++++----------- 2 files changed, 113 insertions(+), 62 deletions(-) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index b14bc1a0c..b13f2f894 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -54,10 +54,6 @@ jobs: - name: Setup and start KinD cluster uses: ./common/github-actions/kind - - name: Deploy Kueue - run: | - make kueue-e2e - - name: Install OLM run: | kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml diff --git a/main.go b/main.go index d96974718..35a51a0e7 100644 --- a/main.go +++ b/main.go @@ -77,6 +77,11 @@ var ( AppWrapperVersion = "UNKNOWN" ) +const ( + workloadAPI = "workloads.kueue.x-k8s.io" + rayclusterAPI = "rayclusters.ray.io" +) + func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) // Ray @@ -115,7 +120,7 @@ func main() { "date", BuildDate, ) - ctx := ctrl.SetupSignalHandler() + ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) cfg := &config.CodeFlareOperatorConfiguration{ ClientConnection: &config.ClientConnection{ @@ -188,7 +193,7 @@ func main() { } setupLog.Info("setting up indexers") - exitOnError(setupIndexers(ctx, mgr, cfg), "unable to setup indexers") + exitOnError(setupWorkloadIndexer(ctx, cancel, mgr, cfg), "unable to setup indexers") setupLog.Info("setting up health endpoints") exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") @@ -197,7 +202,7 @@ func main() { go waitForRayClusterAPIandSetupController(ctx, mgr, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) setupLog.Info("setting up AppWrapper controller") - go setupAppWrapperController(mgr, cfg, certsReady) + go waitForWorkloadAPIAndSetupAppWrapperController(ctx, mgr, cfg, certsReady) setupLog.Info("starting manager") exitOnError(mgr.Start(ctx), "error running manager") @@ -222,75 +227,60 @@ func setupRayClusterController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorCo return rayClusterController.SetupWithManager(mgr) } -// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch - func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) { - crdClient, err := apiextensionsclientset.NewForConfig(mgr.GetConfig()) - exitOnError(err, "unable to create CRD client") - - crdList, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) - exitOnError(err, "unable to list CRDs") - - if slices.ContainsFunc(crdList.Items, func(crd apiextensionsv1.CustomResourceDefinition) bool { - return crd.Name == "rayclusters.ray.io" - }) { + if isAPIAvailable(ctx, mgr, rayclusterAPI) { exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") - } - - retryWatcher, err := retrywatch.NewRetryWatcher(crdList.ResourceVersion, &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return crdClient.ApiextensionsV1().CustomResourceDefinitions().Watch(ctx, metav1.ListOptions{}) - }, - }) - exitOnError(err, "unable to create retry watcher") - - defer retryWatcher.Stop() - for { - select { - case <-ctx.Done(): - return - case event := <-retryWatcher.ResultChan(): - switch event.Type { - case watch.Error: - exitOnError(apierrors.FromObject(event.Object), "error watching for RayCluster API") - - case watch.Added, watch.Modified: - if crd := event.Object.(*apiextensionsv1.CustomResourceDefinition); crd.Name == "rayclusters.ray.io" && - slices.ContainsFunc(crd.Status.Conditions, func(condition apiextensionsv1.CustomResourceDefinitionCondition) bool { - return condition.Type == apiextensionsv1.Established && condition.Status == apiextensionsv1.ConditionTrue - }) { - setupLog.Info("RayCluster API installed, setting up controller") - exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") - return - } - } - } + } else { + waitForAPI(ctx, mgr, rayclusterAPI, func() { + exitOnError(setupRayClusterController(mgr, cfg, isOpenShift, certsReady), "unable to setup RayCluster controller") + }) } } -func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { +func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Certs ready") - if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) { - exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper webhook") - exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "error setting up AppWrapper controller") + setupLog.Info("Setting up AppWrapper webhook and controller") + if err := awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config); err != nil { + return err + } + return awctrl.SetupControllers(mgr, cfg.AppWrapper.Config) +} + +func waitForWorkloadAPIAndSetupAppWrapperController(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { + if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { + setupLog.Info("AppWrapper controller disabled by config") + } + + if isAPIAvailable(ctx, mgr, workloadAPI) { + exitOnError(setupAppWrapperController(mgr, cfg, certsReady), "unable to setup AppWrapper controller") } else { - setupLog.Info("AppWrapper controller disabled", "Config flag value", *cfg.AppWrapper.Enabled) + waitForAPI(ctx, mgr, workloadAPI, func() { + exitOnError(setupAppWrapperController(mgr, cfg, certsReady), "unable to setup AppWrapper controller") + }) } } -func setupIndexers(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration) error { - if cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.Enabled, false) { - if err := awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config); err != nil { - return fmt.Errorf("workload indexer: %w", err) - } +func setupWorkloadIndexer(ctx context.Context, cancel context.CancelFunc, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration) error { + if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { + setupLog.Info("Workload indexer disabled by config") + return nil + } + + if isAPIAvailable(ctx, mgr, workloadAPI) { + return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config) + } else { + // If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly + // restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator. + // It is not possible to add an indexer once the mgr has started so, a restart if the only avenue. + go waitForAPI(ctx, mgr, workloadAPI, func() { + setupLog.Info("Workload API now available; triggering controller restart") + cancel() + }) + return nil } - return nil } // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update @@ -437,3 +427,68 @@ func getClusterDomain(ctx context.Context, configClient *clientset.Clientset) (s return domain, nil } + +// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch + +func isAPIAvailable(ctx context.Context, mgr ctrl.Manager, apiName string) bool { + crdClient, err := apiextensionsclientset.NewForConfig(mgr.GetConfig()) + exitOnError(err, "unable to create CRD client") + + crdList, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) + exitOnError(err, "unable to list CRDs") + + return slices.ContainsFunc(crdList.Items, func(crd apiextensionsv1.CustomResourceDefinition) bool { + return crd.Name == apiName + }) +} + +func waitForAPI(ctx context.Context, mgr ctrl.Manager, apiName string, action func()) { + crdClient, err := apiextensionsclientset.NewForConfig(mgr.GetConfig()) + exitOnError(err, "unable to create CRD client") + + crdList, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) + exitOnError(err, "unable to list CRDs") + + // If API is already available, just invoke action + if slices.ContainsFunc(crdList.Items, func(crd apiextensionsv1.CustomResourceDefinition) bool { + return crd.Name == apiName + }) { + action() + return + } + + // Wait for the API to become available then invoke action + setupLog.Info(fmt.Sprintf("API %v not available, setting up retry watcher", apiName)) + retryWatcher, err := retrywatch.NewRetryWatcher(crdList.ResourceVersion, &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return crdClient.ApiextensionsV1().CustomResourceDefinitions().Watch(ctx, metav1.ListOptions{}) + }, + }) + exitOnError(err, "unable to create retry watcher") + + defer retryWatcher.Stop() + for { + select { + case <-ctx.Done(): + return + case event := <-retryWatcher.ResultChan(): + switch event.Type { + case watch.Error: + exitOnError(apierrors.FromObject(event.Object), fmt.Sprintf("error watching for API %v", apiName)) + + case watch.Added, watch.Modified: + if crd := event.Object.(*apiextensionsv1.CustomResourceDefinition); crd.Name == apiName && + slices.ContainsFunc(crd.Status.Conditions, func(condition apiextensionsv1.CustomResourceDefinitionCondition) bool { + return condition.Type == apiextensionsv1.Established && condition.Status == apiextensionsv1.ConditionTrue + }) { + setupLog.Info(fmt.Sprintf("API %v installed, invoking deferred action", apiName)) + action() + return + } + } + } + } +} From 095162e73a6a96a8b35d2e38cba703aca29db256 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 29 Apr 2024 13:34:18 -0400 Subject: [PATCH 212/369] update to appwrapper 0.10.0 --- Makefile | 2 +- README.md | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 3 +++ go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f2e767926..c7a41593e 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller - APPWRAPPER_VERSION ?= v0.9.0 + APPWRAPPER_VERSION ?= v0.10.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/README.md b/README.md index b21f3643f..344dacbed 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ CodeFlare Stack Compatibility Matrix |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | | CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | -| AppWrapper | [v0.9.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.9.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | +| AppWrapper | [v0.9.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.9.0) | ## Development diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 8c79725d7..61bbafad3 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.9.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.10.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index c2e60db1b..dc134a6dd 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -240,6 +240,9 @@ spec: - type type: object type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map phase: description: Phase of the AppWrapper object type: string diff --git a/go.mod b/go.mod index a7ed5e03b..8b056b8e8 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.9.0 + github.com/project-codeflare/appwrapper v0.10.0 github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index d7a45fb7a..3d1004bb5 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.9.0 h1:iQq91Fp3FWYsLgr4XvSTp2asfrItG/BAZ5Zfh50CzyQ= -github.com/project-codeflare/appwrapper v0.9.0/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= +github.com/project-codeflare/appwrapper v0.10.0 h1:vhnoNK3IY4dtkwesXrcyQquOc5tOAqYgmxPfnESIU+A= +github.com/project-codeflare/appwrapper v0.10.0/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f h1:9Uron4ej4Tt5ULX5CMzjmPqIZu3q/m07d4jhbNSwdPY= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From 244af29f7501aa956253a889b7e49c5f9a298d99 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 29 Apr 2024 14:51:50 -0400 Subject: [PATCH 213/369] configure appwrapper webhooks --- config/webhook/manifests.yaml | 39 ++++++++++++++++++++++++ pkg/controllers/appwrapper_controller.go | 4 +++ 2 files changed, 43 insertions(+) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 5e98107c8..36dbe265c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -5,6 +5,25 @@ metadata: creationTimestamp: null name: mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-workload-codeflare-dev-v1beta2-appwrapper + failurePolicy: Fail + name: mappwrapper.kb.io + rules: + - apiGroups: + - workload.codeflare.dev + apiVersions: + - v1beta2 + operations: + - CREATE + resources: + - appwrappers + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -31,6 +50,26 @@ metadata: creationTimestamp: null name: validating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-workload-codeflare-dev-v1beta2-appwrapper + failurePolicy: Fail + name: vappwrapper.kb.io + rules: + - apiGroups: + - workload.codeflare.dev + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - appwrappers + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index 27884be90..581b29090 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -41,6 +41,10 @@ package controllers // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch +// webhook configuration +//+kubebuilder:webhook:path=/mutate-workload-codeflare-dev-v1beta2-appwrapper,mutating=true,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create,versions=v1beta2,name=mappwrapper.kb.io,admissionReviewVersions=v1 +//+kubebuilder:webhook:path=/validate-workload-codeflare-dev-v1beta2-appwrapper,mutating=false,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create;update,versions=v1beta2,name=vappwrapper.kb.io,admissionReviewVersions=v1 + // permissions needed by Webhook to enable SubjectAccessReview //+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create //+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=list From 60692fbe6cac75c307357c3fffab92ae590b4224 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 2 May 2024 17:11:00 -0400 Subject: [PATCH 214/369] restructure AppWrapper startup to ensure all-or-nothing setup --- main.go | 49 +++++++++++++++++-------------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/main.go b/main.go index 35a51a0e7..b7c3f7f9e 100644 --- a/main.go +++ b/main.go @@ -192,17 +192,14 @@ func main() { exitOnError(err, cfg.KubeRay.IngressDomain) } - setupLog.Info("setting up indexers") - exitOnError(setupWorkloadIndexer(ctx, cancel, mgr, cfg), "unable to setup indexers") - setupLog.Info("setting up health endpoints") exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") setupLog.Info("setting up RayCluster controller") go waitForRayClusterAPIandSetupController(ctx, mgr, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) - setupLog.Info("setting up AppWrapper controller") - go waitForWorkloadAPIAndSetupAppWrapperController(ctx, mgr, cfg, certsReady) + setupLog.Info("setting up AppWrapper components") + exitOnError(setupAppWrapperComponents(ctx, cancel, mgr, cfg, certsReady), "unable to setup AppWrapper") setupLog.Info("starting manager") exitOnError(mgr.Start(ctx), "error running manager") @@ -237,44 +234,22 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage } } -func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { - setupLog.Info("Waiting for certificate generation to complete") - <-certsReady - setupLog.Info("Certs ready") - - setupLog.Info("Setting up AppWrapper webhook and controller") - if err := awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config); err != nil { - return err - } - return awctrl.SetupControllers(mgr, cfg.AppWrapper.Config) -} - -func waitForWorkloadAPIAndSetupAppWrapperController(ctx context.Context, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { +func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, mgr ctrl.Manager, + cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { setupLog.Info("AppWrapper controller disabled by config") - } - - if isAPIAvailable(ctx, mgr, workloadAPI) { - exitOnError(setupAppWrapperController(mgr, cfg, certsReady), "unable to setup AppWrapper controller") - } else { - waitForAPI(ctx, mgr, workloadAPI, func() { - exitOnError(setupAppWrapperController(mgr, cfg, certsReady), "unable to setup AppWrapper controller") - }) - } -} - -func setupWorkloadIndexer(ctx context.Context, cancel context.CancelFunc, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration) error { - if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { - setupLog.Info("Workload indexer disabled by config") return nil } if isAPIAvailable(ctx, mgr, workloadAPI) { + setupLog.Info("Workload API available; enabling AppWrappers") + go setupAppWrapperController(mgr, cfg, certsReady) return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config) } else { // If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly // restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator. // It is not possible to add an indexer once the mgr has started so, a restart if the only avenue. + setupLog.Info("Workload API not available; setting up waiter for Workload API availability") go waitForAPI(ctx, mgr, workloadAPI, func() { setupLog.Info("Workload API now available; triggering controller restart") cancel() @@ -283,6 +258,16 @@ func setupWorkloadIndexer(ctx context.Context, cancel context.CancelFunc, mgr ct } } +func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { + setupLog.Info("Waiting for certificate generation to complete") + <-certsReady + setupLog.Info("Certs ready") + + setupLog.Info("Setting up AppWrapper webhook and controller") + exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") + exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper controller") +} + // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;update From dbf72d77e1e77cb3dda9e2988c84e051cb5d5761 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 6 May 2024 11:05:03 -0400 Subject: [PATCH 215/369] AppWrapper webhook can be registered without Kueue APIs --- main.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index b7c3f7f9e..64b8227d0 100644 --- a/main.go +++ b/main.go @@ -237,10 +237,17 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { - setupLog.Info("AppWrapper controller disabled by config") + setupLog.Info("AppWrappers are disabled by operator configuration") return nil } + // AppWrapper webhook doesn't depend on WorkloadAPI availablity but does need certsReady + go func() { + <-certsReady + setupLog.Info("Setting up AppWrapper webhook") + exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") + }() + if isAPIAvailable(ctx, mgr, workloadAPI) { setupLog.Info("Workload API available; enabling AppWrappers") go setupAppWrapperController(mgr, cfg, certsReady) @@ -261,10 +268,7 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { setupLog.Info("Waiting for certificate generation to complete") <-certsReady - setupLog.Info("Certs ready") - - setupLog.Info("Setting up AppWrapper webhook and controller") - exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") + setupLog.Info("Setting up AppWrapper controller") exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper controller") } From bba949c1e8924c906b2ade3f5db577efc06d855a Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 6 May 2024 13:45:05 -0400 Subject: [PATCH 216/369] improved appwrapper webhook configuration Define mock appwrapper webhooks to enable reasonable error messages for users when AppWrappers are disabled by configuration. --- main.go | 15 +++-- pkg/config/config.go | 2 + pkg/controllers/appwrapper_controller.go | 8 --- pkg/controllers/appwrapper_webhook.go | 74 ++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 pkg/controllers/appwrapper_webhook.go diff --git a/main.go b/main.go index 64b8227d0..b25d549f4 100644 --- a/main.go +++ b/main.go @@ -145,8 +145,9 @@ func main() { CertGeneratorImage: "registry.access.redhat.com/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", }, AppWrapper: &config.AppWrapperConfiguration{ - Enabled: ptr.To(false), - Config: awconfig.NewAppWrapperConfig(), + Enabled: ptr.To(false), + ExternalController: ptr.To(false), + Config: awconfig.NewAppWrapperConfig(), }, } @@ -237,14 +238,20 @@ func waitForRayClusterAPIandSetupController(ctx context.Context, mgr ctrl.Manage func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { - setupLog.Info("AppWrappers are disabled by operator configuration") + setupLog.Info("Embedded AppWrapper controller is disabled by config") + externalController := cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.ExternalController, false) + go func() { + <-certsReady + setupLog.Info("Setting up mock AppWrapper webhooks", "externalController", externalController) + exitOnError(controllers.SetupMockAppWrapperWebhooks(mgr, externalController), "unable to setup AppWrapper webhooks") + }() return nil } // AppWrapper webhook doesn't depend on WorkloadAPI availablity but does need certsReady go func() { <-certsReady - setupLog.Info("Setting up AppWrapper webhook") + setupLog.Info("Setting up AppWrapper webhooks") exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") }() diff --git a/pkg/config/config.go b/pkg/config/config.go index 0df71d451..4e3fde3b0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,6 +38,8 @@ type CodeFlareOperatorConfiguration struct { type AppWrapperConfiguration struct { // Enabled controls whether or not the AppWrapper Controller is enabled Enabled *bool `json:"enabled,omitempty"` + // ExternalController indicates that an AppWrapper Controller external to the Codeflare Operator is deployed + ExternalController *bool `json:"externalController,omitempty"` // AppWrapper contains the AppWrapper controller configuration Config *awconfig.AppWrapperConfig `json:"config,omitempty"` } diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index 581b29090..11684f5bf 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -40,11 +40,3 @@ package controllers // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/finalizers,verbs=update // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch - -// webhook configuration -//+kubebuilder:webhook:path=/mutate-workload-codeflare-dev-v1beta2-appwrapper,mutating=true,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create,versions=v1beta2,name=mappwrapper.kb.io,admissionReviewVersions=v1 -//+kubebuilder:webhook:path=/validate-workload-codeflare-dev-v1beta2-appwrapper,mutating=false,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create;update,versions=v1beta2,name=vappwrapper.kb.io,admissionReviewVersions=v1 - -// permissions needed by Webhook to enable SubjectAccessReview -//+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create -//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=list diff --git a/pkg/controllers/appwrapper_webhook.go b/pkg/controllers/appwrapper_webhook.go new file mode 100644 index 000000000..c019c4da3 --- /dev/null +++ b/pkg/controllers/appwrapper_webhook.go @@ -0,0 +1,74 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + awv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// webhook configuration +//+kubebuilder:webhook:path=/mutate-workload-codeflare-dev-v1beta2-appwrapper,mutating=true,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create,versions=v1beta2,name=mappwrapper.kb.io,admissionReviewVersions=v1 +//+kubebuilder:webhook:path=/validate-workload-codeflare-dev-v1beta2-appwrapper,mutating=false,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create;update,versions=v1beta2,name=vappwrapper.kb.io,admissionReviewVersions=v1 + +// permissions needed by the "real" Webhook in the appwrapper project to enable SubjectAccessReview +//+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=list + +type appwrapperWebhook struct { + externalController bool +} + +var _ webhook.CustomDefaulter = &appwrapperWebhook{} + +func (w *appwrapperWebhook) Default(ctx context.Context, obj runtime.Object) error { + return nil +} + +var _ webhook.CustomValidator = &appwrapperWebhook{} + +func (w *appwrapperWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + if w.externalController { + return nil, nil + } else { + return nil, fmt.Errorf("AppWrappers disabled by CodeFlare operator configuration") + } +} + +func (w *appwrapperWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func (w *appwrapperWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +func SetupMockAppWrapperWebhooks(mgr ctrl.Manager, externalController bool) error { + wh := &appwrapperWebhook{externalController: externalController} + return ctrl.NewWebhookManagedBy(mgr). + For(&awv1beta2.AppWrapper{}). + WithDefaulter(wh). + WithValidator(wh). + Complete() +} From 162b17e55c20ff2e5d4238186869c9bacc1ffed3 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 6 May 2024 21:11:55 -0400 Subject: [PATCH 217/369] update to appwrapper v0.11.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8b056b8e8..37990383f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.10.0 + github.com/project-codeflare/appwrapper v0.11.0 github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index 3d1004bb5..8292b8b0d 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.10.0 h1:vhnoNK3IY4dtkwesXrcyQquOc5tOAqYgmxPfnESIU+A= -github.com/project-codeflare/appwrapper v0.10.0/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= +github.com/project-codeflare/appwrapper v0.11.0 h1:5XeZuUUkC5N03M7q2IyhSL4TqPjyDfhWJ2YKERkBFuw= +github.com/project-codeflare/appwrapper v0.11.0/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f h1:9Uron4ej4Tt5ULX5CMzjmPqIZu3q/m07d4jhbNSwdPY= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From 040ca39e051b1f6cfd609299b92b4041bb881e16 Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 7 May 2024 11:32:47 -0400 Subject: [PATCH 218/369] adjust e2e test so that RayCluster really is managed by Kueue --- .github/workflows/e2e_tests.yaml | 4 - Makefile | 6 +- README.md | 23 +++--- test/e2e/kueue.sh | 31 -------- test/e2e/mnist_rayjob_raycluster_test.go | 93 ++++++++++++++---------- test/e2e/setup.sh | 11 +++ test/e2e/support.go | 12 +++ 7 files changed, 89 insertions(+), 91 deletions(-) delete mode 100755 test/e2e/kueue.sh diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 96e5d08f7..d84ce09c2 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -55,10 +55,6 @@ jobs: - name: Setup and start KinD cluster uses: ./common/github-actions/kind - - name: Deploy Kueue - run: | - make kueue-e2e - - name: Deploy CodeFlare stack id: deploy run: | diff --git a/Makefile b/Makefile index c7a41593e..21dd914d9 100644 --- a/Makefile +++ b/Makefile @@ -393,13 +393,9 @@ test-e2e: manifests fmt vet ## Run e2e tests. kind-e2e: ## Set up e2e KinD cluster test/e2e/kind.sh -.PHONY: kueue-e2e -kueue-e2e: ## Deploy Kueue for e2e tests - KUEUE_VERSION=$(KUEUE_VERSION) test/e2e/kueue.sh - .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. - KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh + KUBERAY_VERSION=$(KUBERAY_VERSION) KUEUE_VERSION=$(KUEUE_VERSION) test/e2e/setup.sh .PHONY: imports imports: openshift-goimports ## Organize imports in go files using openshift-goimports. Example: make imports diff --git a/README.md b/README.md index 344dacbed..cf08ef684 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,6 @@ The e2e tests can be executed locally by running the following commands: ```bash # Create a KinD cluster make kind-e2e - # Deploy Kueue - make kueue-e2e # Install the CRDs make install ``` @@ -43,20 +41,15 @@ The e2e tests can be executed locally by running the following commands: Some e2e tests cover the access to services via Ingresses, as end-users would do, which requires access to the Ingress controller load balancer by its IP. For it to work on macOS, this requires installing [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). -2. Start the operator locally: - - ```bash - NAMESPACE=default make run - ``` - - Alternatively, You can run the operator from your IDE / debugger. - -3. Set up the test CodeFlare stack: +2. Setup the rest of the CodeFlare stack. ```bash make setup-e2e ``` + [!NOTE] + Kueue will only activate its Ray integration if KubeRay is installed before Kueue (as done by this make target). + [!NOTE] In OpenShift the KubeRay operator pod gets random user assigned. This user is then used to run Ray cluster. However the random user assigned by OpenShift doesn't have rights to store dataset downloaded as part of test execution, causing tests to fail. @@ -76,6 +69,14 @@ The e2e tests can be executed locally by running the following commands: - 'system:serviceaccount:$(namespace):kuberay-operator' ``` +3. Start the operator locally: + + ```bash + NAMESPACE=default make run + ``` + + Alternatively, You can run the operator from your IDE / debugger. + 4. In a separate terminal, set your output directory for test files, and run the e2e suite: ```bash export CODEFLARE_TEST_OUTPUT_DIR= diff --git a/test/e2e/kueue.sh b/test/e2e/kueue.sh deleted file mode 100755 index e057a023e..000000000 --- a/test/e2e/kueue.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright 2024 IBM, Red Hat -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Installs a Kueue release on a cluster - -set -euo pipefail -: "${KUEUE_VERSION}" - -echo "Deploying Kueue version $KUEUE_VERSION" -kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml - -# Sleep until the kueue manager is running -echo "Waiting for pods in the kueue-system namespace to become ready" -while [[ $(kubectl get pods -n kueue-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}' | tr ' ' '\n' | sort -u) != "True" ]] -do - echo -n "." && sleep 1; -done -echo "" diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index b3cee102c..cd36d995e 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -35,13 +35,57 @@ import ( // directly managed by Kueue, and asserts successful completion of the training job. func TestMNISTRayJobRayCluster(t *testing.T) { test := With(t) - test.T().Parallel() // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() + localQueue := CreateKueueLocalQueue(test, namespace.Name, "e2e-cluster-queue") - // MNIST training script - mnist := &corev1.ConfigMap{ + // Create MNIST training script + mnist := constructMNISTConfigMap(test, namespace) + mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) + + // Create RayCluster and assign it to the localqueue + rayCluster := constructRayCluster(test, namespace, mnist) + AssignToLocalQueue(rayCluster, localQueue) + rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name) + + test.T().Logf("Waiting for RayCluster %s/%s to be running", rayCluster.Namespace, rayCluster.Name) + test.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) + + // Create RayJob + rayJob := constructRayJob(test, namespace, rayCluster) + rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + + rayDashboardURL := getRayDashboardURL(test, rayCluster.Namespace, rayCluster.Name) + + test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) + rayClient := NewRayClusterClient(rayDashboardURL) + + // Wait for Ray job id to be available, this value is needed for writing logs in defer + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutShort). + Should(WithTransform(RayJobId, Not(BeEmpty()))) + + // Retrieving the job logs once it has completed or timed out + defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) + + test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). + Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) + + // Assert the Ray job has completed successfully + test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). + To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) +} + +func constructMNISTConfigMap(test Test, namespace *corev1.Namespace) *corev1.ConfigMap { + return &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -55,12 +99,10 @@ func TestMNISTRayJobRayCluster(t *testing.T) { }, Immutable: Ptr(true), } - mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) +} - // RayCluster - rayCluster := &rayv1.RayCluster{ +func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.ConfigMap) *rayv1.RayCluster { + return &rayv1.RayCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1.GroupVersion.String(), Kind: "RayCluster", @@ -173,16 +215,10 @@ func TestMNISTRayJobRayCluster(t *testing.T) { }, }, } +} - _, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name) - - test.T().Logf("Waiting for RayCluster %s/%s to be running", rayCluster.Namespace, rayCluster.Name) - test.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). - Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) - - rayJob := &rayv1.RayJob{ +func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayCluster) *rayv1.RayJob { + return &rayv1.RayJob{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1.GroupVersion.String(), Kind: "RayJob", @@ -220,29 +256,6 @@ func TestMNISTRayJobRayCluster(t *testing.T) { }, }, } - rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) - - rayDashboardURL := getRayDashboardURL(test, rayCluster.Namespace, rayCluster.Name) - - test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) - rayClient := NewRayClusterClient(rayDashboardURL) - - // Wait for Ray job id to be available, this value is needed for writing logs in defer - test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutShort). - Should(WithTransform(RayJobId, Not(BeEmpty()))) - - // Retrieving the job logs once it has completed or timed out - defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) - - test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) - test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). - Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) - - // Assert the Ray job has completed successfully - test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). - To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) } func getRayDashboardURL(test Test, namespace, rayClusterName string) url.URL { diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 29ad8926d..8d0bb2eee 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -57,6 +57,17 @@ roleRef: name: e2e-controller-rayclusters EOF +echo "Deploying Kueue $KUEUE_VERSION" +kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml + +# Sleep until the kueue manager is running +echo "Waiting for pods in the kueue-system namespace to become ready" +while [[ $(kubectl get pods -n kueue-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}' | tr ' ' '\n' | sort -u) != "True" ]] +do + echo -n "." && sleep 1; +done +echo "" + echo Creating Kueue ResourceFlavor and ClusterQueue cat < Date: Tue, 7 May 2024 12:12:16 -0400 Subject: [PATCH 219/369] add AppWrapper tests --- test/e2e/mnist_pytorch_appwrapper_test.go | 165 ++++++++++++++++++++++ test/e2e/mnist_rayjob_raycluster_test.go | 83 +++++++++++ 2 files changed, 248 insertions(+) create mode 100644 test/e2e/mnist_pytorch_appwrapper_test.go diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go new file mode 100644 index 000000000..e918b15b1 --- /dev/null +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -0,0 +1,165 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + mcadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + . "github.com/project-codeflare/codeflare-common/support" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +// Trains the MNIST dataset as a batch Job in an AppWrapper, and asserts successful completion of the training job. +func TestMNISTPyTorchAppWrapper(t *testing.T) { + test := With(t) + test.T().Parallel() + + // Create a namespace and localqueue in that namespace + namespace := test.NewTestNamespace() + localQueue := CreateKueueLocalQueue(test, namespace.Name, "e2e-cluster-queue") + + // Test configuration + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-mcad", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }, + Immutable: Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // Batch Job + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "job", + Image: GetPyTorchImage(), + Env: []corev1.EnvVar{ + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, + {Name: "PIP_INDEX_URL", Value: GetPipIndexURL()}, + {Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()}, + }, + Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + { + Name: "workdir", + MountPath: "/workdir", + }, + }, + WorkingDir: "/workdir", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + // Create an AppWrapper resource + aw := &mcadv1beta2.AppWrapper{ + TypeMeta: metav1.TypeMeta{ + APIVersion: mcadv1beta2.GroupVersion.String(), + Kind: "AppWrapper", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, + }, + Spec: mcadv1beta2.AppWrapperSpec{ + Components: []mcadv1beta2.AppWrapperComponent{ + { + Template: Raw(test, job), + }, + }, + }, + } + + appWrapperResource := mcadv1beta2.GroupVersion.WithResource("appwrappers") + awMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) + test.Expect(err).NotTo(HaveOccurred()) + unstruct := unstructured.Unstructured{Object: awMap} + _, err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) + + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning))) + + test.T().Logf("Waiting for AppWrapper %s/%s to complete", job.Namespace, job.Name) + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( + Or( + WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperSucceeded)), + WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperFailed)), + )) +} diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index cd36d995e..9279980eb 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -23,18 +23,22 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" . "github.com/project-codeflare/codeflare-common/support" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" ) // Trains the MNIST dataset as a RayJob, executed by a Ray cluster // directly managed by Kueue, and asserts successful completion of the training job. func TestMNISTRayJobRayCluster(t *testing.T) { test := With(t) + test.T().Parallel() // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() @@ -84,6 +88,85 @@ func TestMNISTRayJobRayCluster(t *testing.T) { To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) } +// Same as TestMNISTRayJobRayCluster, except the RayCluster is wrapped in an AppWrapper +func TestMNISTRayJobRayClusterAppWrapper(t *testing.T) { + test := With(t) + test.T().Parallel() + test.T().Skip("Disabled due to Kueue 0.6.x overly eager reconcilliation bug") + + // Create a namespace and localqueue in that namespace + namespace := test.NewTestNamespace() + localQueue := CreateKueueLocalQueue(test, namespace.Name, "e2e-cluster-queue") + + // Create MNIST training script + mnist := constructMNISTConfigMap(test, namespace) + mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) + + // Create RayCluster, wrap in AppWrapper and assign to localqueue + rayCluster := constructRayCluster(test, namespace, mnist) + aw := &mcadv1beta2.AppWrapper{ + TypeMeta: metav1.TypeMeta{ + APIVersion: mcadv1beta2.GroupVersion.String(), + Kind: "AppWrapper", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: rayCluster.Name, + Namespace: namespace.Name, + Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, + }, + Spec: mcadv1beta2.AppWrapperSpec{ + Components: []mcadv1beta2.AppWrapperComponent{ + { + Template: Raw(test, rayCluster), + }, + }, + }, + } + appWrapperResource := mcadv1beta2.GroupVersion.WithResource("appwrappers") + awMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) + test.Expect(err).NotTo(HaveOccurred()) + unstruct := unstructured.Unstructured{Object: awMap} + _, err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) + + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning))) + + test.T().Logf("Waiting for RayCluster %s/%s to be running", rayCluster.Namespace, rayCluster.Name) + test.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) + + // Create RayJob + rayJob := constructRayJob(test, namespace, rayCluster) + rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + + rayDashboardURL := getRayDashboardURL(test, rayCluster.Namespace, rayCluster.Name) + + test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) + rayClient := NewRayClusterClient(rayDashboardURL) + + // Wait for Ray job id to be available, this value is needed for writing logs in defer + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutShort). + Should(WithTransform(RayJobId, Not(BeEmpty()))) + + // Retrieving the job logs once it has completed or timed out + defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) + + test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). + Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) + + // Assert the Ray job has completed successfully + test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). + To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) +} + func constructMNISTConfigMap(test Test, namespace *corev1.Namespace) *corev1.ConfigMap { return &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ From 8993b2d185e80c1a82b8bc90bbb221ba6af0b726 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 13 May 2024 20:57:07 -0400 Subject: [PATCH 220/369] update to appwrapper v0.11.1 --- Makefile | 2 +- README.md | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 5 ++++- go.sum | 10 ++++++++-- test/e2e/mnist_rayjob_raycluster_test.go | 1 - 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 21dd914d9..6b9954963 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller - APPWRAPPER_VERSION ?= v0.10.0 + APPWRAPPER_VERSION ?= v0.11.1 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/README.md b/README.md index cf08ef684..96b996459 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ CodeFlare Stack Compatibility Matrix | CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | | CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | -| AppWrapper | [v0.9.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.9.0) | +| AppWrapper | [v0.11.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.11.1) | ## Development diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 61bbafad3..3896466a1 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.10.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.11.1 diff --git a/go.mod b/go.mod index 37990383f..4d95b1c86 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.11.0 + github.com/project-codeflare/appwrapper v0.11.1 github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 @@ -41,6 +41,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect @@ -72,6 +73,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -99,6 +101,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.29.2 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/kubernetes v1.29.2 // indirect sigs.k8s.io/jobset v0.3.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 8292b8b0d..2accf6fb2 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -224,6 +226,8 @@ github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5n github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= @@ -238,8 +242,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.11.0 h1:5XeZuUUkC5N03M7q2IyhSL4TqPjyDfhWJ2YKERkBFuw= -github.com/project-codeflare/appwrapper v0.11.0/go.mod h1:FMQ2lI3fz6LakUVXgN1FTdpsc3BBkNIZZgtMmM9J5UM= +github.com/project-codeflare/appwrapper v0.11.1 h1:bQ+OLzO+osR1Lo9nNl4Lk/QIFl0y8Cf6byPSPFJfkH4= +github.com/project-codeflare/appwrapper v0.11.1/go.mod h1:MLmgm7cVHgeatBKhANSpos7GcB0bdllaWF9c1jEJKho= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f h1:9Uron4ej4Tt5ULX5CMzjmPqIZu3q/m07d4jhbNSwdPY= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -520,6 +524,8 @@ k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kubernetes v1.29.2 h1:8hh1cntqdulanjQt7wSSSsJfBgOyx6fUdFWslvGL5m0= +k8s.io/kubernetes v1.29.2/go.mod h1:xZPKU0yO0CBbLTnbd+XGyRmmtmaVuJykDb8gNCkeeUE= k8s.io/metrics v0.29.1 h1:qutc3aIPMCniMuEApuLaeYX47rdCn8eycVDx7R6wMlQ= k8s.io/metrics v0.29.1/go.mod h1:JrbV2U71+v7d/9qb90UVKL8r0uJ6Z2Hy4V7mDm05cKs= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 9279980eb..b7f039d91 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -92,7 +92,6 @@ func TestMNISTRayJobRayCluster(t *testing.T) { func TestMNISTRayJobRayClusterAppWrapper(t *testing.T) { test := With(t) test.T().Parallel() - test.T().Skip("Disabled due to Kueue 0.6.x overly eager reconcilliation bug") // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() From c4e79bf09d39285ccdc2c5258887f01c715ce2a6 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 15 May 2024 14:03:39 -0400 Subject: [PATCH 221/369] address smaller review comments --- .../workflows/project-codeflare-release.yml | 22 +++++++++++++++++-- .github/workflows/tag-and-build.yml | 5 +++++ README.md | 3 ++- config/crd/kustomization.yaml | 4 ++-- .../crd/patches/webhook_in_appwrappers.yaml | 16 -------------- main.go | 19 +++++++++------- 6 files changed, 40 insertions(+), 29 deletions(-) delete mode 100644 config/crd/patches/webhook_in_appwrappers.yaml diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index ae2dec938..93f7b6e07 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -19,6 +19,9 @@ on: kuberay-version: description: 'Tested version of KubeRay (for example: v0.0.0)' required: true + kueue-version: + description: 'Tested version of Kueue (for example: v0.0.0)' + required: true is-stable: description: 'Select if the built images should be tagged as stable' required: true @@ -48,13 +51,14 @@ jobs: echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}" echo "Tested AppWrapper Version: ${{ github.event.inputs.appwrapper-version }}" echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}" + echo "Tested Kueue Version: ${{ github.event.inputs.kueue-version }}" echo "Is Stable: ${{ github.event.inputs.is-stable }}" echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" echo "Quay Organization: ${{ github.event.inputs.quay-organization }}" echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" release-codeflare-sdk: - needs: [check-appwrapper-version, check-kuberay-version] + needs: [check-appwrapper-version, check-kuberay-version, check-kueue-version] runs-on: ubuntu-latest steps: @@ -94,7 +98,7 @@ jobs: steps: - name: Release CodeFlare operator run: | - gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field kueue-version=${{ github.event.inputs.kueue-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash @@ -136,3 +140,17 @@ jobs: echo "KubeRay release with version ${{ github.event.inputs.kuberay-version }} does not exist. Please select an existing version." exit 1 fi + + check-kueue-version: + runs-on: ubuntu-latest + + steps: + - name: Check if Kueue release exists + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/kubernetes-sigs/kueue/releases/tag/${{ github.event.inputs.kueue-version }}) + if [[ "$status_code" == "200" ]]; then + echo "Kueue release with version ${{ github.event.inputs.kueue-version }} exists and available to use." + else + echo "Kueue release with version ${{ github.event.inputs.kueue-version }} does not exist. Please select an existing version." + exit 1 + fi diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index c4e33b312..0e59b5625 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -24,6 +24,10 @@ on: description: 'Tested version of KubeRay' required: true default: 'v0.0.0-dev' + kueue-version: + description: 'Tested version of Kueue' + required: true + default: 'v0.0.0-dev' is-stable: description: 'Select if the built image should be tagged as stable' required: true @@ -89,6 +93,7 @@ jobs: sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md sed -i -E "s|(.*AppWrapper.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.appwrapper-version }}\2${{ github.event.inputs.appwrapper-version }}\3|" README.md sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md + sed -i -E "s|(.*Kueue.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kueue-version }}\2${{ github.event.inputs.kueue-version }}\3|" README.md - name: Adjust AppWrapper dependency in the code run: | diff --git a/README.md b/README.md index 96b996459..890f28de9 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,9 @@ CodeFlare Stack Compatibility Matrix |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | | CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | -| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | AppWrapper | [v0.11.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.11.1) | +| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | +| Kueue | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2) | ## Development diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 042f878f3..5523c011e 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,10 +4,10 @@ resources: - crd-appwrapper.yml -patches: +#patches: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -- path: patches/webhook_in_appwrappers.yaml +# - path: patches/webhook_in_appwrappers.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [WEBHOOK] To enable webhook, uncomment the following section diff --git a/config/crd/patches/webhook_in_appwrappers.yaml b/config/crd/patches/webhook_in_appwrappers.yaml deleted file mode 100644 index bc147c122..000000000 --- a/config/crd/patches/webhook_in_appwrappers.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: appwrappers.workload.codeflare.dev -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/main.go b/main.go index b25d549f4..07b508e43 100644 --- a/main.go +++ b/main.go @@ -249,29 +249,32 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m } // AppWrapper webhook doesn't depend on WorkloadAPI availablity but does need certsReady - go func() { - <-certsReady - setupLog.Info("Setting up AppWrapper webhooks") - exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") - }() + go setupAppWrapperWebhooks(mgr, cfg, certsReady) if isAPIAvailable(ctx, mgr, workloadAPI) { - setupLog.Info("Workload API available; enabling AppWrappers") + setupLog.Info("Workload API available, enabling AppWrappers") go setupAppWrapperController(mgr, cfg, certsReady) return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config) } else { // If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly // restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator. // It is not possible to add an indexer once the mgr has started so, a restart if the only avenue. - setupLog.Info("Workload API not available; setting up waiter for Workload API availability") + setupLog.Info("Workload API not available, setting up waiter for Workload API availability") go waitForAPI(ctx, mgr, workloadAPI, func() { - setupLog.Info("Workload API now available; triggering controller restart") + setupLog.Info("Workload API now available, triggering controller restart") cancel() }) return nil } } +func setupAppWrapperWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { + setupLog.Info("Waiting for certificate generation to complete") + <-certsReady + setupLog.Info("Setting up AppWrapper webhooks") + exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") +} + func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { setupLog.Info("Waiting for certificate generation to complete") <-certsReady From 50a94ee861e88cd69337aa77498dc9329655fc74 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 15 May 2024 16:25:01 -0400 Subject: [PATCH 222/369] simplify mock AppWrapper webhooks --- main.go | 10 ++++------ pkg/config/config.go | 2 -- pkg/controllers/appwrapper_webhook.go | 26 ++++++++++---------------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index 07b508e43..a17d8e66d 100644 --- a/main.go +++ b/main.go @@ -145,9 +145,8 @@ func main() { CertGeneratorImage: "registry.access.redhat.com/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", }, AppWrapper: &config.AppWrapperConfiguration{ - Enabled: ptr.To(false), - ExternalController: ptr.To(false), - Config: awconfig.NewAppWrapperConfig(), + Enabled: ptr.To(false), + Config: awconfig.NewAppWrapperConfig(), }, } @@ -239,11 +238,10 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { setupLog.Info("Embedded AppWrapper controller is disabled by config") - externalController := cfg.AppWrapper != nil && ptr.Deref(cfg.AppWrapper.ExternalController, false) go func() { <-certsReady - setupLog.Info("Setting up mock AppWrapper webhooks", "externalController", externalController) - exitOnError(controllers.SetupMockAppWrapperWebhooks(mgr, externalController), "unable to setup AppWrapper webhooks") + setupLog.Info("Setting up mock AppWrapper webhooks") + exitOnError(controllers.SetupMockAppWrapperWebhooks(mgr), "unable to setup AppWrapper webhooks") }() return nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index 4e3fde3b0..0df71d451 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,8 +38,6 @@ type CodeFlareOperatorConfiguration struct { type AppWrapperConfiguration struct { // Enabled controls whether or not the AppWrapper Controller is enabled Enabled *bool `json:"enabled,omitempty"` - // ExternalController indicates that an AppWrapper Controller external to the Codeflare Operator is deployed - ExternalController *bool `json:"externalController,omitempty"` // AppWrapper contains the AppWrapper controller configuration Config *awconfig.AppWrapperConfig `json:"config,omitempty"` } diff --git a/pkg/controllers/appwrapper_webhook.go b/pkg/controllers/appwrapper_webhook.go index c019c4da3..f30d25cdb 100644 --- a/pkg/controllers/appwrapper_webhook.go +++ b/pkg/controllers/appwrapper_webhook.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" awv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" @@ -36,36 +35,31 @@ import ( //+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create //+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=list -type appwrapperWebhook struct { - externalController bool +type mockAppWrapperWebhook struct { } -var _ webhook.CustomDefaulter = &appwrapperWebhook{} +var _ webhook.CustomDefaulter = &mockAppWrapperWebhook{} -func (w *appwrapperWebhook) Default(ctx context.Context, obj runtime.Object) error { +func (w *mockAppWrapperWebhook) Default(ctx context.Context, obj runtime.Object) error { return nil } -var _ webhook.CustomValidator = &appwrapperWebhook{} +var _ webhook.CustomValidator = &mockAppWrapperWebhook{} -func (w *appwrapperWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { - if w.externalController { - return nil, nil - } else { - return nil, fmt.Errorf("AppWrappers disabled by CodeFlare operator configuration") - } +func (w *mockAppWrapperWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } -func (w *appwrapperWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { +func (w *mockAppWrapperWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { return nil, nil } -func (w *appwrapperWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) { +func (w *mockAppWrapperWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) { return nil, nil } -func SetupMockAppWrapperWebhooks(mgr ctrl.Manager, externalController bool) error { - wh := &appwrapperWebhook{externalController: externalController} +func SetupMockAppWrapperWebhooks(mgr ctrl.Manager) error { + wh := &mockAppWrapperWebhook{} return ctrl.NewWebhookManagedBy(mgr). For(&awv1beta2.AppWrapper{}). WithDefaulter(wh). From 852e94c4ac773905253ecc5e9e6b663d4d98f08b Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 15 May 2024 16:40:49 -0400 Subject: [PATCH 223/369] inline AppWrapperConfig struct --- main.go | 8 ++++---- pkg/config/config.go | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index a17d8e66d..0bb350fdc 100644 --- a/main.go +++ b/main.go @@ -146,7 +146,7 @@ func main() { }, AppWrapper: &config.AppWrapperConfiguration{ Enabled: ptr.To(false), - Config: awconfig.NewAppWrapperConfig(), + Config: *awconfig.NewAppWrapperConfig(), }, } @@ -252,7 +252,7 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m if isAPIAvailable(ctx, mgr, workloadAPI) { setupLog.Info("Workload API available, enabling AppWrappers") go setupAppWrapperController(mgr, cfg, certsReady) - return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config) + return awctrl.SetupIndexers(ctx, mgr, &cfg.AppWrapper.Config) } else { // If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly // restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator. @@ -270,14 +270,14 @@ func setupAppWrapperWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConf setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Setting up AppWrapper webhooks") - exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") + exitOnError(awctrl.SetupWebhooks(mgr, &cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") } func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Setting up AppWrapper controller") - exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper controller") + exitOnError(awctrl.SetupControllers(mgr, &cfg.AppWrapper.Config), "unable to setup AppWrapper controller") } // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update diff --git a/pkg/config/config.go b/pkg/config/config.go index 0df71d451..c4a5be090 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,8 +38,10 @@ type CodeFlareOperatorConfiguration struct { type AppWrapperConfiguration struct { // Enabled controls whether or not the AppWrapper Controller is enabled Enabled *bool `json:"enabled,omitempty"` + // AppWrapper contains the AppWrapper controller configuration - Config *awconfig.AppWrapperConfig `json:"config,omitempty"` + // +optional + Config awconfig.AppWrapperConfig `json:"config,omitempty"` } type KubeRayConfiguration struct { From 73f7e1a9bd6d68cd63534fa5765687e87ae66da6 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 15 May 2024 16:52:35 -0400 Subject: [PATCH 224/369] missed makefile adjustment for kueue in tag-and-build --- .github/workflows/tag-and-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 0e59b5625..3b3f4a8e4 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -103,6 +103,10 @@ jobs: run: | sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile + - name: Adjust Kueue dependency in the code + run: | + sed -i -E "s/(.*KUEUE_VERSION \?= ).*/\1${{ github.event.inputs.kueue-version }}/" Makefile + - name: Update image version in params.env run: | VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env From d75fcdc701eb0221a588ddc91624f3ac9ff7ce57 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 16 May 2024 09:59:30 -0400 Subject: [PATCH 225/369] remove kustomize patches/config for conversion webhook --- config/crd/kustomization.yaml | 11 ----------- config/crd/kustomizeconfig.yaml | 19 ------------------- 2 files changed, 30 deletions(-) delete mode 100644 config/crd/kustomizeconfig.yaml diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 5523c011e..ab7273833 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,14 +3,3 @@ # It should be run by config/default resources: - crd-appwrapper.yml - -#patches: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -# - path: patches/webhook_in_appwrappers.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [WEBHOOK] To enable webhook, uncomment the following section -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a9..000000000 --- a/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations From 980311d4ff8ca8475c78b6377da736c69ba96b82 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 16 May 2024 10:05:49 -0400 Subject: [PATCH 226/369] drop mock appwrapper webhook --- main.go | 5 ---- pkg/controllers/appwrapper_webhook.go | 43 --------------------------- 2 files changed, 48 deletions(-) diff --git a/main.go b/main.go index 0bb350fdc..ae055508c 100644 --- a/main.go +++ b/main.go @@ -238,11 +238,6 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { setupLog.Info("Embedded AppWrapper controller is disabled by config") - go func() { - <-certsReady - setupLog.Info("Setting up mock AppWrapper webhooks") - exitOnError(controllers.SetupMockAppWrapperWebhooks(mgr), "unable to setup AppWrapper webhooks") - }() return nil } diff --git a/pkg/controllers/appwrapper_webhook.go b/pkg/controllers/appwrapper_webhook.go index f30d25cdb..f9b9b982e 100644 --- a/pkg/controllers/appwrapper_webhook.go +++ b/pkg/controllers/appwrapper_webhook.go @@ -16,17 +16,6 @@ limitations under the License. package controllers -import ( - "context" - - awv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" - - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - // webhook configuration //+kubebuilder:webhook:path=/mutate-workload-codeflare-dev-v1beta2-appwrapper,mutating=true,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create,versions=v1beta2,name=mappwrapper.kb.io,admissionReviewVersions=v1 //+kubebuilder:webhook:path=/validate-workload-codeflare-dev-v1beta2-appwrapper,mutating=false,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create;update,versions=v1beta2,name=vappwrapper.kb.io,admissionReviewVersions=v1 @@ -34,35 +23,3 @@ import ( // permissions needed by the "real" Webhook in the appwrapper project to enable SubjectAccessReview //+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create //+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=list - -type mockAppWrapperWebhook struct { -} - -var _ webhook.CustomDefaulter = &mockAppWrapperWebhook{} - -func (w *mockAppWrapperWebhook) Default(ctx context.Context, obj runtime.Object) error { - return nil -} - -var _ webhook.CustomValidator = &mockAppWrapperWebhook{} - -func (w *mockAppWrapperWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { - return nil, nil -} - -func (w *mockAppWrapperWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { - return nil, nil -} - -func (w *mockAppWrapperWebhook) ValidateDelete(context.Context, runtime.Object) (admission.Warnings, error) { - return nil, nil -} - -func SetupMockAppWrapperWebhooks(mgr ctrl.Manager) error { - wh := &mockAppWrapperWebhook{} - return ctrl.NewWebhookManagedBy(mgr). - For(&awv1beta2.AppWrapper{}). - WithDefaulter(wh). - WithValidator(wh). - Complete() -} From 92e6eb5a41b8eb802b6d76bf9706198c4713aded Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 16 May 2024 10:18:20 -0400 Subject: [PATCH 227/369] inline appwrapper config struct at the json level --- main.go | 8 ++++---- pkg/config/config.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index ae055508c..d78d82564 100644 --- a/main.go +++ b/main.go @@ -146,7 +146,7 @@ func main() { }, AppWrapper: &config.AppWrapperConfiguration{ Enabled: ptr.To(false), - Config: *awconfig.NewAppWrapperConfig(), + Config: awconfig.NewAppWrapperConfig(), }, } @@ -247,7 +247,7 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m if isAPIAvailable(ctx, mgr, workloadAPI) { setupLog.Info("Workload API available, enabling AppWrappers") go setupAppWrapperController(mgr, cfg, certsReady) - return awctrl.SetupIndexers(ctx, mgr, &cfg.AppWrapper.Config) + return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config) } else { // If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly // restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator. @@ -265,14 +265,14 @@ func setupAppWrapperWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConf setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Setting up AppWrapper webhooks") - exitOnError(awctrl.SetupWebhooks(mgr, &cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") + exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") } func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Setting up AppWrapper controller") - exitOnError(awctrl.SetupControllers(mgr, &cfg.AppWrapper.Config), "unable to setup AppWrapper controller") + exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper controller") } // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update diff --git a/pkg/config/config.go b/pkg/config/config.go index c4a5be090..49749dfec 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -41,7 +41,7 @@ type AppWrapperConfiguration struct { // AppWrapper contains the AppWrapper controller configuration // +optional - Config awconfig.AppWrapperConfig `json:"config,omitempty"` + Config *awconfig.AppWrapperConfig `json:",inline"` } type KubeRayConfiguration struct { From dae4accf014e91cf8df93e97ce6d372e59754917 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 20 May 2024 09:44:42 -0400 Subject: [PATCH 228/369] add Kueue and AppWrapper to modules target in Makefile --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 6b9954963..3b6aae41a 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,8 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. go get github.com/ray-project/kuberay/ray-operator@$(KUBERAY_VERSION) + go get sigs.k8s.io/kueue@$(KUEUE_VERSION) + go get github.com/project-codeflare/appwrapper@$(APPWRAPPER_VERSION) go mod tidy .PHONY: build From e71b554a4ca88b4bfda03356b22b4b600d0644ec Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 20 May 2024 22:02:48 +0200 Subject: [PATCH 229/369] Enable mTLS for e2e tests and HTTPS for KinD cluster --- config/e2e/config.yaml | 2 +- test/e2e/kind.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index e216371fe..538b7ad5d 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -7,6 +7,6 @@ data: kuberay: rayDashboardOAuthEnabled: false ingressDomain: "kind" - mTLSEnabled: false + certGeneratorImage: quay.io/project-codeflare/ray:latest-py39-cu118 appwrapper: enabled: true diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 7b7e9053e..29b19c347 100755 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -28,6 +28,9 @@ nodes: - containerPort: 80 hostPort: 80 protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP kubeadmConfigPatches: - | kind: InitConfiguration @@ -39,4 +42,5 @@ EOF echo "Deploying Ingress controller into KinD cluster" curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${INGRESS_NGINX_VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n - --status-update-interval=10/g" | kubectl apply -f - kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true" +kubectl patch deploy --type json --patch '[{"op":"add","path": "/spec/template/spec/containers/0/args/-","value":"--enable-ssl-passthrough"}]' ingress-nginx-controller -n ingress-nginx kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all From c604296ae3f03043ff38efcaa35623cde1877688 Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 21 May 2024 16:37:37 -0400 Subject: [PATCH 230/369] upgrade AppWrapper from v0.11.1 to v0.12.0 --- Makefile | 2 +- README.md | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 101 ++++++++++++++++++++--- go.mod | 3 +- go.sum | 6 +- 6 files changed, 97 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 3b6aae41a..6f3d20c71 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller - APPWRAPPER_VERSION ?= v0.11.1 + APPWRAPPER_VERSION ?= v0.12.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/README.md b/README.md index 890f28de9..395d1e021 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ CodeFlare Stack Compatibility Matrix |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | | CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | -| AppWrapper | [v0.11.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.11.1) | +| AppWrapper | [v0.12.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.12.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2) | diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 3896466a1..994e183b3 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.11.1 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.12.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index dc134a6dd..f8cbd4e6d 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -5,16 +5,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.14.0 name: appwrappers.workload.codeflare.dev spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: webhook-service - namespace: system - path: /convert - conversionReviewVersions: - - v1 group: workload.codeflare.dev names: kind: AppWrapper @@ -168,6 +158,97 @@ spec: status: description: AppWrapperStatus defines the observed state of the appwrapper properties: + componentStatus: + description: ComponentStatus parallels the Components array in the Spec and tracks the actually deployed resources + items: + description: AppWrapperComponentStatus tracks the status of a single managed Component + properties: + apiVersion: + description: APIVersion is the APIVersion of the Component + type: string + conditions: + description: |- + Conditions hold the latest available observations of the Component's current state. + + + The type of the condition could be: + + + - ResourcesDeployed: The component is deployed on the cluster + items: + description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + kind: + description: Kind is the Kind of the Component + type: string + name: + description: Name is the name of the Component + type: string + required: + - apiVersion + - kind + - name + type: object + type: array conditions: description: |- Conditions hold the latest available observations of the AppWrapper current state. diff --git a/go.mod b/go.mod index 4d95b1c86..637bcf5e9 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.11.1 + github.com/project-codeflare/appwrapper v0.12.0 github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 @@ -101,7 +101,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.29.2 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/kubernetes v1.29.2 // indirect sigs.k8s.io/jobset v0.3.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 2accf6fb2..59ec28e4f 100644 --- a/go.sum +++ b/go.sum @@ -242,8 +242,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.11.1 h1:bQ+OLzO+osR1Lo9nNl4Lk/QIFl0y8Cf6byPSPFJfkH4= -github.com/project-codeflare/appwrapper v0.11.1/go.mod h1:MLmgm7cVHgeatBKhANSpos7GcB0bdllaWF9c1jEJKho= +github.com/project-codeflare/appwrapper v0.12.0 h1:QMxryBPK6ir3VK6Qx4NWOA05/s4xU6uDHI/nXmLznvw= +github.com/project-codeflare/appwrapper v0.12.0/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f h1:9Uron4ej4Tt5ULX5CMzjmPqIZu3q/m07d4jhbNSwdPY= github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -524,8 +524,6 @@ k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/kubernetes v1.29.2 h1:8hh1cntqdulanjQt7wSSSsJfBgOyx6fUdFWslvGL5m0= -k8s.io/kubernetes v1.29.2/go.mod h1:xZPKU0yO0CBbLTnbd+XGyRmmtmaVuJykDb8gNCkeeUE= k8s.io/metrics v0.29.1 h1:qutc3aIPMCniMuEApuLaeYX47rdCn8eycVDx7R6wMlQ= k8s.io/metrics v0.29.1/go.mod h1:JrbV2U71+v7d/9qb90UVKL8r0uJ6Z2Hy4V7mDm05cKs= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= From 35c458dde76821b655e8c2c6f6c337935e72d219 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Thu, 23 May 2024 18:07:32 +0000 Subject: [PATCH 231/369] Update dependency versions for release v1.4.2 --- README.md | 4 ++-- config/manager/params.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 395d1e021..887fd6718 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1) | -| CodeFlare-SDK | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1) | +| CodeFlare Operator | [v1.4.2](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.2) | +| CodeFlare-SDK | [v0.16.2](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.2) | | AppWrapper | [v0.12.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.12.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2) | diff --git a/config/manager/params.env b/config/manager/params.env index e3c1fd4c7..0824a531b 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.1 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.2 namespace=opendatahub From ea80802ccf055c32357d76f63ce8f02a224d2d55 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 8 May 2024 16:11:07 +0100 Subject: [PATCH 232/369] Check if workerGroupSpec is not empty before accessing it --- pkg/controllers/raycluster_webhook.go | 29 ++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 3e0b29624..d1500559a 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -100,11 +100,11 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } // WorkerGroupSpec - - // Append the list of environment variables for the worker container - for _, envVar := range envVarList() { - rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) - } + if len(rayCluster.Spec.WorkerGroupSpecs) != 0 { + // Append the list of environment variables for the worker container + for _, envVar := range envVarList() { + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) + } // Append the CA volumes for _, caVol := range caVolumes(rayCluster) { @@ -118,6 +118,7 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err // Append the create-cert Init Container rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(w.Config), withContainerName(initContainerName)) + } } return nil @@ -386,6 +387,10 @@ func validateHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.Kube func validateWorkerInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) field.ErrorList { var allErrors field.ErrorList + if len(rayCluster.Spec.WorkerGroupSpecs) == 0 { + return allErrors + } + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(config), byContainerName, field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "initContainers"), "create-cert Init Container is immutable"); err != nil { @@ -404,10 +409,12 @@ func validateCaVolumes(rayCluster *rayv1.RayCluster) field.ErrorList { "ca-vol and server-cert Secret volumes are immutable"); err != nil { allErrors = append(allErrors, err) } - if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, byVolumeName, - field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "volumes"), - "ca-vol and server-cert Secret volumes are immutable"); err != nil { - allErrors = append(allErrors, err) + if len(rayCluster.Spec.WorkerGroupSpecs) != 0 { + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, byVolumeName, + field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "volumes"), + "ca-vol and server-cert Secret volumes are immutable"); err != nil { + allErrors = append(allErrors, err) + } } } @@ -431,6 +438,10 @@ func validateHeadEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { func validateWorkerEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { var allErrors field.ErrorList + if len(rayCluster.Spec.WorkerGroupSpecs) == 0 { + return allErrors + } + for _, envVar := range envVarList() { if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, byEnvVarName, field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "containers", strconv.Itoa(0), "env"), From ff3f43a11444f01f67aaed14ddfd61b281a2681d Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 9 May 2024 11:06:26 +0100 Subject: [PATCH 233/369] Add positive and negative test cases for RayCluster Webhook functionality --- pkg/controllers/raycluster_webhook.go | 65 +- pkg/controllers/raycluster_webhook_test.go | 668 +++++++++++++++++++++ 2 files changed, 701 insertions(+), 32 deletions(-) create mode 100644 pkg/controllers/raycluster_webhook_test.go diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index d1500559a..dab128115 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -100,24 +100,26 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } // WorkerGroupSpec - if len(rayCluster.Spec.WorkerGroupSpecs) != 0 { + for i := range rayCluster.Spec.WorkerGroupSpecs { + workerSpec := &rayCluster.Spec.WorkerGroupSpecs[i] + // Append the list of environment variables for the worker container for _, envVar := range envVarList() { - rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) + workerSpec.Template.Spec.Containers[0].Env = upsert(workerSpec.Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) } - // Append the CA volumes - for _, caVol := range caVolumes(rayCluster) { - rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) - } + // Append the CA volumes + for _, caVol := range caVolumes(rayCluster) { + workerSpec.Template.Spec.Volumes = upsert(workerSpec.Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) + } - // Append the certificate volume mounts - for _, mount := range certVolumeMounts() { - rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].VolumeMounts = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].VolumeMounts, mount, byVolumeMountName) - } + // Append the certificate volume mounts + for _, mount := range certVolumeMounts() { + workerSpec.Template.Spec.Containers[0].VolumeMounts = upsert(workerSpec.Template.Spec.Containers[0].VolumeMounts, mount, byVolumeMountName) + } - // Append the create-cert Init Container - rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(w.Config), withContainerName(initContainerName)) + // Append the create-cert Init Container + workerSpec.Template.Spec.InitContainers = upsert(workerSpec.Template.Spec.InitContainers, rayWorkerInitContainer(w.Config), withContainerName(initContainerName)) } } @@ -387,14 +389,13 @@ func validateHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.Kube func validateWorkerInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) field.ErrorList { var allErrors field.ErrorList - if len(rayCluster.Spec.WorkerGroupSpecs) == 0 { - return allErrors - } - - if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(config), byContainerName, - field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "initContainers"), - "create-cert Init Container is immutable"); err != nil { - allErrors = append(allErrors, err) + for i := range rayCluster.Spec.WorkerGroupSpecs { + workerSpec := &rayCluster.Spec.WorkerGroupSpecs[i] + if err := contains(workerSpec.Template.Spec.InitContainers, rayWorkerInitContainer(config), byContainerName, + field.NewPath("spec", "workerGroupSpecs", strconv.Itoa(i), "template", "spec", "initContainers"), + "create-cert Init Container is immutable"); err != nil { + allErrors = append(allErrors, err) + } } return allErrors @@ -409,9 +410,10 @@ func validateCaVolumes(rayCluster *rayv1.RayCluster) field.ErrorList { "ca-vol and server-cert Secret volumes are immutable"); err != nil { allErrors = append(allErrors, err) } - if len(rayCluster.Spec.WorkerGroupSpecs) != 0 { - if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, byVolumeName, - field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "volumes"), + for i := range rayCluster.Spec.WorkerGroupSpecs { + workerSpec := &rayCluster.Spec.WorkerGroupSpecs[i] + if err := contains(workerSpec.Template.Spec.Volumes, caVol, byVolumeName, + field.NewPath("spec", "workerGroupSpecs", strconv.Itoa(i), "template", "spec", "volumes"), "ca-vol and server-cert Secret volumes are immutable"); err != nil { allErrors = append(allErrors, err) } @@ -438,15 +440,14 @@ func validateHeadEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { func validateWorkerEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { var allErrors field.ErrorList - if len(rayCluster.Spec.WorkerGroupSpecs) == 0 { - return allErrors - } - - for _, envVar := range envVarList() { - if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, byEnvVarName, - field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "containers", strconv.Itoa(0), "env"), - "RAY_TLS related environment variables are immutable"); err != nil { - allErrors = append(allErrors, err) + for i := range rayCluster.Spec.WorkerGroupSpecs { + workerSpec := &rayCluster.Spec.WorkerGroupSpecs[i] + for _, envVar := range envVarList() { + if err := contains(workerSpec.Template.Spec.Containers[0].Env, envVar, byEnvVarName, + field.NewPath("spec", "workerGroupSpecs", strconv.Itoa(i), "template", "spec", "containers", strconv.Itoa(0), "env"), + "RAY_TLS related environment variables are immutable"); err != nil { + allErrors = append(allErrors, err) + } } } diff --git a/pkg/controllers/raycluster_webhook_test.go b/pkg/controllers/raycluster_webhook_test.go new file mode 100644 index 000000000..282fcdd45 --- /dev/null +++ b/pkg/controllers/raycluster_webhook_test.go @@ -0,0 +1,668 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "testing" + + . "github.com/onsi/gomega" + "github.com/project-codeflare/codeflare-common/support" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/project-codeflare/codeflare-operator/pkg/config" +) + +var ( + namespace = "test-namespace" + rayClusterName = "test-raycluster" + + rcWebhook = &rayClusterWebhook{ + Config: &config.KubeRayConfiguration{}, + } +) + +func TestRayClusterWebhookDefault(t *testing.T) { + test := support.NewTest(t) + + validRayCluster := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + Namespace: namespace, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{}, + }, + }, + RayStartParams: map[string]string{}, + }, + WorkerGroupSpecs: []rayv1.WorkerGroupSpec{ + { + GroupName: "worker-group-1", + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "worker-container-1", + }, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + { + GroupName: "worker-group-2", + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "worker-container-2", + }, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + }, + }, + } + + // Create the RayClusters + if _, err := test.Client().Ray().RayV1().RayClusters(namespace).Create(test.Ctx(), validRayCluster, metav1.CreateOptions{}); err != nil { + test.T().Fatalf("Failed to create RayCluster: %v", err) + } + + // Call to default function is made + err := rcWebhook.Default(test.Ctx(), runtime.Object(validRayCluster)) + t.Run("Expected no errors on call to Default function", func(t *testing.T) { + test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to Default function") + }) + + t.Run("Expected required OAuth proxy container for the head group", func(t *testing.T) { + test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers). + To( + And( + HaveLen(1), + ContainElement(WithTransform(ContainerName, Equal(oauthProxyContainerName))), + ), + "Expected the OAuth proxy container to be present in the head group", + ) + }) + + t.Run("Expected required OAuth proxy TLS secret and CA volumes for the head group", func(t *testing.T) { + test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes). + To( + And( + HaveLen(3), + ContainElement(WithTransform(VolumeName, Equal(oauthProxyVolumeName))), + ContainElement(WithTransform(VolumeName, Equal("ca-vol"))), + ContainElement(WithTransform(VolumeName, Equal("server-cert"))), + ), + "Expected the OAuth proxy TLS secret and CA volumes to be present in the head group", + ) + }) + + t.Run("Expected required service account name for the head group", func(t *testing.T) { + test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName). + To(Equal(validRayCluster.Name+"-oauth-proxy"), + "Expected the service account name to be set correctly") + }) + + t.Run("Expected required environment variables for the head group", func(t *testing.T) { + test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env). + To( + And( + HaveLen(6), + ContainElement(WithTransform(EnvVarName, Equal("COOKIE_SECRET"))), + ContainElement(WithTransform(EnvVarName, Equal("MY_POD_IP"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_USE_TLS"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_CERT"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_KEY"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_CA_CERT"))), + ), + "Expected the required environment variables to be present in the head group", + ) + }) + + t.Run("Expected required create-cert init container for the head group", func(t *testing.T) { + test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers). + To( + And( + HaveLen(1), + ContainElement(WithTransform(ContainerName, Equal(initContainerName))), + ), + "Expected the create-cert init container to be present in the head group", + ) + }) + + t.Run("Expected required OAuth proxy TLS secret and CA volume mounts for the head group", func(t *testing.T) { + test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].VolumeMounts). + To( + And( + HaveLen(3), + ContainElement(WithTransform(VolumeMountName, Equal(oauthProxyVolumeName))), + ContainElement(WithTransform(VolumeMountName, Equal("ca-vol"))), + ContainElement(WithTransform(VolumeMountName, Equal("server-cert"))), + ), + "Expected the OAuth proxy TLS secret and CA volume mounts to be present in the head group", + ) + }) + + t.Run("Expected required environment variables for each worker group", func(t *testing.T) { + for _, workerGroup := range validRayCluster.Spec.WorkerGroupSpecs { + test.Expect(workerGroup.Template.Spec.Containers[0].Env). + To( + And( + HaveLen(5), + ContainElement(WithTransform(EnvVarName, Equal("MY_POD_IP"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_USE_TLS"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_CERT"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_KEY"))), + ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_CA_CERT"))), + ), + "Expected the required environment variables to be present in each worker group", + ) + } + }) + + t.Run("Expected required CA Volumes for each worker group", func(t *testing.T) { + for _, workerGroup := range validRayCluster.Spec.WorkerGroupSpecs { + test.Expect(workerGroup.Template.Spec.Volumes). + To( + And( + HaveLen(2), + ContainElement(WithTransform(VolumeName, Equal("ca-vol"))), + ContainElement(WithTransform(VolumeName, Equal("server-cert"))), + ), + "Expected the required CA volumes to be present in each worker group", + ) + } + }) + + t.Run("Expected required certificate volume mounts for each worker group", func(t *testing.T) { + for _, workerSpec := range validRayCluster.Spec.WorkerGroupSpecs { + test.Expect(workerSpec.Template.Spec.Containers[0].VolumeMounts). + To( + And( + HaveLen(2), + ContainElement(WithTransform(VolumeMountName, Equal("ca-vol"))), + ContainElement(WithTransform(VolumeMountName, Equal("server-cert"))), + ), + "Expected the required certificate volume mounts to be present in each worker group", + ) + } + }) + + t.Run("Expected required init container for each worker group", func(t *testing.T) { + for _, workerSpec := range validRayCluster.Spec.WorkerGroupSpecs { + test.Expect(workerSpec.Template.Spec.InitContainers). + To( + And( + HaveLen(1), + ContainElement(WithTransform(ContainerName, Equal(initContainerName))), + ), + "Expected the required init container to be present in each worker group", + ) + } + }) + +} + +func TestValidateCreate(t *testing.T) { + test := support.NewTest(t) + + validRayCluster := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + Namespace: namespace, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: oauthProxyContainerName, + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: rayClusterName + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + }, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + rayClusterName + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"" + namespace + "\",\"verb\":\"get\"}}", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: oauthProxyVolumeName, + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: oauthProxyVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: rayClusterName + "-proxy-tls-secret", + }, + }, + }, + }, + ServiceAccountName: rayClusterName + "-oauth-proxy", + }, + }, + RayStartParams: map[string]string{}, + }, + }, + } + + // Create the RayClusters + if _, err := test.Client().Ray().RayV1().RayClusters(namespace).Create(test.Ctx(), validRayCluster, metav1.CreateOptions{}); err != nil { + test.T().Fatalf("Failed to create RayCluster: %v", err) + } + + // Call to ValidateCreate function is made + warnings, err := rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(validRayCluster)) + t.Run("Expected no warnings or errors on call to ValidateCreate function", func(t *testing.T) { + test.Expect(warnings).Should(BeNil(), "Expected no warnings on call to ValidateCreate function") + test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to ValidateCreate function") + }) + + // Negative Test: Invalid RayCluster with EnableIngress set to true + trueBool := true + invalidRayCluster := validRayCluster.DeepCopy() + + t.Run("Negative: Expected errors on call to ValidateCreate function due to EnableIngress set to True", func(t *testing.T) { + invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = &trueBool + _, err := rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateCreate function due to EnableIngress set to True") + }) + + t.Run("Negative: Expected errors on call to ValidateCreate function due to manipulated OAuth Proxy Container", func(t *testing.T) { + for i, headContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if headContainer.Name == oauthProxyContainerName { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[i].Args = []string{"--invalid-arg"} + break + } + } + _, err = rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateCreate function due to manipulated OAuth Proxy Container") + }) + + t.Run("Negative: Expected errors on call to ValidateCreate function due to manipulated OAuth Proxy Volume", func(t *testing.T) { + for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { + if headVolume.Name == oauthProxyVolumeName { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" + break + } + } + _, err = rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateCreate function due to manipulated OAuth Proxy Volume") + }) + + t.Run("Negative: Expected errors on call to ValidateCreate function due to manipulated head group service account name", func(t *testing.T) { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = "invalid-service-account-name" + _, err = rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateCreate function due to manipulated head group service account name") + }) + +} + +func TestValidateUpdate(t *testing.T) { + test := support.NewTest(t) + + validRayCluster := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + Namespace: namespace, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: oauthProxyContainerName, + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: rayClusterName + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + }, + { + Name: "MY_POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + {Name: "RAY_USE_TLS", Value: "1"}, + {Name: "RAY_TLS_SERVER_CERT", Value: "/home/ray/workspace/tls/server.crt"}, + {Name: "RAY_TLS_SERVER_KEY", Value: "/home/ray/workspace/tls/server.key"}, + {Name: "RAY_TLS_CA_CERT", Value: "/home/ray/workspace/tls/ca.crt"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + rayClusterName + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"" + namespace + "\",\"verb\":\"get\"}}", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: oauthProxyVolumeName, + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "create-cert", + Image: "", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-` + rayClusterName + `-` + namespace + `.\nDNS.6 = ` + rayClusterName + `-head-svc.` + namespace + `.svc` + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "ca-vol", + MountPath: "/home/ray/workspace/ca", + ReadOnly: true, + }, + { + Name: "server-cert", + MountPath: "/home/ray/workspace/tls", + ReadOnly: false, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: oauthProxyVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: rayClusterName + "-proxy-tls-secret", + }, + }, + }, + { + Name: "ca-vol", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: `ca-secret-` + rayClusterName, + }, + }, + }, + { + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + ServiceAccountName: rayClusterName + "-oauth-proxy", + }, + }, + RayStartParams: map[string]string{}, + }, + WorkerGroupSpecs: []rayv1.WorkerGroupSpec{ + { + GroupName: "worker-group-1", + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "worker-container-1", + Env: []corev1.EnvVar{ + { + Name: "MY_POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + {Name: "RAY_USE_TLS", Value: "1"}, + {Name: "RAY_TLS_SERVER_CERT", Value: "/home/ray/workspace/tls/server.crt"}, + {Name: "RAY_TLS_SERVER_KEY", Value: "/home/ray/workspace/tls/server.key"}, + {Name: "RAY_TLS_CA_CERT", Value: "/home/ray/workspace/tls/ca.crt"}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "create-cert", + Image: "", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: certVolumeMounts(), + }, + }, + Volumes: []corev1.Volume{ + { + Name: "ca-vol", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: `ca-secret-` + rayClusterName, + }, + }, + }, + { + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + }, + }, + } + + // Create the RayClusters + if _, err := test.Client().Ray().RayV1().RayClusters(namespace).Create(test.Ctx(), validRayCluster, metav1.CreateOptions{}); err != nil { + test.T().Fatalf("Failed to create RayCluster: %v", err) + } + + // Positive Test Case: Valid RayCluster with immutable fields + t.Run("Expected no warnings or errors on call to ValidateUpdate function", func(t *testing.T) { + warnings, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(validRayCluster)) + test.Expect(warnings).Should(BeNil(), "Expected no warnings on call to ValidateUpdate function") + test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to ValidateUpdate function") + }) + + // Negative Test Cases + trueBool := true + invalidRayCluster := validRayCluster.DeepCopy() + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to EnableIngress set to True", func(t *testing.T) { + invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = &trueBool + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to EnableIngress set to True") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Container", func(t *testing.T) { + for i, headContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if headContainer.Name == oauthProxyContainerName { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[i].Args = []string{"--invalid-arg"} + break + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Container") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Volume", func(t *testing.T) { + for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { + if headVolume.Name == oauthProxyVolumeName { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" + break + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Volume") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated head group service account name", func(t *testing.T) { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = "invalid-service-account-name" + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated head group service account name") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Init Container in the head group", func(t *testing.T) { + for i, headInitContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers { + if headInitContainer.Name == "create-cert" { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers[i].Command = []string{"manipulated command"} + break + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Init Container in the head group") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Init Container in the worker group", func(t *testing.T) { + for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { + for i, workerInitContainer := range workerGroup.Template.Spec.InitContainers { + if workerInitContainer.Name == "create-cert" { + workerGroup.Template.Spec.InitContainers[i].Command = []string{"manipulated command"} + break + } + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Init Container in the worker group") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Volume in the head group", func(t *testing.T) { + for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { + if headVolume.Name == "ca-vol" { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" + break + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Volume in the head group") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Volume in the worker group", func(t *testing.T) { + for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { + for i, workerVolume := range workerGroup.Template.Spec.Volumes { + if workerVolume.Name == "ca-vol" { + workerGroup.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" + break + } + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Volume in the worker group") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated env vars in the head group", func(t *testing.T) { + for i, headEnvVar := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env { + if headEnvVar.Name == "RAY_USE_TLS" { + invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env[i].Value = "invalid-value" + break + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated env vars in the head group") + }) + + t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated env vars in the worker group", func(t *testing.T) { + for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { + for i, workerEnvVar := range workerGroup.Template.Spec.Containers[0].Env { + if workerEnvVar.Name == "RAY_USE_TLS" { + workerGroup.Template.Spec.Containers[0].Env[i].Value = "invalid-value" + break + } + } + } + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated env vars in the worker group") + }) +} + +func ContainerName(container corev1.Container) string { + return container.Name +} + +func VolumeName(volume corev1.Volume) string { + return volume.Name +} + +func ServiceAccountName(serviceAccount corev1.ServiceAccount) string { + return serviceAccount.Name +} + +func VolumeMountName(volumeMount corev1.VolumeMount) string { + return volumeMount.Name +} + +func EnvVarName(envVar corev1.EnvVar) string { + return envVar.Name +} From 5b4a41f48cbe670b356b6d078efe6f2a8c4d485b Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 21 May 2024 09:21:30 +0100 Subject: [PATCH 234/369] Run controller tests with test-unit make target --- Makefile | 2 +- pkg/controllers/raycluster_webhook_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6f3d20c71..b2673c4f8 100644 --- a/Makefile +++ b/Makefile @@ -381,7 +381,7 @@ catalog-push: ## Push a catalog image. .PHONY: test-unit test-unit: manifests fmt vet envtest ## Run unit tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./pkg/controllers/ -coverprofile cover.out .PHONY: test-component test-component: envtest ginkgo ## Run component tests. diff --git a/pkg/controllers/raycluster_webhook_test.go b/pkg/controllers/raycluster_webhook_test.go index 282fcdd45..3586bc062 100644 --- a/pkg/controllers/raycluster_webhook_test.go +++ b/pkg/controllers/raycluster_webhook_test.go @@ -310,11 +310,10 @@ func TestValidateCreate(t *testing.T) { }) // Negative Test: Invalid RayCluster with EnableIngress set to true - trueBool := true invalidRayCluster := validRayCluster.DeepCopy() t.Run("Negative: Expected errors on call to ValidateCreate function due to EnableIngress set to True", func(t *testing.T) { - invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = &trueBool + invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = support.Ptr(true) _, err := rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateCreate function due to EnableIngress set to True") }) From 4ba15ae31d0be5a10ff02a722cce52db6ed34be6 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 27 May 2024 10:40:00 +0100 Subject: [PATCH 235/369] Use ResourceName generic function for raycluster_webhook tests --- go.mod | 2 +- go.sum | 4 +- pkg/controllers/raycluster_webhook_test.go | 68 ++++++++-------------- 3 files changed, 27 insertions(+), 47 deletions(-) diff --git a/go.mod b/go.mod index 637bcf5e9..2fd0a7aa8 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.12.0 - github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f + github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0 github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 diff --git a/go.sum b/go.sum index 59ec28e4f..8901492a0 100644 --- a/go.sum +++ b/go.sum @@ -244,8 +244,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.12.0 h1:QMxryBPK6ir3VK6Qx4NWOA05/s4xU6uDHI/nXmLznvw= github.com/project-codeflare/appwrapper v0.12.0/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= -github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f h1:9Uron4ej4Tt5ULX5CMzjmPqIZu3q/m07d4jhbNSwdPY= -github.com/project-codeflare/codeflare-common v0.0.0-20240422163521-380101642c8f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= +github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0 h1:3Vz7D9/TwzrBNujHQZGb4L6UKu3siAWwVP4Bj3ByUrU= +github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/pkg/controllers/raycluster_webhook_test.go b/pkg/controllers/raycluster_webhook_test.go index 3586bc062..44927309d 100644 --- a/pkg/controllers/raycluster_webhook_test.go +++ b/pkg/controllers/raycluster_webhook_test.go @@ -103,7 +103,7 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(1), - ContainElement(WithTransform(ContainerName, Equal(oauthProxyContainerName))), + ContainElement(WithTransform(support.ResourceName, Equal(oauthProxyContainerName))), ), "Expected the OAuth proxy container to be present in the head group", ) @@ -114,9 +114,9 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(3), - ContainElement(WithTransform(VolumeName, Equal(oauthProxyVolumeName))), - ContainElement(WithTransform(VolumeName, Equal("ca-vol"))), - ContainElement(WithTransform(VolumeName, Equal("server-cert"))), + ContainElement(WithTransform(support.ResourceName, Equal(oauthProxyVolumeName))), + ContainElement(WithTransform(support.ResourceName, Equal("ca-vol"))), + ContainElement(WithTransform(support.ResourceName, Equal("server-cert"))), ), "Expected the OAuth proxy TLS secret and CA volumes to be present in the head group", ) @@ -133,12 +133,12 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(6), - ContainElement(WithTransform(EnvVarName, Equal("COOKIE_SECRET"))), - ContainElement(WithTransform(EnvVarName, Equal("MY_POD_IP"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_USE_TLS"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_CERT"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_KEY"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_CA_CERT"))), + ContainElement(WithTransform(support.ResourceName, Equal("COOKIE_SECRET"))), + ContainElement(WithTransform(support.ResourceName, Equal("MY_POD_IP"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_USE_TLS"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_TLS_SERVER_CERT"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_TLS_SERVER_KEY"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_TLS_CA_CERT"))), ), "Expected the required environment variables to be present in the head group", ) @@ -149,7 +149,7 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(1), - ContainElement(WithTransform(ContainerName, Equal(initContainerName))), + ContainElement(WithTransform(support.ResourceName, Equal(initContainerName))), ), "Expected the create-cert init container to be present in the head group", ) @@ -160,9 +160,9 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(3), - ContainElement(WithTransform(VolumeMountName, Equal(oauthProxyVolumeName))), - ContainElement(WithTransform(VolumeMountName, Equal("ca-vol"))), - ContainElement(WithTransform(VolumeMountName, Equal("server-cert"))), + ContainElement(WithTransform(support.ResourceName, Equal(oauthProxyVolumeName))), + ContainElement(WithTransform(support.ResourceName, Equal("ca-vol"))), + ContainElement(WithTransform(support.ResourceName, Equal("server-cert"))), ), "Expected the OAuth proxy TLS secret and CA volume mounts to be present in the head group", ) @@ -174,11 +174,11 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(5), - ContainElement(WithTransform(EnvVarName, Equal("MY_POD_IP"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_USE_TLS"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_CERT"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_SERVER_KEY"))), - ContainElement(WithTransform(EnvVarName, Equal("RAY_TLS_CA_CERT"))), + ContainElement(WithTransform(support.ResourceName, Equal("MY_POD_IP"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_USE_TLS"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_TLS_SERVER_CERT"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_TLS_SERVER_KEY"))), + ContainElement(WithTransform(support.ResourceName, Equal("RAY_TLS_CA_CERT"))), ), "Expected the required environment variables to be present in each worker group", ) @@ -191,8 +191,8 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(2), - ContainElement(WithTransform(VolumeName, Equal("ca-vol"))), - ContainElement(WithTransform(VolumeName, Equal("server-cert"))), + ContainElement(WithTransform(support.ResourceName, Equal("ca-vol"))), + ContainElement(WithTransform(support.ResourceName, Equal("server-cert"))), ), "Expected the required CA volumes to be present in each worker group", ) @@ -205,8 +205,8 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(2), - ContainElement(WithTransform(VolumeMountName, Equal("ca-vol"))), - ContainElement(WithTransform(VolumeMountName, Equal("server-cert"))), + ContainElement(WithTransform(support.ResourceName, Equal("ca-vol"))), + ContainElement(WithTransform(support.ResourceName, Equal("server-cert"))), ), "Expected the required certificate volume mounts to be present in each worker group", ) @@ -219,7 +219,7 @@ func TestRayClusterWebhookDefault(t *testing.T) { To( And( HaveLen(1), - ContainElement(WithTransform(ContainerName, Equal(initContainerName))), + ContainElement(WithTransform(support.ResourceName, Equal(initContainerName))), ), "Expected the required init container to be present in each worker group", ) @@ -645,23 +645,3 @@ func TestValidateUpdate(t *testing.T) { test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated env vars in the worker group") }) } - -func ContainerName(container corev1.Container) string { - return container.Name -} - -func VolumeName(volume corev1.Volume) string { - return volume.Name -} - -func ServiceAccountName(serviceAccount corev1.ServiceAccount) string { - return serviceAccount.Name -} - -func VolumeMountName(volumeMount corev1.VolumeMount) string { - return volumeMount.Name -} - -func EnvVarName(envVar corev1.EnvVar) string { - return envVar.Name -} From 9ecefc547cadfa48e52a62c538245924c063b344 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 9 May 2024 11:33:21 +0100 Subject: [PATCH 236/369] Added all-in-one command to Makefile --- Makefile | 117 ++++++++++++++++++ contrib/configuration/default-dsc.yaml | 34 +++++ .../nfd-operator-subscription.yaml | 22 ++++ .../nvidia-operator-subscription.yaml | 22 ++++ .../opendatahub-operator-subscription.yaml | 13 ++ .../rhoai-operator-subscription.yaml | 13 ++ 6 files changed, 221 insertions(+) create mode 100644 contrib/configuration/default-dsc.yaml create mode 100644 contrib/configuration/nfd-operator-subscription.yaml create mode 100644 contrib/configuration/nvidia-operator-subscription.yaml create mode 100644 contrib/configuration/opendatahub-operator-subscription.yaml create mode 100644 contrib/configuration/rhoai-operator-subscription.yaml diff --git a/Makefile b/Makefile index b2673c4f8..690f12493 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ BUNDLE_VERSION ?= $(VERSION:v%=%) # KUEUE_VERSION defines the default version of Kueue (used for testing) KUEUE_VERSION ?= v0.6.2 +USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v1.1.0 @@ -419,3 +420,119 @@ image-mnist-job-test-push: image-mnist-job-test-build ## Push container image wi .PHONY: kueue-setup kueue-setup: bash scripts/setup-kueue-resources.sh +# RHOAI/ODH related resources installation + +##@ all-in-one +.PHONY: all-in-one +all-in-one: + @echo -e "\n ==> Installing Everything needed for distributed AI platform on OpenShift cluster \n" + -make delete-nfd-operator + -make delete-ai-platform-operator + -make delete-nvidia-operator + -make install-ai-platform-operator + -make install-nfd-operator + -make install-nvidia-operator + +.PHONY: delete-all-in-one +delete-all-in-one: + @echo -e "\n ==> Removing Everything needed for distributed AI platform on OpenShift cluster \n" + -make delete-rhoai + -make delete-nfd-operator + -make delete-nvidia-operator + -make delete-ai-platform-operator + +##@ general +.PHONY: delete-ai-platform-operator +delete-ai-platform-operator: +ifeq ($(USE_RHOAI), true) ## Delete RHOAI Operator + -make delete-rhoai-operator +else ## Delete Open Data Hub Operator + -make delete-opendatahub-operator +endif + +.PHONY: install-ai-platform-operator +install-ai-platform-operator: +ifeq ($(USE_RHOAI), true) ## Delete RHOAI Operator + -make install-rhoai-operator +else ## Delete Open Data Hub Operator + -make install-opendatahub-operator +endif + +.PHONY: delete-rhoai-operator +delete-rhoai-operator: ## Delete RHOAI Operator + @echo -e "\n==> Deleting OpenShift AI Operator \n" + -oc delete subscription rhods-operator -n redhat-ods-operator + -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n redhat-ods-operator -l operators.coreos.com/rhods-operator.redhat-ods-operator -o custom-columns=:metadata.name`; \ + oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n redhat-ods-operator + +.PHONY: install-rhoai-operator +install-rhoai-operator: ## Install RHOAI Operator + @echo -e "\n==> Installing OpenShift AI Operator \n" + -oc create ns redhat-ods-operator + oc create -f contrib/configuration/rhoai-operator-subscription.yaml + @echo Waiting for rhoai-operator Subscription to be ready + oc wait -n redhat-ods-operator subscription/rhods-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + @echo -e "\n==> Creating default Data Science Cluster \n" + oc apply -f contrib/configuration/default-dsc.yaml + +.PHONY: delete-opendatahub-operator +delete-opendatahub-operator: ## Delete OpenDataHub operator + @echo -e "\n==> Deleting OpenDataHub Operator \n" + -oc delete subscription opendatahub-operator -n openshift-operators + -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n openshift-operators -l operators.coreos.com/opendatahub-operator.openshift-operators -o custom-columns=:metadata.name`; \ + oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-operators + +.PHONY: install-opendatahub-operator +install-opendatahub-operator: ## Install OpenDataHub operator + @echo -e "\n==> Installing OpenDataHub Operator \n" + -oc create ns opendatahub + oc create -f contrib/configuration/opendatahub-operator-subscription.yaml + @echo Waiting for opendatahub-operator Subscription to be ready + oc wait -n openshift-operators subscription/opendatahub-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + +##@ GPU Support +.PHONY: install-nfd-operator +install-nfd-operator: ## Install NFD operator ( Node Feature Discovery ) + @echo -e "\n==> Installing NFD Operator \n" + -oc create ns openshift-nfd + oc create -f contrib/configuration/nfd-operator-subscription.yaml + @echo -e "\n==> Creating default NodeFeatureDiscovery CR \n" + @while [[ -z $$(oc get customresourcedefinition nodefeaturediscoveries.nfd.openshift.io) ]]; do echo "."; sleep 10; done + @while [[ -z $$(oc get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd) ]]; do echo "."; sleep 10; done + oc get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq '.[] | select(.kind=="NodeFeatureDiscovery")' | oc apply -f - + +.PHONY: delete-nfd-operator +delete-nfd-operator: ## Delete NFD operator + @echo -e "\n==> Deleting NodeFeatureDiscovery CR \n" + oc delete NodeFeatureDiscovery --all -n openshift-nfd + @while [[ -n $$(oc get NodeFeatureDiscovery -n openshift-nfd) ]]; do echo "."; sleep 10; done + @echo -e "\n==> Deleting NFD Operator \n" + -oc delete subscription nfd -n openshift-nfd + -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n openshift-nfd -l operators.coreos.com/nfd.openshift-nfd -o custom-columns=:metadata.name`; \ + oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-nfd + -oc delete ns openshift-nfd + +.PHONY: install-nvidia-operator +install-nvidia-operator: ## Install nvidia operator + @echo -e "\n==> Installing nvidia Operator \n" + -oc create ns nvidia-gpu-operator + oc create -f contrib/configuration/nvidia-operator-subscription.yaml + @echo -e "\n==> Creating default ClusterPolicy CR \n" + @while [[ -z $$(oc get customresourcedefinition clusterpolicies.nvidia.com) ]]; do echo "."; sleep 10; done + @while [[ -z $$(oc get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator) ]]; do echo "."; sleep 10; done + oc get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq .[] | oc apply -f - +#ifeq ($(USE_RHOAI), true) ## Delete RHOAI Operator +# oc delete configmap migration-gpu-status -n redhat-ods-applications +# -export REPLICASET_NAME=`oc get replicaset -n redhat-ods-applications -l app=rhods-dashboard -o custom-columns=:metadata.name` +# oc delete replicaset $$REPLICASET_NAME -n redhat-ods-applications +#endif +.PHONY: delete-nvidia-operator +delete-nvidia-operator: ## Delete nvidia operator + @echo -e "\n==> Deleting ClusterPolicy CR \n" + oc delete ClusterPolicy --all -n nvidia-gpu-operator + @while [[ -n $$(oc get ClusterPolicy -n nvidia-gpu-operator) ]]; do echo "."; sleep 10; done + @echo -e "\n==> Deleting nvidia Operator \n" + -oc delete subscription gpu-operator-certified -n nvidia-gpu-operator + -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n nvidia-gpu-operator -l operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator -o custom-columns=:metadata.name`; \ + oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n nvidia-gpu-operator + -oc delete ns nvidia-gpu-operator diff --git a/contrib/configuration/default-dsc.yaml b/contrib/configuration/default-dsc.yaml new file mode 100644 index 000000000..718d87b07 --- /dev/null +++ b/contrib/configuration/default-dsc.yaml @@ -0,0 +1,34 @@ +kind: DataScienceCluster +apiVersion: datasciencecluster.opendatahub.io/v1 +metadata: + labels: + app.kubernetes.io/created-by: rhods-operator + app.kubernetes.io/instance: default-dsc + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: datasciencecluster + app.kubernetes.io/part-of: rhods-operator + name: default-dsc +spec: + components: + codeflare: + managementState: Managed + dashboard: + managementState: Managed + datasciencepipelines: + managementState: Managed + kserve: + managementState: Managed + serving: + ingressGateway: + certificate: + type: SelfSigned + managementState: Managed + name: knative-serving + kueue: + managementState: Managed + modelmeshserving: + managementState: Managed + ray: + managementState: Managed + workbenches: + managementState: Managed diff --git a/contrib/configuration/nfd-operator-subscription.yaml b/contrib/configuration/nfd-operator-subscription.yaml new file mode 100644 index 000000000..5653ac3a9 --- /dev/null +++ b/contrib/configuration/nfd-operator-subscription.yaml @@ -0,0 +1,22 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: nfd + namespace: openshift-nfd +spec: + targetNamespaces: + - openshift-nfd +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: nfd + labels: + operators.coreos.com/nfd.openshift-nfd: '' + namespace: openshift-nfd +spec: + channel: stable + name: nfd + installPlanApproval: Automatic + source: redhat-operators + sourceNamespace: openshift-marketplace diff --git a/contrib/configuration/nvidia-operator-subscription.yaml b/contrib/configuration/nvidia-operator-subscription.yaml new file mode 100644 index 000000000..3fa10297c --- /dev/null +++ b/contrib/configuration/nvidia-operator-subscription.yaml @@ -0,0 +1,22 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: gpu-operator-certified + namespace: nvidia-gpu-operator +spec: + targetNamespaces: + - opendatahub +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: gpu-operator-certified + labels: + operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator: '' + namespace: nvidia-gpu-operator +spec: + channel: stable + name: gpu-operator-certified + installPlanApproval: Automatic + source: certified-operators + sourceNamespace: openshift-marketplace diff --git a/contrib/configuration/opendatahub-operator-subscription.yaml b/contrib/configuration/opendatahub-operator-subscription.yaml new file mode 100644 index 000000000..6f7521595 --- /dev/null +++ b/contrib/configuration/opendatahub-operator-subscription.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: opendatahub-operator + labels: + operators.coreos.com/opendatahub-operator.openshift-operators: '' + namespace: openshift-operators +spec: + channel: fast + name: opendatahub-operator + installPlanApproval: Automatic + source: community-operators + sourceNamespace: openshift-marketplace diff --git a/contrib/configuration/rhoai-operator-subscription.yaml b/contrib/configuration/rhoai-operator-subscription.yaml new file mode 100644 index 000000000..e5c133c55 --- /dev/null +++ b/contrib/configuration/rhoai-operator-subscription.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: rhods-operator + labels: + operators.coreos.com/rhods-operator.redhat-ods-operator: '' + namespace: redhat-ods-operator +spec: + channel: fast + name: rhods-operator + installPlanApproval: Automatic + source: redhat-operators + sourceNamespace: openshift-marketplace From 700ef92d604fb1e5bdee1aab2948536af5784731 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 9 May 2024 16:08:37 +0100 Subject: [PATCH 237/369] Added additional GPU step for RHOAI --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 690f12493..323ce5865 100644 --- a/Makefile +++ b/Makefile @@ -521,11 +521,12 @@ install-nvidia-operator: ## Install nvidia operator @while [[ -z $$(oc get customresourcedefinition clusterpolicies.nvidia.com) ]]; do echo "."; sleep 10; done @while [[ -z $$(oc get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator) ]]; do echo "."; sleep 10; done oc get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq .[] | oc apply -f - -#ifeq ($(USE_RHOAI), true) ## Delete RHOAI Operator -# oc delete configmap migration-gpu-status -n redhat-ods-applications -# -export REPLICASET_NAME=`oc get replicaset -n redhat-ods-applications -l app=rhods-dashboard -o custom-columns=:metadata.name` -# oc delete replicaset $$REPLICASET_NAME -n redhat-ods-applications -#endif +ifeq ($(USE_RHOAI), true) ## Additional steps required for RHOAI + oc delete configmap migration-gpu-status -n redhat-ods-applications --ignore-not-found=true + -export REPLICASET_NAME=`oc get replicaset -n redhat-ods-applications -l app=rhods-dashboard -o custom-columns=:metadata.name`; \ + oc delete replicaset $$REPLICASET_NAME -n redhat-ods-applications +endif + .PHONY: delete-nvidia-operator delete-nvidia-operator: ## Delete nvidia operator @echo -e "\n==> Deleting ClusterPolicy CR \n" From 8209b174003133a00c8428a7cbf6e98d02b49641 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 13 May 2024 14:19:17 +0100 Subject: [PATCH 238/369] Converted oc to kubectl & updated resource structure --- Makefile | 85 ++++++++++--------- contrib/configuration/odh/default-dsc.yaml | 40 +++++++++ contrib/configuration/odh/default-dsci.yaml | 24 ++++++ .../opendatahub-operator-subscription.yaml | 0 .../{ => rhoai}/default-dsc.yaml | 0 contrib/configuration/rhoai/default-dsci.yaml | 24 ++++++ .../rhoai-operator-subscription.yaml | 0 7 files changed, 134 insertions(+), 39 deletions(-) create mode 100644 contrib/configuration/odh/default-dsc.yaml create mode 100644 contrib/configuration/odh/default-dsci.yaml rename contrib/configuration/{ => odh}/opendatahub-operator-subscription.yaml (100%) rename contrib/configuration/{ => rhoai}/default-dsc.yaml (100%) create mode 100644 contrib/configuration/rhoai/default-dsci.yaml rename contrib/configuration/{ => rhoai}/rhoai-operator-subscription.yaml (100%) diff --git a/Makefile b/Makefile index 323ce5865..5a0e64c64 100644 --- a/Makefile +++ b/Makefile @@ -429,8 +429,8 @@ all-in-one: -make delete-nfd-operator -make delete-ai-platform-operator -make delete-nvidia-operator - -make install-ai-platform-operator -make install-nfd-operator + -make install-ai-platform-operator -make install-nvidia-operator .PHONY: delete-all-in-one @@ -461,79 +461,86 @@ endif .PHONY: delete-rhoai-operator delete-rhoai-operator: ## Delete RHOAI Operator @echo -e "\n==> Deleting OpenShift AI Operator \n" - -oc delete subscription rhods-operator -n redhat-ods-operator - -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n redhat-ods-operator -l operators.coreos.com/rhods-operator.redhat-ods-operator -o custom-columns=:metadata.name`; \ - oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n redhat-ods-operator + -kubectl delete subscription rhods-operator -n redhat-ods-operator + -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n redhat-ods-operator -l operators.coreos.com/rhods-operator.redhat-ods-operator -o custom-columns=:metadata.name`; \ + kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n redhat-ods-operator + kubectl delete namespace redhat-ods-operator .PHONY: install-rhoai-operator install-rhoai-operator: ## Install RHOAI Operator @echo -e "\n==> Installing OpenShift AI Operator \n" - -oc create ns redhat-ods-operator - oc create -f contrib/configuration/rhoai-operator-subscription.yaml + -kubectl create ns redhat-ods-operator + kubectl create -f contrib/configuration/rhoai/rhoai-operator-subscription.yaml @echo Waiting for rhoai-operator Subscription to be ready - oc wait -n redhat-ods-operator subscription/rhods-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + kubectl wait -n redhat-ods-operator subscription/rhods-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s @echo -e "\n==> Creating default Data Science Cluster \n" - oc apply -f contrib/configuration/default-dsc.yaml + kubectl apply -f contrib/configuration/rhoai/default-dsci.yaml --server-side + kubectl apply -f contrib/configuration/rhoai/default-dsc.yaml --server-side .PHONY: delete-opendatahub-operator delete-opendatahub-operator: ## Delete OpenDataHub operator @echo -e "\n==> Deleting OpenDataHub Operator \n" - -oc delete subscription opendatahub-operator -n openshift-operators - -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n openshift-operators -l operators.coreos.com/opendatahub-operator.openshift-operators -o custom-columns=:metadata.name`; \ - oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-operators + -kubectl delete subscription opendatahub-operator -n openshift-operators + -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n openshift-operators -l operators.coreos.com/opendatahub-operator.openshift-operators -o custom-columns=:metadata.name`; \ + kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-operators + -kubectl delete namespace opendatahub .PHONY: install-opendatahub-operator install-opendatahub-operator: ## Install OpenDataHub operator @echo -e "\n==> Installing OpenDataHub Operator \n" - -oc create ns opendatahub - oc create -f contrib/configuration/opendatahub-operator-subscription.yaml + -kubectl create ns opendatahub + kubectl create -f contrib/configuration/odh/opendatahub-operator-subscription.yaml @echo Waiting for opendatahub-operator Subscription to be ready - oc wait -n openshift-operators subscription/opendatahub-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + kubectl wait -n openshift-operators subscription/opendatahub-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + sleep 2 + kubectl wait --for=condition=available deployment/opendatahub-operator-controller-manager -n openshift-operators --timeout=180s + kubectl apply -f contrib/configuration/odh/default-dsci.yaml --server-side + kubectl apply -f contrib/configuration/odh/default-dsc.yaml --server-side ##@ GPU Support .PHONY: install-nfd-operator install-nfd-operator: ## Install NFD operator ( Node Feature Discovery ) @echo -e "\n==> Installing NFD Operator \n" - -oc create ns openshift-nfd - oc create -f contrib/configuration/nfd-operator-subscription.yaml + -kubectl create ns openshift-nfd + kubectl create -f contrib/configuration/nfd-operator-subscription.yaml @echo -e "\n==> Creating default NodeFeatureDiscovery CR \n" - @while [[ -z $$(oc get customresourcedefinition nodefeaturediscoveries.nfd.openshift.io) ]]; do echo "."; sleep 10; done - @while [[ -z $$(oc get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd) ]]; do echo "."; sleep 10; done - oc get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq '.[] | select(.kind=="NodeFeatureDiscovery")' | oc apply -f - + @while [[ -z $$(kubectl get customresourcedefinition nodefeaturediscoveries.nfd.openshift.io) ]]; do echo "."; sleep 10; done + @while [[ -z $$(kubectl get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd) ]]; do echo "."; sleep 10; done + kubectl get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq '.[] | select(.kind=="NodeFeatureDiscovery")' | kubectl apply -f - .PHONY: delete-nfd-operator delete-nfd-operator: ## Delete NFD operator @echo -e "\n==> Deleting NodeFeatureDiscovery CR \n" - oc delete NodeFeatureDiscovery --all -n openshift-nfd - @while [[ -n $$(oc get NodeFeatureDiscovery -n openshift-nfd) ]]; do echo "."; sleep 10; done + kubectl delete NodeFeatureDiscovery --all -n openshift-nfd + @while [[ -n $$(kubectl get NodeFeatureDiscovery -n openshift-nfd) ]]; do echo "."; sleep 10; done @echo -e "\n==> Deleting NFD Operator \n" - -oc delete subscription nfd -n openshift-nfd - -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n openshift-nfd -l operators.coreos.com/nfd.openshift-nfd -o custom-columns=:metadata.name`; \ - oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-nfd - -oc delete ns openshift-nfd + -kubectl delete subscription nfd -n openshift-nfd + -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n openshift-nfd -l operators.coreos.com/nfd.openshift-nfd -o custom-columns=:metadata.name`; \ + kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-nfd + -kubectl delete ns openshift-nfd .PHONY: install-nvidia-operator install-nvidia-operator: ## Install nvidia operator @echo -e "\n==> Installing nvidia Operator \n" - -oc create ns nvidia-gpu-operator - oc create -f contrib/configuration/nvidia-operator-subscription.yaml + -kubectl create ns nvidia-gpu-operator + kubectl create -f contrib/configuration/nvidia-operator-subscription.yaml @echo -e "\n==> Creating default ClusterPolicy CR \n" - @while [[ -z $$(oc get customresourcedefinition clusterpolicies.nvidia.com) ]]; do echo "."; sleep 10; done - @while [[ -z $$(oc get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator) ]]; do echo "."; sleep 10; done - oc get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq .[] | oc apply -f - + @while [[ -z $$(kubectl get customresourcedefinition clusterpolicies.nvidia.com) ]]; do echo "."; sleep 10; done + @while [[ -z $$(kubectl get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator) ]]; do echo "."; sleep 10; done + kubectl get csv -n nvidia-gpu-operator --selector operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq .[] | kubectl apply -f - ifeq ($(USE_RHOAI), true) ## Additional steps required for RHOAI - oc delete configmap migration-gpu-status -n redhat-ods-applications --ignore-not-found=true - -export REPLICASET_NAME=`oc get replicaset -n redhat-ods-applications -l app=rhods-dashboard -o custom-columns=:metadata.name`; \ - oc delete replicaset $$REPLICASET_NAME -n redhat-ods-applications + kubectl delete configmap migration-gpu-status -n redhat-ods-applications --ignore-not-found=true + -export REPLICASET_NAME=`kubectl get replicaset -n redhat-ods-applications -l app=rhods-dashboard -o custom-columns=:metadata.name`; \ + kubectl delete replicaset $$REPLICASET_NAME -n redhat-ods-applications endif .PHONY: delete-nvidia-operator delete-nvidia-operator: ## Delete nvidia operator @echo -e "\n==> Deleting ClusterPolicy CR \n" - oc delete ClusterPolicy --all -n nvidia-gpu-operator - @while [[ -n $$(oc get ClusterPolicy -n nvidia-gpu-operator) ]]; do echo "."; sleep 10; done + kubectl delete ClusterPolicy --all -n nvidia-gpu-operator + @while [[ -n $$(kubectl get ClusterPolicy -n nvidia-gpu-operator) ]]; do echo "."; sleep 10; done @echo -e "\n==> Deleting nvidia Operator \n" - -oc delete subscription gpu-operator-certified -n nvidia-gpu-operator - -export CLUSTER_SERVICE_VERSION=`oc get clusterserviceversion -n nvidia-gpu-operator -l operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator -o custom-columns=:metadata.name`; \ - oc delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n nvidia-gpu-operator - -oc delete ns nvidia-gpu-operator + -kubectl delete subscription gpu-operator-certified -n nvidia-gpu-operator + -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n nvidia-gpu-operator -l operators.coreos.com/gpu-operator-certified.nvidia-gpu-operator -o custom-columns=:metadata.name`; \ + kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n nvidia-gpu-operator + -kubectl delete ns nvidia-gpu-operator diff --git a/contrib/configuration/odh/default-dsc.yaml b/contrib/configuration/odh/default-dsc.yaml new file mode 100644 index 000000000..d5e4d902a --- /dev/null +++ b/contrib/configuration/odh/default-dsc.yaml @@ -0,0 +1,40 @@ +kind: DataScienceCluster +apiVersion: datasciencecluster.opendatahub.io/v1 +metadata: + labels: + app.kubernetes.io/created-by: opendatahub-operator + app.kubernetes.io/instance: default + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: datasciencecluster + app.kubernetes.io/part-of: opendatahub-operator + name: default-dsc +spec: + components: + codeflare: + managementState: Managed + dashboard: + managementState: Managed + datasciencepipelines: + managementState: Managed + kserve: + managementState: Managed + serving: + ingressGateway: + certificate: + type: SelfSigned + managementState: Managed + name: knative-serving + kueue: + managementState: Managed + modelmeshserving: + managementState: Managed + modelregistry: + managementState: Removed + ray: + managementState: Managed + trainingoperator: + managementState: Removed + trustyai: + managementState: Managed + workbenches: + managementState: Managed diff --git a/contrib/configuration/odh/default-dsci.yaml b/contrib/configuration/odh/default-dsci.yaml new file mode 100644 index 000000000..e9f1b2995 --- /dev/null +++ b/contrib/configuration/odh/default-dsci.yaml @@ -0,0 +1,24 @@ +kind: DSCInitialization +apiVersion: dscinitialization.opendatahub.io/v1 +metadata: + labels: + app.kubernetes.io/created-by: opendatahub-operator + app.kubernetes.io/instance: default + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: dscinitialization + app.kubernetes.io/part-of: opendatahub-operator + name: default-dsci +spec: + applicationsNamespace: opendatahub + monitoring: + managementState: Managed + namespace: opendatahub + serviceMesh: + controlPlane: + metricsCollection: Istio + name: data-science-smcp + namespace: istio-system + managementState: Managed + trustedCABundle: + customCABundle: '' + managementState: Managed diff --git a/contrib/configuration/opendatahub-operator-subscription.yaml b/contrib/configuration/odh/opendatahub-operator-subscription.yaml similarity index 100% rename from contrib/configuration/opendatahub-operator-subscription.yaml rename to contrib/configuration/odh/opendatahub-operator-subscription.yaml diff --git a/contrib/configuration/default-dsc.yaml b/contrib/configuration/rhoai/default-dsc.yaml similarity index 100% rename from contrib/configuration/default-dsc.yaml rename to contrib/configuration/rhoai/default-dsc.yaml diff --git a/contrib/configuration/rhoai/default-dsci.yaml b/contrib/configuration/rhoai/default-dsci.yaml new file mode 100644 index 000000000..91d3d1b00 --- /dev/null +++ b/contrib/configuration/rhoai/default-dsci.yaml @@ -0,0 +1,24 @@ +kind: DSCInitialization +apiVersion: dscinitialization.opendatahub.io/v1 +metadata: + labels: + app.kubernetes.io/created-by: rhods-operator + app.kubernetes.io/instance: default-dsci + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: dscinitialization + app.kubernetes.io/part-of: rhods-operator + name: default-dsci +spec: + applicationsNamespace: redhat-ods-applications + monitoring: + managementState: Managed + namespace: redhat-ods-monitoring + serviceMesh: + controlPlane: + metricsCollection: Istio + name: data-science-smcp + namespace: istio-system + managementState: Managed + trustedCABundle: + customCABundle: '' + managementState: Managed diff --git a/contrib/configuration/rhoai-operator-subscription.yaml b/contrib/configuration/rhoai/rhoai-operator-subscription.yaml similarity index 100% rename from contrib/configuration/rhoai-operator-subscription.yaml rename to contrib/configuration/rhoai/rhoai-operator-subscription.yaml From 76f70bacb3f72e50160516d9d3752fec45cab947 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 13 May 2024 14:46:43 +0100 Subject: [PATCH 239/369] Added Operator group for RHOAI --- .../configuration/rhoai/rhoai-operator-subscription.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/configuration/rhoai/rhoai-operator-subscription.yaml b/contrib/configuration/rhoai/rhoai-operator-subscription.yaml index e5c133c55..15373ca51 100644 --- a/contrib/configuration/rhoai/rhoai-operator-subscription.yaml +++ b/contrib/configuration/rhoai/rhoai-operator-subscription.yaml @@ -1,3 +1,9 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: rhods-operator + namespace: redhat-ods-operator +--- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: From c93f86cd2c1af4991c1f167501f81050b3910559 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 13 May 2024 15:46:43 +0100 Subject: [PATCH 240/369] Review comments: unnecessary function call Review changes: removal of uneccesary deletion --- Makefile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 5a0e64c64..79f993ce0 100644 --- a/Makefile +++ b/Makefile @@ -422,13 +422,16 @@ kueue-setup: bash scripts/setup-kueue-resources.sh # RHOAI/ODH related resources installation +# Basic Usage +# all-in-one will create all resources necessary to create GPU enabled ML workloads via OpenShift AI +# Users have the choice between installing RHOAI and ODH +# For RHOAI use `make all-in-one` and to remove all of the operators run `make delete-all-in-one` +# For ODH use `make all-in-one -e USE_RHOAI=false` and to remove all of the operators run `make delete-all-in-one -e USE_RHOAI=false` + ##@ all-in-one .PHONY: all-in-one all-in-one: @echo -e "\n ==> Installing Everything needed for distributed AI platform on OpenShift cluster \n" - -make delete-nfd-operator - -make delete-ai-platform-operator - -make delete-nvidia-operator -make install-nfd-operator -make install-ai-platform-operator -make install-nvidia-operator @@ -436,10 +439,9 @@ all-in-one: .PHONY: delete-all-in-one delete-all-in-one: @echo -e "\n ==> Removing Everything needed for distributed AI platform on OpenShift cluster \n" - -make delete-rhoai -make delete-nfd-operator - -make delete-nvidia-operator -make delete-ai-platform-operator + -make delete-nvidia-operator ##@ general .PHONY: delete-ai-platform-operator @@ -460,7 +462,7 @@ endif .PHONY: delete-rhoai-operator delete-rhoai-operator: ## Delete RHOAI Operator - @echo -e "\n==> Deleting OpenShift AI Operator \n" + @echo -e "\n ==> Deleting OpenShift AI Operator \n" -kubectl delete subscription rhods-operator -n redhat-ods-operator -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n redhat-ods-operator -l operators.coreos.com/rhods-operator.redhat-ods-operator -o custom-columns=:metadata.name`; \ kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n redhat-ods-operator @@ -468,7 +470,7 @@ delete-rhoai-operator: ## Delete RHOAI Operator .PHONY: install-rhoai-operator install-rhoai-operator: ## Install RHOAI Operator - @echo -e "\n==> Installing OpenShift AI Operator \n" + @echo -e "\n ==> Installing OpenShift AI Operator \n" -kubectl create ns redhat-ods-operator kubectl create -f contrib/configuration/rhoai/rhoai-operator-subscription.yaml @echo Waiting for rhoai-operator Subscription to be ready From e50e679ba0af1a291c43a6c85a1d3964af5becf0 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 14 May 2024 14:17:53 +0100 Subject: [PATCH 241/369] Fixed NFD --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 79f993ce0..5bfe77903 100644 --- a/Makefile +++ b/Makefile @@ -508,7 +508,7 @@ install-nfd-operator: ## Install NFD operator ( Node Feature Discovery ) @echo -e "\n==> Creating default NodeFeatureDiscovery CR \n" @while [[ -z $$(kubectl get customresourcedefinition nodefeaturediscoveries.nfd.openshift.io) ]]; do echo "."; sleep 10; done @while [[ -z $$(kubectl get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd) ]]; do echo "."; sleep 10; done - kubectl get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq '.[] | select(.kind=="NodeFeatureDiscovery")' | kubectl apply -f - + kubectl get csv -n openshift-nfd --selector operators.coreos.com/nfd.openshift-nfd -ojsonpath={.items[0].metadata.annotations.alm-examples} | jq '.[] | select(.kind=="NodeFeatureDiscovery")' | kubectl apply -f - --validate=false .PHONY: delete-nfd-operator delete-nfd-operator: ## Delete NFD operator From cc062e00ad915f332a31dc67d7a2ff37e4c20d90 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 27 May 2024 16:52:52 +0100 Subject: [PATCH 242/369] Added Service Mesh Operator && DSC deletion --- Makefile | 29 ++++++++++++++++++- .../service-mesh-operator-subscription.yaml | 13 +++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 contrib/configuration/service-mesh-operator-subscription.yaml diff --git a/Makefile b/Makefile index 5bfe77903..5de676ba5 100644 --- a/Makefile +++ b/Makefile @@ -433,6 +433,7 @@ kueue-setup: all-in-one: @echo -e "\n ==> Installing Everything needed for distributed AI platform on OpenShift cluster \n" -make install-nfd-operator + -make install-service-mesh-operator -make install-ai-platform-operator -make install-nvidia-operator @@ -441,6 +442,7 @@ delete-all-in-one: @echo -e "\n ==> Removing Everything needed for distributed AI platform on OpenShift cluster \n" -make delete-nfd-operator -make delete-ai-platform-operator + -make delete-service-mesh-operator -make delete-nvidia-operator ##@ general @@ -463,6 +465,10 @@ endif .PHONY: delete-rhoai-operator delete-rhoai-operator: ## Delete RHOAI Operator @echo -e "\n ==> Deleting OpenShift AI Operator \n" + kubectl delete datasciencecluster/default-dsc + kubectl wait --for=delete datasciencecluster/default-dsc --timeout=180s + kubectl delete dsci/default-dsci + kubectl wait --for=delete dsci/default-dsci --timeout=180s -kubectl delete subscription rhods-operator -n redhat-ods-operator -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n redhat-ods-operator -l operators.coreos.com/rhods-operator.redhat-ods-operator -o custom-columns=:metadata.name`; \ kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n redhat-ods-operator @@ -475,6 +481,8 @@ install-rhoai-operator: ## Install RHOAI Operator kubectl create -f contrib/configuration/rhoai/rhoai-operator-subscription.yaml @echo Waiting for rhoai-operator Subscription to be ready kubectl wait -n redhat-ods-operator subscription/rhods-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + -export RHOAI_POD_NAME=`kubectl get -n redhat-ods-operator pod -o custom-columns=:metadata.name | grep rhods-operator`; \ + kubectl wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' pod/$$RHOAI_POD_NAME -n redhat-ods-operator @echo -e "\n==> Creating default Data Science Cluster \n" kubectl apply -f contrib/configuration/rhoai/default-dsci.yaml --server-side kubectl apply -f contrib/configuration/rhoai/default-dsc.yaml --server-side @@ -482,6 +490,10 @@ install-rhoai-operator: ## Install RHOAI Operator .PHONY: delete-opendatahub-operator delete-opendatahub-operator: ## Delete OpenDataHub operator @echo -e "\n==> Deleting OpenDataHub Operator \n" + kubectl delete datasciencecluster/default-dsc + kubectl wait --for=delete datasciencecluster/default-dsc --timeout=180s + kubectl delete dsci/default-dsci + kubectl wait --for=delete dsci/default-dsci --timeout=180s -kubectl delete subscription opendatahub-operator -n openshift-operators -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n openshift-operators -l operators.coreos.com/opendatahub-operator.openshift-operators -o custom-columns=:metadata.name`; \ kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-operators @@ -494,11 +506,26 @@ install-opendatahub-operator: ## Install OpenDataHub operator kubectl create -f contrib/configuration/odh/opendatahub-operator-subscription.yaml @echo Waiting for opendatahub-operator Subscription to be ready kubectl wait -n openshift-operators subscription/opendatahub-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s - sleep 2 kubectl wait --for=condition=available deployment/opendatahub-operator-controller-manager -n openshift-operators --timeout=180s + -export ODH_POD_NAME=`kubectl get -n openshift-operators pod -o custom-columns=:metadata.name | grep opendatahub-operator-controller-manager`; \ + kubectl wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' pod/$$ODH_POD_NAME -n openshift-operators kubectl apply -f contrib/configuration/odh/default-dsci.yaml --server-side kubectl apply -f contrib/configuration/odh/default-dsc.yaml --server-side +.PHONY: delete-service-mesh-operator +delete-service-mesh-operator: ## Delete Service Mesh Operator + @echo -e "\n==> Deleting Service Mesh Operator \n" + kubectl delete subscription servicemeshoperator -n openshift-operators + -export CLUSTER_SERVICE_VERSION=`kubectl get clusterserviceversion -n openshift-operators -l operators.coreos.com/servicemeshoperator.openshift-operators -o custom-columns=:metadata.name`; \ + kubectl delete clusterserviceversion $$CLUSTER_SERVICE_VERSION -n openshift-operators + +.PHONY: install-service-mesh-operator +install-service-mesh-operator: ## Install Service Mesh Operator + @echo -e "\n==> Installing OpenShift Service Mesh Operator" + kubectl create -f contrib/configuration/service-mesh-operator-subscription.yaml + kubectl wait -n openshift-operators subscription/servicemeshoperator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + kubectl wait --for=condition=available deployment/istio-operator -n openshift-operators --timeout=180s + ##@ GPU Support .PHONY: install-nfd-operator install-nfd-operator: ## Install NFD operator ( Node Feature Discovery ) diff --git a/contrib/configuration/service-mesh-operator-subscription.yaml b/contrib/configuration/service-mesh-operator-subscription.yaml new file mode 100644 index 000000000..9e21a4c77 --- /dev/null +++ b/contrib/configuration/service-mesh-operator-subscription.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: servicemeshoperator + labels: + operators.coreos.com/servicemeshoperator.openshift-operators: '' + namespace: openshift-operators +spec: + channel: stable + name: servicemeshoperator + installPlanApproval: Automatic + source: redhat-operators + sourceNamespace: openshift-marketplace From 1705b8aa90113a2ec81f7350fa3c60df951792e5 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 28 May 2024 14:14:24 +0100 Subject: [PATCH 243/369] Improved wait conditions for Operator deployments --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5de676ba5..bc47fa7d0 100644 --- a/Makefile +++ b/Makefile @@ -481,8 +481,9 @@ install-rhoai-operator: ## Install RHOAI Operator kubectl create -f contrib/configuration/rhoai/rhoai-operator-subscription.yaml @echo Waiting for rhoai-operator Subscription to be ready kubectl wait -n redhat-ods-operator subscription/rhods-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + @while [[ -z $$(kubectl get deployment/rhods-operator -n redhat-ods-operator) ]]; do echo "."; sleep 10; done -export RHOAI_POD_NAME=`kubectl get -n redhat-ods-operator pod -o custom-columns=:metadata.name | grep rhods-operator`; \ - kubectl wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' pod/$$RHOAI_POD_NAME -n redhat-ods-operator + kubectl wait --for=condition=Ready pod/$$RHOAI_POD_NAME -n redhat-ods-operator @echo -e "\n==> Creating default Data Science Cluster \n" kubectl apply -f contrib/configuration/rhoai/default-dsci.yaml --server-side kubectl apply -f contrib/configuration/rhoai/default-dsc.yaml --server-side @@ -506,9 +507,10 @@ install-opendatahub-operator: ## Install OpenDataHub operator kubectl create -f contrib/configuration/odh/opendatahub-operator-subscription.yaml @echo Waiting for opendatahub-operator Subscription to be ready kubectl wait -n openshift-operators subscription/opendatahub-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + @while [[ -z $$(kubectl get deployment/opendatahub-operator-controller-manager -n openshift-operators) ]]; do echo "."; sleep 10; done kubectl wait --for=condition=available deployment/opendatahub-operator-controller-manager -n openshift-operators --timeout=180s -export ODH_POD_NAME=`kubectl get -n openshift-operators pod -o custom-columns=:metadata.name | grep opendatahub-operator-controller-manager`; \ - kubectl wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' pod/$$ODH_POD_NAME -n openshift-operators + kubectl wait --for=condition=Ready pod/$$ODH_POD_NAME -n openshift-operators kubectl apply -f contrib/configuration/odh/default-dsci.yaml --server-side kubectl apply -f contrib/configuration/odh/default-dsc.yaml --server-side @@ -524,6 +526,7 @@ install-service-mesh-operator: ## Install Service Mesh Operator @echo -e "\n==> Installing OpenShift Service Mesh Operator" kubectl create -f contrib/configuration/service-mesh-operator-subscription.yaml kubectl wait -n openshift-operators subscription/servicemeshoperator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + @while [[ -z $$(kubectl get deployment/istio-operator -n openshift-operators) ]]; do echo "."; sleep 10; done kubectl wait --for=condition=available deployment/istio-operator -n openshift-operators --timeout=180s ##@ GPU Support From 5f1bec5024e19e1e72d5dbc8fbc1f5bd5dd4cf08 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 29 May 2024 12:44:11 +0100 Subject: [PATCH 244/369] Added custom accelerator and removal of nvidia driver --- Makefile | 5 +++++ contrib/configuration/accelerator-profile.yaml | 9 +++++++++ 2 files changed, 14 insertions(+) create mode 100644 contrib/configuration/accelerator-profile.yaml diff --git a/Makefile b/Makefile index bc47fa7d0..203b02d17 100644 --- a/Makefile +++ b/Makefile @@ -450,16 +450,20 @@ delete-all-in-one: delete-ai-platform-operator: ifeq ($(USE_RHOAI), true) ## Delete RHOAI Operator -make delete-rhoai-operator + -kubectl delete -f contrib/configuration/accelerator-profile.yaml -n redhat-ods-applications else ## Delete Open Data Hub Operator -make delete-opendatahub-operator + -kubectl delete -f contrib/configuration/accelerator-profile.yaml -n opendatahub endif .PHONY: install-ai-platform-operator install-ai-platform-operator: ifeq ($(USE_RHOAI), true) ## Delete RHOAI Operator -make install-rhoai-operator + -kubectl apply -f contrib/configuration/accelerator-profile.yaml -n redhat-ods-applications else ## Delete Open Data Hub Operator -make install-opendatahub-operator + -kubectl apply -f contrib/configuration/accelerator-profile.yaml -n opendatahub endif .PHONY: delete-rhoai-operator @@ -569,6 +573,7 @@ endif .PHONY: delete-nvidia-operator delete-nvidia-operator: ## Delete nvidia operator @echo -e "\n==> Deleting ClusterPolicy CR \n" + kubectl delete --ignore-not-found=true NVIDIADriver gpu-driver kubectl delete ClusterPolicy --all -n nvidia-gpu-operator @while [[ -n $$(kubectl get ClusterPolicy -n nvidia-gpu-operator) ]]; do echo "."; sleep 10; done @echo -e "\n==> Deleting nvidia Operator \n" diff --git a/contrib/configuration/accelerator-profile.yaml b/contrib/configuration/accelerator-profile.yaml new file mode 100644 index 000000000..37f64ea04 --- /dev/null +++ b/contrib/configuration/accelerator-profile.yaml @@ -0,0 +1,9 @@ +apiVersion: dashboard.opendatahub.io/v1 +kind: AcceleratorProfile +metadata: + name: gpu-accelerator-profile +spec: { + displayName: nvidia-gpu, + enabled: true, + identifier: nvidia.com/gpu +} From 4eff35de60a3821dfccf5de3d7998469e19988eb Mon Sep 17 00:00:00 2001 From: Ignas Baranauskas Date: Thu, 9 May 2024 11:14:40 +0100 Subject: [PATCH 245/369] Add new quickstart doc --- docs/Quick-Start.md | 220 ++++++++++++++++++++++++++++++++ docs/images/rhoai-dashboard.png | Bin 0 -> 21949 bytes 2 files changed, 220 insertions(+) create mode 100644 docs/Quick-Start.md create mode 100644 docs/images/rhoai-dashboard.png diff --git a/docs/Quick-Start.md b/docs/Quick-Start.md new file mode 100644 index 000000000..d3ece1244 --- /dev/null +++ b/docs/Quick-Start.md @@ -0,0 +1,220 @@ +# Quick Start Guide for Distributed Workloads with the CodeFlare Stack + + +- [Quick Start Guide for Distributed Workloads with the CodeFlare Stack](#quick-start-guide-for-distributed-workloads-with-the-codeflare-stack) + - [Prerequisites](#prerequisites) + - [Resources](#resources) + - [GPU Worker Node](#gpu-worker-node) + - [Automatic deployment for developers (optional - skip to run steps manually)](#automatic-deployment-for-developers-optional-skip-to-run-steps-manually) + - [Next Steps After Automatic Deployment](#next-steps-after-automatic-deployment) + - [Red Hat OpenShift AI](#red-hat-openshift-ai) + - [Configure Data Science Components](#configure-data-science-components) + - [Preparing Your Cluster for GPU Workloads](#preparing-your-cluster-for-gpu-workloads) + - [Installing the Operators](#installing-the-operators) + - [Configure the Node Feature Discovery Operator](#configure-the-node-feature-discovery-operator) + - [Configure the NVIDIA GPU Operator](#configure-the-nvidia-gpu-operator) + - [Configure Kueue for Task Scheduling](#configure-kueue-for-task-scheduling) + - [Launch a Jupyter Notebook Instance](#launch-a-jupyter-notebook-instance) + - [Submit your first job](#submit-your-first-job) + - [Clone the demo code](#clone-the-demo-code) + - [Run the Guided Demo Notebooks](#run-the-guided-demo-notebooks) + - [Cleaning up the AI Platform Install](#cleaning-up-the-ai-platform-install) + - [Manual Cleanup Steps](#manual-cleanup-steps) + - [Next Steps](#next-steps) + + +This quick start guide is intended to walk users through installation of the CodeFlare stack and an initial demo using the CodeFlare-SDK from within a Jupyter notebook environment. This will enable users to run and submit distributed workloads. + +The CodeFlare-SDK was built to make managing distributed compute infrastructure in the cloud easy and intuitive for Data Scientists. However, that means there +needs to be some cloud infrastructure on the backend for users to get the benefit of using the SDK. Currently, we support the CodeFlare stack. + +This stack integrates well with Red Hat OpenShift AI and [Open Data Hub](https://opendatahub.io/), and helps to bring batch workloads, jobs, and queuing to the Data Science platform. Although this document will guide you through setting up with Red Hat OpenShift AI (RHOAI), the steps are also applicable if you are using Open Data Hub (ODH). Both platforms are available in the OperatorHub, and the installation and configuration steps are quite similar. This guide will proceed with RHOAI, but feel free to apply the instructions to ODH as needed. + +## Prerequisites + +### Resources + +In addition to the resources required by default Red Hat OpenShift AI deployments, you will need the following to deploy the Distributed Workloads stack infrastructure pods: + +```text +Total: + CPU: 1600m (1.6 vCPU) + Memory: 2048Mi (2 GiB) +``` + +> [!NOTE] +> The above resources are just for the infrastructure pods. To be able to run actual +> workloads on your cluster you will need additional resources based on the size +> and type of workload. + +### GPU Worker Node + +> [!IMPORTANT] +> This step is necessary only if you require GPU capabilities for your workloads and your OpenShift cluster does not already include GPU-equipped nodes, follow these steps: + +1. **Open the OpenShift Cluster Console.** +2. Navigate to **your-cluster** -> **Machine pools**. +3. Click on **“Add machine pool”**. +4. Provide a **name** for the new machine pool. +5. In the **“Compute node instance type”** dropdown, scroll all the way down and search for the GPU instance type `g4dn.xlarge` or similar. +6. Click on **Add machine pool** to finalize the creation of your new GPU-enabled machine pool. + +After adding the machine pool, OpenShift will begin provisioning the new GPU worker +node. This process can take a few minutes. Once completed, the new node will be ready to handle GPU-accelerated workloads. + +> [!NOTE] +> The `g4dn.xlarge` instance type is used for GPU worker nodes. Ensure this instance type meets your application needs or select another as required. + +## Automatic deployment for developers (optional - skip to run steps manually) + +As a quick alternative to the following manual deployment steps an automatic _Makefile_ script can be used to deploy the CodeFlare stack. This script also deploys the prerequisite operators and the entire CodeFlare stack. + +1. Clone the Repository + +```bash +git clone https://github.com/project-codeflare/codeflare-operator.git +cd codeflare-operator +``` + +2. Run the Makefile script + +```bash +make all-in-one +``` + +> [!TIP] +> Execute `make help` to list additional available operations. + +### Next Steps After Automatic Deployment + +After the automatic deployment is complete, you can proceed directly to the section [Configure Kueue for Task Scheduling](#configure-kueue-for-task-scheduling) to finish setting up your environment. + +## Red Hat OpenShift AI + +This Quick Start guide assumes that you have administrator access to an OpenShift cluster and an existing Red Hat OpenShift AI (RHOAI) installation with version **>2.9** is present on your cluster. But the quick step to install RHOAI is as follows: + +1. Using the OpenShift web console, navigate to **Operators** -> **OperatorHub**. +2. Search for `Red Hat OpenShift AI`. +3. Install it using the `fast` channel. + +### Configure Data Science Components + +After the installation of the Red Hat OpenShift AI Operator, proceed to configure the necessary components for data science work: + +1. From the OpenShift web console, navigate to the installed RHOAI Operator. +2. Look for tab labeled DSC Initialization. +3. If it has not already been created - Locate `Create DSCInitialization` and create one. +4. Look for tab labeled Data Science Cluster. +5. Locate `Create DataScienceCluster` and create one. + +## Preparing Your Cluster for GPU Workloads + +To leverage GPU-enabled workloads on your OpenShift cluster, you need to install both the Node Feature Discovery (NFD) Operator and the NVIDIA GPU Operator. + +### Installing the Operators + +Both the NFD and the NVIDIA GPU Operators can be installed from the OperatorHub. +Detailed steps for installation and configuration are provided in the NVIDIA +documentation, which can be accessed [here](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/openshift/steps-overview.html#high-level-steps). + +1. **Open the OpenShift dashboard.** +2. Navigate to **OperatorHub**. +3. Search for and install the following operators (default settings are fine): + - **Node Feature Discovery Operator** + - **NVIDIA GPU Operator** + +### Configure the Node Feature Discovery Operator + +After installing the Node Feature Discovery Operator, you need to create a Node Feature Discovery Custom Resource (CR). You can use the default settings for this CR: + +1. Create the Node Feature Discovery CR in the dashboard. +2. Several pods will start in the `openshift-nfd` namespace (which is the default). + Wait for all these pods to become operational. Once active, your nodes will be + labeled with numerous feature flags, indicating that the operator is functioning + correctly. + +### Configure the NVIDIA GPU Operator + +After installing the NVIDIA GPU Operator, proceed with creating a GPU ClusterPolicy Custom Resource (CR): + +1. Create the GPU ClusterPolicy CR through the dashboard. +2. This action will trigger several pods to start in the NVIDIA GPU namespace. + +> [!NOTE] +> These pods may take some time to become operational as they compile the necessary drivers. + +## Configure Kueue for Task Scheduling + +Kueue is used for managing and scheduling task workflows in your cluster. To configure Kueue in your environment, follow the detailed steps provided + +1. Install Kueue resources, namely Cluster Queue, Resource Flavor, and Local Queue: + - Visit [Kueue Resources configuration](https://github.com/project-codeflare/codeflare-sdk/blob/main/docs/setup-kueue.md) + +## Launch a Jupyter Notebook Instance + +After setting up the Data Science Cluster components, you can start using the Jupyter notebooks for your data science projects. Here’s how to launch a Jupyter notebook: + +1. Access the RHOAI Dashboard: + - ![RHOAI Dashboard](./images/rhoai-dashboard.png) + - Navigate to the Red Hat OpenShift AI dashboard within your OpenShift web console. +2. Create a Data Science Project: + - Go to the Data Science Projects section from the dashboard menu. + - Click on `Create data science project` and follow the prompts to set up a new project. +3. Launch Jupyter Workbench: + - Inside your newly created project, find and click on the "Create Workbench" button. + - On the Workbench creation page, select "Standard Data Science" from the list of available notebook images. This image will include common data science libraries and tools that you might need. + - Configure any additional settings such as compute resources or environment variables as needed and `Create workbench` +4. Access Your Notebook: + - Once the workbench is ready, click on the provided link or button to open your Jupyter notebook. +## Submit your first job + +We can now go ahead and submit our first distributed model training job to our cluster. + +This can be done from any python based environment, including a script or a jupyter notebook. For this guide, we'll assume you've selected the "Jupyter Data Science" from the list of available images on your notebook spawner page. + +### Clone the demo code + +Once your notebook environment is ready, in order to test our CodeFlare stack we will want to run through some of the demo notebooks provided by the CodeFlare community. So let's start by cloning their repo into our working environment. + +```bash +git clone https://github.com/project-codeflare/codeflare-sdk +cd codeflare-sdk +``` + +For further development guidelines and instructions on setting up your development environment for codeflare-sdk, please refer to the [CodeFlare SDK README](https://github.com/project-codeflare/codeflare-sdk?tab=readme-ov-file#development). + +### Run the Guided Demo Notebooks + +Get started with the guided demo notebooks for the CodeFlare-SDK by following these steps: + +1. Access Your Jupyter Notebook Server: +2. Update Your Notebook with Access Token and Server Details: + - Retrieve your OpenShift access token by selecting your username in the console, choosing "Copy Login Command", and then "Display Token". + - Open your desired demo notebook from the `codeflare-sdk/demo-notebooks/guided-demos` directory. + - Update the notebook with your access token and server details and run the demos. + +## Cleaning up the AI Platform Install + +To completely clean up all the components after an install, follow these steps: + +```bash +make delete-all-in-one +``` +### Manual Cleanup Steps + +If you prefer to manually clean up the installation or need to manually remove individual components and operators, follow these steps: + +1. Uninstall Operators + - Open the OpenShift dashboard. + - Go to Installed Operators. + - Look for any operators you have installed, such as the NVIDIA GPU Operator, Node Feature Discovery Operator, and Red Hat OpenShift AI Operator. + - Click on the operator and then click Uninstall Operator. Follow the prompts to remove the operator and its associated resources. + +## Next Steps + +And with that you have gotten started using the CodeFlare stack alongside your Red Hat OpenShift AI Deployment to add distributed workloads and batch computing to your machine learning platform. + +You are now ready to try out the stack with your own machine learning workloads. If you'd like some more examples, you can also run through the existing demo code provided by the Codeflare-SDK community. + +- [Submit basic job](https://github.com/project-codeflare/codeflare-sdk/blob/main/demo-notebooks/guided-demos/1_cluster_job_client.ipynb) +- [Run an interactive session](https://github.com/project-codeflare/codeflare-sdk/blob/main/demo-notebooks/guided-demos/2_basic_interactive.ipynb) diff --git a/docs/images/rhoai-dashboard.png b/docs/images/rhoai-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..c56a9553a51f4c441ec1551a9e09cbf21acd52b0 GIT binary patch literal 21949 zcmdqJWmFtp*tLmE(2xWP?(P;G0>Rz2A-KD{B!L7cSkNR`aCdi?;DiPmf=lBx&J<6Q zZ@!sXYi6za`Lc>1$f>TXt~zJm*WP!3R8^M2dh+rK92^{$oUD{O931=^@QsFs0{o`% z1IjiWoH(4El!T@?{6Qva#32xtf2!yxnV{e*SGK`6ezdp`-5=TC4|2O1xfyHjWT&8j zeB_9SLy!A!qp?P~M{k1d3;+E&E|^YSVv8UINjCU#!-I!^;!D-=u>}c_0{(IP`t?*u z9D(M)?PD+;vdB}g6bRw(VAY8$(m(%35h5iX9GZpZNX7ndR}9;?*neI#PMnGYHx;cc zB;~)|fRw)4Oj$MsVGyt#?FRNqZ9O4iD@nKcUgq=A7U1i*($-qVZYo4K_B7prX8r!= zIAzWBw79cQE*Qy>mct8b$f%JY)p&V%{M+JpUv#Md@N|(DGowLW$=UAAa7L@m6rDsH zKOI_+3?k}IS?jN=_xzrm(%0|y-bE0HS}!sn1;oFc@Gh>PKf4d(M_#%x=nuf72=S(L zULSZ`eEK@KKbqJ|XTlE#b)dcfkxH*ZR(!o&fk;{ZqTvk&!6%_)UtQf)VVEKMV1_{T zEo>#I%CtAz_2E9isX^F{91>zZSySkx5X@DfEQ@8dP=p@8=@ zjoGC1*Zp2K`n6JYEDpLd5{<=YzItv#}xl*~nrC zwBS+;M?c3zm*gkU87O)`z4&aSQ6Rg>SGsXNCp6rAc$U=BA-?l7C%n4I#TV1ja}i1H z9RWh52tjkJV4XwHOq+%A#N1oSaH-wu0}tKVkHvvau4MY#WvZq`zy;H#hgf5SvzV8h z`}=(hgoP!(MRagj&EzmuoAqPctdx}s`<@MeM+!9xYCO{ERX}bae@BxEyqS((&_k5( zMb9$mYJQ0O`t73V3FR+cVuaYMhpwlUxYDhc&~}G*#_c+(1{DMvTy`Wp4OJ4dOEp@Z zA>B`nKS950Y&K>~)B7A-%7FXF)FV(561D9~QUmtfNGp9AwL!XgpA&St8RP^jsk1o1 z|Fos}#8-F)E-MR3wnM6etBX`%UM8`*)x5l<)%7@jE5`a;31w*6S*`7iZ33?y-xooP zPt3i)Yoc;Y_=n-d_Zqh?t5h26TX|R8It0BEO*;Jy9vHV{p!lc2?I_!Uh)`)UC~+&~ z0c9b6xHp(BGx{wtZrUcVIfXJdW^L0b#7vOhpK-o|>*mZ4K1>cc{%~(_fx7tI-GB;A z54>pv2&n}1H7%LQO^q6SAhy2V&rP-)t24DYq9hck*d%eszxx~KefJ5fGjycS>M7%J zg}S-IwH()*f44^5Ek_2jF9YL+$xGaF$}f7rRVDlm8WXKMjuq@QkZvSAN!5(Ow8+~h zDLIRuPINOGm$yHX3aR^dkYn$*{a|}n` zW_bVM!w@~S%+SVg%0^BsimbPHL(cc(gxHXX>duH9Wgq=mtqj>Zj#?S#4bP$lFu(3WZWt#DJf5O3UX-V5@=4& zOY{;9?(je%p~!iv-(LG3pPrt6QV3gI;_EDWUrZV)f-wgoUF2J{T>%+KF(RCGF5gw^ z(3f5B?F+j%3gvX0Jx*p26nI_tK5!-R%szfVnG`y@+!u9nzxJR9S&QY}C55qUF>*$+ zGH#R7uJ5(5Nau^|(5fH5SqDK6(eNKzcMzQR+G%B;G|^%?eg+)1uepV;)kt3`W`ZR z|4j}E@Vb6|!UBf1xF32loLi+?;8_g2MKV>(geYOzp;@U9Ro)JNW@1svUCW~6lUeZrqf-xA^ZS&zS$&;L0*6v|znMj-#+%pRAD zt~&ATbZMjZ&oXg|3IC*N9Bd0>_-}9Groa-RGUAmoC;i{QL%5L`w*OyWTvqi#Kq2e) zeZXel(QDvASxfGS5Bqq-Kl?kGZXBN+$@*JAsz=u6ziW0LaGG*E;$i62)Uw}V6I~IZD{aw$*Yk(uqiH^hix6s@^lf?~H*3rEU zQkzG_J3>bApS|odQf`?`r9cunUfW9gH*f^FLj9OBb+7-fZO8J#p=9NkKCb@`f(W=| zsjCCpf7V4=3J~y0zh8){DE}S6DG3bVCRup$cP;E;23F3NwXMO&i2>LsEYVaa%sx;X zQ2w`$0!s|7xJ0bZt+2qq0l-!E9dR~i{__^n;FcrZo)9o?@;*d=xw0adc__KiWmU-% zPVSGL!6Rc^wMEC4BPy@(297YKM*(tCnvREyOL4Dzd)GTYZhU)pE9I~qc@e2g4?<|E zdZ$c5Ve~;Uto4{mF$Xi^dWx2T!KS7ujZ)IrM)lzZqjpUMs4L4yy3DAZuHI?IrFbck zQ8L$S99p${7_EIi;do)cu)g2urhTPlTtDo9uPMC(Eqdvp-v8RH!DL?vU;;@M12Tb)JCh(2!J5`|9i5y-*)x|A_c~u7Up$SOn zQcHXOt<~EK$hZyhcu{Te4Vl5VM?Vg~C zE|@@0X1beAKUjC@jcYsH_qk6u{J(dmgBl@(Y(~B#3Az$Q5mcsY(-yj=kR11`)^@09 z#%{#txH6teOMWnow|GiJr{wZ`38-3|Nyk^N%5sp(YAnk`e|aV>bu5+BExEXtI;r~L zM$Dj2nyDM}zdI*no)+R3>KOUV6rYgrYc<9TCWdHYzBu#2grOO`IrFi4w~e(opY4dX z{d=EmoRchm;m*Ra`7MoAlY}+5yfLg}z6=wyE{6x7q_z{k`r3Yb-ri1)#qnBlO8(k? z=d3bAz*RO+{)NTzExCKUP>mfmFeAo~eBmy%Lr?4ukKOhp7xkwZi_xnhJnFpR3S2X9 zbou>ZeKcJrjn{^TQ9T#e46erE#&hSymwqr#w6)B0ttW`fZ_G=5wb#{j(Tig=jW-fX zetm8mY1yQtc?;{1xO&@3<#VmB>)}Bov@cNDC z;Q})j@$~00(#2u=>_EES$QN4LE^qX1f2&**B4LJFuzzb6bvmyRe2#ZAE#ee0)g6wL zcWzfMe{QQ~0c|xKBKO>~w{c`TXqmK9tt)SXWdQmqlvb{YUU!$gVAGT5DGDSz*<3sC~AJufX5mc5) zZ3_}+V355&gVLrV=+@r{#b1MRe6>XQV?8;8C9?AsGs{AwZ#~-g=8B8;)-no|GZPLs z$29zzrp18#}G_#uHd zhB!>rWl$j=_&e^EkR{vmO0^PeHQM~S^~30NP4Vvo-z38=?u?1gelFHKDwMwD&6VGf zA8K~ibKSzOE+w;R7r*x}5@UA%3fY;mGVd*#QP(|Rku~{rT4K-+8r@1aeFk|uP=F#` zUIedw?|)^!avv1EX87l3ctrtm!xxk~oLfn@A=zRF#WmI1F)9$07hKv&hy`p$5zQR` zre9xfz0ntbi{6}2#uzW^d=#hkGaN@LG)vgWb!V|4#eShmNttQ*IU=$So)rGi`E3JP@$sM5s~^PKZV9E%=T4&&)dgvBN;1>P4G+R&x;0#rhs z#22Jmuf?2|sYU!v+MQNcW6ycNlSq*r!+YC7dQUcT^=FudZa8MGL+$>o1VTvuW3TJY zE8ZYGNysv_Tz=nQYE~H+L#0F446V|LAr)dHm??hiy=1=yT^cH4v96On-59r9h#Fpu zhzjpuy)kRFUu=`~Lya1WoeqX)v-T@A?Ip12k1?vUJPCa%}(bVZ|skhai z%vYkGNgBQx_-VG8pFzq^@M(U&+N(TGNTqBI88p2XBjBbeK!JvX(B`@1yHK-3-st@E zxk0e)*mX?c^be4dpCCu6n%2kDPRUtBvPIE#HMd<^W}VXb!_840M_3}0bjMa-R-Nvn zwiZWFM!!7Mz$*huMe77KR^b=DiA<5r`EXt@ zGw0Q{HzD)X7HaS=VW`*~s0XZuHS>nc3!LZ4?TsXSKcQW3#C98+$vm2R<`20^2?&HI&>`x6G$PY0u)eLfVBSM_=G#Qh|t z&0c#fhHk zq5C7gLTSWP{6bqC_Ic3b#1)@?9Tv&|Sk>A<2bofXx?rVmQU8?}^HhMmV6C!G`EQm1 zG*=v;s<7XlYW|ZK27qd-e3N|eM-s+~lK{1SKc#)vr@sSCBY^>R164-<6p$pKfL^^D z)%Yiw?4JSy@b&5dnf_lL?;;B*pzsPGrhfxifdTVW!=C>X5cZ>F!sSzW^LIey8(@He z`2yQN1w;ubAd;@l!hag8?ol#DvMkd6J*QVRz&T0I3%~d`03R4&A8*n8@0`Mcb8J8g)rv6+$uzH+bMW4uVz=Pi~Q;^%VR@a>zQ-kX?E-I>WrSIp!A8K=xh5O`$` zevu3f3!_d7Y?P<_sFHcvuA zLK3(vB7jwHG{dSnUeLq#%KTE57Y&UY#}T^JOz+l~PLF}lHrvaos$1=uxj)}HT-Ri9 zL&zc=NW$iIxa?M(mD6?btA(z-)|T<+;vl8|x{aW4e&A_$uKC9;9_k8H*X)jAEvCK0 zBsI60v|88wd6!c>{N&+RJ-q0bJn;J4WZ*zd1YxB|iJBdKEk!2cOCu?EFQwl$tK8== zsvNmHvZEnfXTNlSQ?iuInl`u@R42QLotbA8KKCde@|;&+b9qC88PXSP{bE_}kLmR) zZEG7@kcbd&?@`GI<8x#@?)iiLX zk>t~=fofBZ@`CdVK6@&Cp0PJ2UHY`c3-#rwvz;k}+H^rJCdOlMwIKV)m-8Gx(s0W} zxn}(;sra;;vijTIt~PJyh;D&8haPgO@`b0~@zW)0;~j7CV!($q6)*Zu{ez|I^b8>{ z&@T^vBZJK!?YHAeRh~=6C*3Vh77&@c+J>g7GI|TjWPYdR=`i#hQI@qUF_y&j!Oui` z*mIBKz0i29LXZq77v{>0>xWKZUZR#H!bgZk_qBDTBih0(h=6>XgWk{B&Fa+>EM;yh zIzjZw#A1X~5hI;>2lh%^(8#gKY@wzwN286^?(oeUdr`;+aZDMvXtRUvhBP_FChoOX z@m!Hb^t<;PT-Yv32%-ud^vFx8h+@8nF^WQd>L09zpV1q`2?r<8D$s?6w-N791_9LU z%Qu$kERp3zK*HL%+_jB!j)%?i;2$$ zYM}}Hfcd#wCPscTR}T4nuJj>WvqTj{zETYC>i!V)eNohRKXqLj-=odnXJg`9_vT(W z+*?j{)k%skyk=?UqiJs(f55|)T#~2UmMj`qZTsy^)n{u0OJIw7eVu2DmtRg~%g?-O z|3dJvk)YS}2>rjXT)2eEU5jQq~9j z(KM!dAB+Ge=yGfE^(&)$UB@)bRBpd~5;4!$K>1V@J6EEy;;%d${XQ}%adzk&JLUdh zvEJi>N!wd3@cK%pP@_4&k-_jR`sHlOb4GP*_R+Sxhapr%6m%5inPM%iE;S^?Sk!G*Y6B9MQ4?Q0r)~hRjuS32``nE@$l=)vuh{+YSkTry( z&IF96@W)Qu&1;gYWV2;rK6qE0d`LAV$0iZ}(Lx#K9u)0%2(DEK%GPGCzZg(6a~E;C zsd(XbhQsDfhj-+GFg98sC5>9<+Vbf`geQwnclI{3V$gNOS)<2HY4N!^87Wjp*cwZs zPGt4u1+$0t0#{xw?(1BP^>Oy>7&7MS_ABkk-k#&^bk^+<#A}QpHchvA^4Lqw+uOVK z$jY4BOG#O^cG>vXzK#U5JY<)df;Dqnm%DpT>tmThTynLrsWo!*Uy10x9P!jLp2dz5 zNED7DzU{08NgJ4z(j4T!`thD8N?DFyDL1t&NaCos8CS?UvEMm-0U5WuNP04bbR9dj zW!lZ~p)kn~C*QqIBA-!qc77t4#FVGSiRsOu%cM%t&f_pCo3HdBpH^#My0tp>K7UFB zej9Nzs=RIXZsFk&zEHdNll$(xGM~fpccVL09I^y}a@%M#q^l!iyf(BN&7vk1^m>!L zbOs0}{oqbibO$Ov)6J3v?TvC!Gu>&sF-56 z5Tr8&U&{IVQhVv9MiGgqlk<&bGHOT%lou#v+i0E^)rxS!imena5Zo-s@nZ{u zd~Jnq<0|?P=ID~0&z-~2OdH(}JExwI*)K>}d&wqf=F=!pR!nahwtol7V5lB`3m@L1 zq{A3kVDwyWW=^0gtfq>OK}M#n-K*k#LdAfE9%_@TB1&3XD!cDA-yqkP!KIeQW6Q9@ z1kb2b7v;&{r4+*h$?4?sI6*2jD*kZWXqq%ee7CUm3nBak2$aOGQ&%$<{sx@u!yGcB zbNyaD!5E7KRH$LFJ5x&ij7~Y;IQc$*I8(q)+hAgzh{O8F$b)0ulicEbqs*q@>jsa# zWKc!W)rp0G&q<=_JKi$%*~<6=94Y^fQ0XSG2auHVnNBR3=w!6{NU8+RD~^J8am5gi zii~*3B>(kP-MO9XeHZeTKNu;%eqv(PRynAyMk|?z#XI*r=uRd1_AGMQ^DN|WcXB0g zF6$He=X}{reO`;aPwXAAS+2X=VfEI?D^Krd9nJC^Ddx&OCa5cSl-<(XT)*j+XB{`_ zqKSu+OBdIJKQVWNHeM;DurcmE5ZIjCVi?s0EYALt3=M%pQIj3~Tri;_JTGlR7RrH& zbEk`p50y^F_nQu@heO|jE;N^4;BRBH-=JX(TWna$z$lJnottI&Eo8Hm9}d2tl!PGQ z5aKIN5o+-E2{vt+2H-_tA%rZ}IYdW$=KQ2XTGCv?g#p&b@aKLN8ik9DpVm4e)B8x( z4$3KSo)`rDREJF; zt={6-5sa=g^j-K|6g#~_`fTY)sH5J+P!hBL*DwvKz|xVbjN-)y`-Sh3+J%~O`0Q7M zGq%dZl!Q2Hu;`c5DcY>*b}sr2-XHx-S?v87mC~kviL{yA*K3EbPV(cSu$0m0-lzJe ziaI3Ve_))llq&jQ6LEdMYdhQ5Ih;~Xm>xg-3S}G-FJF0icSf?^G$I7=UetL!&Xs9= zD5+Ry)U=G7RrF+QxEOZi`QD-nhMe3I54ncTBGn5Dlhr}bD-6ebWK&{EL`zx?0|V=* zVm9wg*GcT(A(?&_HxY+u*m3ZwtlKVn%kJdPUT5ofU;C^P2oMg=@I||FharrLQJAg2 zZDr@Fu4xHS(l+G&$kG!Kqc2r9(EJpcn+_)#cP7O^Udn}p(P^k=Qld2_#?uHrE^HzFn8`M+B^@MVsQCO=!RcAhj$f;9IwcU;9zUp&6U;*&DoT$u1t{_ptt$5=Wd{w3%0l)i{?$SX0*bWgPZ=|0X|9Xep3Mu}g;joV6C*K-V z+y^XjMq+-4*~Mh?2#cf5=gG_|jec+6s??8u;jCCK7j_S{4K}YEqFmc0qx{Jg3*TW+ zHH-e^uxtlo-@)!KJWqP)URRU6l6+}h(R~ZGAbvyOR4#r3(^CcUX}H}ot{Ym*+g3Bt zWwe@V0|BaMY%4C$==Mx%oGeFGJ54vv!B)|`T3YXJNbcI|EE0K&DAixW+c-zZlHL{8 z#RX}WM^biT8RWUr?#zG742=7o)KI@tMaHll>M1JdWy@}?oJoUb+8x_yQW1P*JYx6T z=qyzp2P%fEtLkj_i`X3Uf*)*bDAt4h9ae&O@-0j(L5(_Gp_(q5+QVYuaQK|Ww>m$% z)+5?$IvSNN5OeF4GlxcDb3|}{FWD5^?jq!*i9!O6Q?dV93%`b8DjFd%j>Dq_Z3`iR zWmgJK-q7?&LCDrnBRS$Y@s?k9!0onrt<1KI1n>0=i?tvxXWT{Ik^fDb@LxVu9%U2~%B`_+MngKpPa3O=~!m{3Ht2w)K;Uz%y* z4?GwJg(?I(xSm&xCd;D^W@_b}R_|JZTI}#n)`w;?)IG}8(o~5YPm9uvC0x-sg#0c_ z?%xt2)l#ks9(QxKF8k6?;tbQMBpbG6*?htbZzdHJH5-XHE%Ruox_H{UROO?wG*)*Y zG_v-g&Y{#`owC+pMQ{0FQAWmu-(FQ^>l^uxhxjyR{z8lhsx2KK0My>#q)FJ zsDEB|CzL%OV>RYWS7Ml3eLpN5oW+N=N`~CHam`ugW}OQz<>QfK54D zG^c7x!A{JN!(>Ka|cxs^k zrQ7M9ru%)l80+Lh%FOeKc-%b>*J&$X@$h=M4n*zd_==Tc6=S0FL1)sl@D8zC{4Utj zV*-Y)eXz99ab@4Fx8g{=%#4=;W$eCZ>^D?Z%}j_3UIYq!d&z>^m#l=J)--g*FL5kO zlp+~lVdJEq5pnem6jFfE;;Q)z$TFWLD%dd>Ca?vJG=NE$xKxae%cL8H1lb>r>9y>R4F%^$RO@;_^Aw?L z84Qv8Lp(oVeg`mBoe8Fn@ChZ9mi#?N=+gdZZ4OI`t5y7I-GN*7|3-qA;PqIcocq#V z&PX+P**wH-DLFss1^tKdoFGpyw0>g7DF&n%SX5*xut03p_g8?65zMJ-M+AV&7HhY zC;o$Y%hR;=H|h?s<+(EKrt;a0P6#2)Lg8nBmm% z@;&#~;B)-N0?>&1qRC20A)8+f-=+)a2AIf&j7L+Mybi%h(AH}yK}&YTP_LayTEB4f zph0{ddr_+C)Gka+|z*uFcwd4$b&XiZ%-7w^^XVgZ`o<<*HNoa-43RU`iGD;t_JqI%)i`8D) zhE4tpdOxkn{Tk&h(1*{X8Z))*W``=x`bQOXb=gii9UuAck&gsyebWpVH}kBw;iD@A zLV2uHc5`AeEDLyp8$hX>8<=Bx0uY^T9@0WKsbRGf4ee~^o$mWXxk}SFc^1WW*pc0# z?uiHp2qdJRwC`aU_>V&M;8Pbw`Bxe-o#Exv1F_u zC$70n4x3LUM)e>lxY2AZG8T*DaH_c=Lo7(*m*=)U*>~E;C0F;siuJUi(IT}bclE3;yw7^jg3_4I#O>f~{CwzR3?0q@$JkIOp!XZSr zvsXX=4C5D;|M40v5s!H)iKwY-e>%7)AkXyu$@L|mO=%bg?%+ReAe%wG=i@>WxMi?I zc(Dj}>sQ7;uLM3w;+UD%3H_C~P@?TPcOc{`SI^%h z8`d}KPSj8!cKbCuR|C2Ou!Mo2hs^1#3(=Q+Kgz%8e`-{7oP&z=S>}5D{(zNhv^bj} z?86lhW1PbF89;y&8=^#_nhA7DOit}neR_piT(;}DbVc%$&r2cAsZWCm>vX*nDyr}Ahy76qOKG}pWPRY&5VK}D7m zmRp+?mW6$C$Au1ed|?q!2eMF=Fw^| zRL@VG8Cv^Q<&n;#b>m?>mw+liPp@*NuKwYj_0!Qr#+Z*=Xs)6!284L*Cx*c=(J?%ixATaw=3D5fK#o_SrjMr(Frn%DPUhR7(A9RjY!v^Tj?)EZ@UzXxu6hP}j>+(d z9Fs6Nov~nZ`nV$Vq5E~K*k24pOU8?YfX&$-HeX>|@38WXEH^L16CS_Vp(TWMBSUUX&-F>0>_s;jZXovaSnJoezMdlH=#4U?ZqwcQ$ zQoFdnyOH)tnXMS%G})eli5QX1-f7bMV(qq5;hz0S5Q4EWFcj9+<=9p`q|dp>&oz4z zczfR)R;mLAG(Bq?Z_;$3#-163&wL)ORRlEgt6RJ|yFaG1b&vbOw= zDUv){xCKMD;^f%KUbVc8rMAK875)qi-{N~ZVE^bfF8m}zB0~@2R-}RKA^FQs93cNc z`OJCrt zVap!NtEBa`L&+((+%h1CdX3UvfEvV~&bwHNTc(Y(GbiN5Q?$oxxO1UCMzY`$oI$yH zF+}LoAUzWQTT9FgVou^#r~Fwk+WvlvyqIb)aqFAWyC3?b6$>t~WeH(`R_Ej_9=tgs z9<>_JQZm@Wb)&vJJ5^`hju@;+VKZ(sZL80YO{XwQ8onE5d{*0F7CisZ9g?7;ca3`k zP>9s@;Ci^QuRxF>{j}n(VS3JCsn)9Xen+XOMz7+0(;O=`BP-KM8xc)xBoV zCK%k}vtsv?39trk{UE`C5mXFz>pwW3TGfG8DGeIc-;g#~1oWJ?1|FWzKpmwPeurf8 z+TmaM<`1f0w9E|f9_y3!6uzmZOOO~{B!A_t>Ihbq_=v53b+S>Tu-y5!c%OTyiCy!@ zS5Kd2yoow>EPQ}>=4@qiZrOi1a=cmcgROVHk4#=DD(BCQIMLV5ud)Df7+eGXPhyGQ zq%IkslX0uB(u#r3;SzaJS=?CWuVMP`aM?M2zqCb8J`Tc8yH61^;`?xhH+?3k_f&yO zTLOue&B;HE1`J4c+#q99$p4kT#EbW=rt$ZXAUNqBUM2}pPmDQQ(0{?ezhi(Udbi|l z7REzRO4S<^Af&D%{n`PIpCAEt)L&TYs|1bD}=It@Ff9d$n|pmrA+4i^6X>-+Wc%MD>Y>IBkbjLhkkcL$-jW>3qjeJ1(#;>1m< zAR73Bj)q>mhtbA<)hUn!<8I8VOiIRDd5_NIsFeM^YAe_e9%%cvETAQB~ObZ6>v^&-B1k{Ywrn z3<7S$f!(-%bP01HwDp;MbM&W!2^;aCb1S=M7&aygexR6+3KoQyNo5L$X}(-(Pn*g5 z;DgVvPHPOr^q7Oioz5p^FQ_-PJui;7uTr^g3_%RfF{;;%=nlebP z2Rlx+eQtp%lZ&X}*pZ^MytN5D*|FOE9!s7Mc|G82KF)SNL&60f-lh(GX9<2c*t9>S zRqRnBxfOG;ajr~yW@o`*M&orz%3@`Rh%k0!04Lqm42#I+XW59)5!L8gG@y&&e^Zt% z8X&!V01HtAj|^nIt%NTReL51cL)3#zCs6yH6H}@_(Z# zUxWo`N_VLSFrn%#$(D>yq$!b-I=fsqtN4bRn)p}(bS*FNSh}ih%T}BZelb7OB5Qea z+Mdp8Jf4+PGEBaYFB6OpUc4w9)U9$xZ2mmK&f#@t=bh~E*7?1FUkj}XTH1z- z?iN~$+xj(ZefgV{z_kCB4eb%3`@DklN3-Cq?UsvOLZ4Ji7alnIOr|6nHMRhqd8zYO z-m$Idz2OwLABW<>_hb;656;<@6koLS9h~-sZln+>AfH1W!M^{&liq<-_vnWTVGkCB$|icRF5d{!+u&Zmb@-@@}W_{FB1%umW%bDcF`3>YgL? zwcGv=L~x?+_-O3ZOudiux3cyInpdFkJlcP=y6{fGal`6_y6*o{ieOYAXhJ4#_TnFe z-~%AU|M(r?Zaa{=&SqyUR0{}={E5wTh5)yQw_(Z)xej798U~4n!)Y5QlRnXZ*J&5`?z^08g4oryClY4kBsB_4%`CYvO903meX>?8K0Qprk9e$E(b zmV55L^Ez*LI4RFHFNH>@%nFD_`ndvy7NO0e8ucry4fz*&>b*}MYtKfHn`A(8Xb=R| zJCQE{r~%v_v{K|-xTPR}yw-O)Zcv)+Yg`n0$q4(9r~?+t2TBo+!MS z$qqxs8T{QJJ9;%QE{0JkN+xK(TR$VgJ}(E9D!#CNHsbqb!b1K_QB9zX1*z_HTMW?< zuz})$YE_D9OAZCa4C1ra{;&>R^2_4s6s(kmKr<4Sevl}na#t+ECS$G>n{$S(vzf^%i#4VSkk^>CP+)zPBb zgVw{5$yo05=|4gjBR7VJ`B%o001tLM04=b#h<(ptF_wR2n`)+5C+R^sD^Ve(T(+^w z!n_0Z?t2Y%sz8NaF6k?Ep4?~s$AneIYW4uTLXc##zzc>rV;n-JR?VLo;nD8jzR6E8 zD-7b%mYlTO&@;%zSvoEjY>uYMF3y&&UmPsD-u46~C@j^(mVFE;PB@cp6AL5<-zf6- zdpP4aYwD4V44WNW%|{d7jAaS!xyEwdt7FqPGYShET8?B`zzDe?fh|ST zW92cN$i~X=2(H0%gA)$8jtY$Eu6p+s=MoYi;Bj5^j&T?hy8hrW&%mr*Hd!)Npg`}C z_=4bNns(y=`9MN;;H(EXGt9mp`(Pb#cYz)({Kk^mM~1gGZmf!bVlfym(p(3Mk9`-Y zoBtSU1~+!Qr?(fvqoTl71pb`ui%vficv}hES(0q-K4gUXx3&1VKLhH~lm)l8&(DU6 zq?&wZcmm);el)E?xN{1@6xOAf=u;{5b7UdIfG7LYg^(yqe-4OXf^@2Gu1@KKJ zZUE0Q`LXUH@9(Sp-A}6m1wSUO58ANN(eZVmVr*A>yVGTP!=|{$zlvFFb*O}IE3=p` z&91W7qEd1MtvQ13{U>wPB$t+Z6;f!lhBN`lsgqV+ZP(S>d)`6qwiy)=(&n*O_gs7G z8PFUtGsW?0!NUAXChE(uSa-nDdJnhj06fOaXENe&^Mc@q21cO z#M;Z0ui(`2e!FkMknW*&&$}kYG+`z5E{{{*ZwJjVsQl>e%z161S}4M*kk@?+{s$gg z(bm1a%=NjvVe^oamnIXcO6ZryK=41$n1Oa{ER!JPA>t+YIkUcIGq}>>3rB41*|?a& zv~YmOXsKi7FMTxF!%cZppJ`JxDh|}H{sFibpRV)U{n7;zgGC}3G2Q}2VV(WyM#ght z7?|Q^Km63c_f%JT?0K>V#+0GA>`j*yq8PUi;9w9a==btijgeIOjtUMdsJ{-99!h2{ zZP3yP)mO$z7c~1Ifr+A`(VjsGq_MS^KT^=PY)p$EIV}Yamu^l z*W`T&Vh@-!Hgcxdl^zd%?3>K%t|%nwjsli516xNPn)yHzQo}hIR18Zqw3(P~62Ks~ z#RpdjwCn4G8h9nUzbW-4k(DVaP0F2DLtS?P9EO8|fsPnvsg%Bbkk0Qkct&KrBkQ7@ zKQ3yZh?*u5H&vD$d4Jb z{0WMNELvJv0i zkd2$@xfNsMxN!M0k?}56APqdgJ&Q}IdldLI#Ll@}R2Aw;(~$DC)?O$DE9I5px~y|y zm+c3NuWa}cUUIy$97sk4n&i$3W^uSn<{8hZD(qln0Bn!NBo$UQ_;AmZKoB4?-5ejo z%o#TCo1k5yPm8tN`W=-Qmse4Xu+Z_~mv2Ydo?@18V;v~kEZI&06tj!;Be<$Rwwty1 zOMRx%Gf-nUgr&%wLsC;HA5=ks51)nPZEa}rAr;X(1f)&&qNXd|GjiEz|rsR&1w`XxOt+322yXM!F5ynLM^71aFEo zwHAWphJLVn(y(U7oY(iaSYW9Q`6-`N^zPwi_Z|W|j5xbh8<@KjW4^ z;FY}W(_CK@c%a2=oT*6AZD==$H@?&8XLbHv5dWS5JoJ)p$ZQt^SQ>>sMpL{d%MJ1y zL)RSO8ow8)5jtPfB9|)N3Obs4{A(ylq=m}LRy}2eJIig0I{7HYalrh5S=0RWsX2j` zt0wBfJBFw4h+jG(Q-WIy@5=d-Z`!_Wg8g=s6BM!ZJ_CH13dLqpy~Vo(bGh z%LySQkh0+<_XMA4&TNqn2CRqvpMzEOwi|ji3P$7lqF>6w_?vfyup`krO#wP(LTmkE zn%Oi$xj)Q3J?#Pf9b6-8JepPt&T?@7N?5HF!wd)a!u#>F09bZ3ipn-_7MfIh!r$8b z&PbJcKhQN4WH^hOUV7 zEX6o3&^BvG*`bZq-N{l z`^MU%7)-RB!+%}nc=s@(U6FI)u-GMvfJr&jSGUX|`kcbQpvq#D35a>;kH5d+_PU1& zk|AChB^Hwhyb;6Qa$f5}XDkjC+?y^cY<+c!g_vmdUzR^C*tm}5ZSuG=wOH2dB|Ono z;V}g_YiuprYs|?!e!Usa83820r3N3#TBm4}ZAh&9(bG+a?(S8ldeNBE?f}fl*%vxx z7C{qV%YuO^YtGoMVXFb|WbO4#Kua8Er?vns8|NAD4Z@84A4 z&lXx94@PsbLU-GWNoZa(uL+w=GUe)A^P<@V`dL=T@DM82sC>GHL76}%(6;hD|2N%A zD&Lnq>LV_!z)_V{NG0L+Lu5xXYss~mFJjCpteH0%!u>I8?4O~;L9wYE79oIfEWSnq znF@t&Cp!Y|Q!*Mtf@8Or)WKf!3hhUsSyrHO@G9Dc*`M|a)hrrg&&E@X|Ezly_$DwG zo*}&^g6NBlELA7G8(&^Nt}8d!8?lhJpQF^1(|ra>t9mrW3~yp5&B1Jdq;n$JFI8;Lal1<#vXlrq@@t z>$9rFmE$iy*IKOGFmrA9zj8gXKVIWpFt7PFl*$>FtDGsQ@6aP4~MGti?Z^WCvL%L0_INMzA<$<9)}{xe6W$_ePAyl8H8 zy}d*)HYXybh|0CU<9&)yqLfKDAoJ#s2$IH>OcQld{V4;ltZpvkNYu zg*6F?ea;Kzo>IJXI)^f{x2Vb=>61?a{iYlBD}gCMXXn(7%W;SsEZ~NfQM=>^jZ&Is zHs0N1UyJ|UIX@mDAp@%cPfE~(2!&3WR9hB*lc!o(LrU%*@zOfbr`dEr@h$8zl9eShorenkDw)Kw7< zah61&DjmaqzCukoOV|cPvH$qqQ9VD4aDqo#;*;2p$LuUG8S0QH|L!L8kb1F!NsBEr zz2|s;^14u6z24D*h|j&E#$is)4^7x5(ya{uw-Wvr=ii4pieJu#)}x$|--AIzA?yGMZ-3!{qFsd#&uXqg86Iw}2%F zAJUr%xJ3m8qCh=6lF7efNayoJ;|fqUX{Qyd1E0YLtqwZ(eQ*0jMhwg1RD>d2^>v9Tt=YlRPmPiY>n zXfs;FhUvbNw1F?HhEq^8X=G2-ya7h^%?5871!_q9xCqfrAfHmL+D50=*>+)cM;URJ zEzr{6@26WvWP|z9Kg)rZMbW*i@`P)9`z1~>FW@=DA4z9H1=_Ffs&Ga-uJk7v00mOJ z+DZ?cd+r)?wE%c9cjhh;QBeS|B;}4>TjJVyg;pwmYN66`?S#2V;4k&jw9nbcU)C$C z_rVt^{I4el?ytTkc4HIN@JqU2J<``~aE3?<_YP>O0-Ophpu$kL@)ONQ z4S^(G)jZ*VWBh=fE z67(XTd|Yre0T@q!=O&zLH|=?F-nOV`Aq;sf1*$^%iyk|%VsKvJ@5rXSz*Zsws<@nt zpfP$>lEOR>aKSfM1vy0QF{LU60?~N_vLoB>(X$;Ch~O@D%L>DHD@#IfogNF~H8?+( zvGo3X?lw}Z#dE1Aeb`^V#OZ=dWM6ftfB~AhW>w^qvz1;sTCUEWZe|9s*F2pJIHBE> z78=Uyt~PnYpV6HIx+xO^fgpOm&r?H{*ia zblJPRt|NNWo5$r$0fw*c2E?$-1!@an?D&jI-nb6zT*!$5T$VKf3z_l@kQ_!{_NtHI zkM0TjXNi@A21S>@wTEw&7?$2Qy43Oda&i!O_I{mrRe|{Lq*-5J$NJ!=f}-&Z*`qOu zh*|yx^U8;Dv{7ua8;&=$m<25+FlkY#(Z(&Xcz^X&LGzwqCQ2p83B6qpWtXOxurXrm zqYZ8~OXUVO$e47*&d^)kx0c`W&=)rESeMq{>;L|}AtPNfB_1=$mn-9!VR>szvela) zB8WfST}dwM-GE+9z8NP$yH-=hx!Haq9+1}FZYAg$dw9A~VrrCr#KxSb^a-dsa|>4m)a0k&3NqF8DU!f@k<97UNy z{X4rMZo-eMEF3zX@A!D12o%>d?N20=DO~Cp#538%UFhmts2M*AWhB&UXO#AY*_`bo zOdgt}?Rv=}OuIisS9n8PgNNREjq^Q#R5|n7z6A`MO3zW}(nes$ap>5z>S2vByl0or zfDEc$x7m_%HT3gT>lGNw=&i72swrUxEF=0NF7tzXB)E)Hb}!P*;dqQz`SI3ldtiI2 zak+)*1(rARmT#<~Esrc^t7zgPb0xKr_%D5w_7XFh&3LN|yd!CaQ4?=IFv)An30PFCrytvQ z32*Ft6q|3^!CEx*?{T1Uo!`u8;)_D1RGdo{WY_HyzqOHq8c6`Ogiu^EdB|eIO&Q{!hn&x&DVqOpBeJ9Y839+aLTgLr&Sq zwb;XobQAh9P-8qK4TX9I5yl^cCf{)0lmx64k)x^)(iobg5gN9!aZuPe78X%VA`}nU z58|MO_usPyDhhMtFLK;P6TOpmT60e0LI&8882e^7bp3QF&@jAi(*J6^&^LP@|6{U* z<`?xCAGkTKQfH07PK3UQD&&a3+=LJ?&_4a%8%r?oX#hTTrTZh%1hlqXtkIPfIYVB= zMM`ydg5Aq@%rA+fLa13VOtS^pO8In!Oo~?!9DUd?a(3|wSRQ2^)`+R0^O~#dB;%{^ zQA`*a@6MS-K{=hy^f^zYEu$_I9Dc7$6-6-Rvea|sx4HZnk|u*jTS%p=FvvGR6caQO zQgo4vD#u{*HW#Um^-}F-i`JOO@pu~Lj%wL4sN4y2VSwk#v-=gdEgSjN64*T(;@V5> z;WcGOeXKPKR#-U`!P@`$FDUmrr#U(0SWk50WhlfQ&Av*2>1lMaHE2eNyOa0dAZ?9$ms zA9@G1Q9d&LI=qqc9|LFV)A{aC`kOhmp98nmWQK0;hUna=nVr$h!kWg3y}lSpbJCDV=b%ya7^NFI)FZ|8Cq*$ zM-E#RI>-Svut=3`y%%#|8(#fXJoB%Ll7}3;%HzQUbT96S|0dk_H~Bvuliozm3Sv)_ zsj%jO# zDaV?~DDxNRqys3xY}_2zzX7j09~eaaj>J98_F^05*dx?67j1-{ZQF>AzShr#qnw(KDvuCy`j|6*P^KzU4ief-W~4wBy+uBc@m_)}gNrrdh_ z;2%xx-Eeb6=3~ge#riM`AOoei?E06?fCM`8y2XC+rvTUQkC3DWL008{p(G9v=B86J zf7gX&k_mc?Qn~(Z(s#fU!@$`U&v~aRt_A6UAmL=WcOY}OKP1FMcwvfu!B=Ge!me6% z0O`CoLuQ5eqZ$?RVTmv>N4&6fCw=#R9J3F!UzF&fxnfT?uaXZJT!5A;8Q|w(uczF)y@OKCCS3hUJOW{6`D- zRR@-5To)`Nxp%K3tlC4nA`1Ad1(el55KP!URb!On(Xi-g+&lE%rG4d7sLOKF-6vM3 zyO9}d`O9%&Z|d(x04A%R%X(K7SjqKqNe? zt=(2D>|!-O`qjz1T0+({q&bk9QQC1Nsks(&r#or`xV*gFpeX+}t&Y%VCF0ePF!Zi% z$gARlqD#~{8rr;xbmvDhM|N_)Nb^>z;yJ@&he9!QUSgH;Eab%q#MzSR&Vo2V`QQK@ zwMRt9IGM^I_B%p*wdRL+MH$=}31O;Bu!^p{Mn;{Ro$zVM>KfPT-$Q$1b;7(D+ zX;E;bY(+D^#(y6UwWJGxPoA({fStb#bSX1(IhScJ&7c0NzxmdNgC2tdSqb3NxLvL9 zbjc#|dnnW@3u*JR!Z{_VUO@pN(mz168&$5+H$@R9FrWJ8*6V`J>;5FF=2(@sDXk$? zD(w4pfD_`S12OJ`Jqy0yYXZ}UovdSZ-j$3V904yt+hKi1IInp}f Ef1zCvk^lez literal 0 HcmV?d00001 From af775f9b58af9ba20ed47285997ed222fbbb02aa Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 3 Jun 2024 09:24:49 +0200 Subject: [PATCH 246/369] Delete Pytorch MNIST sample image sources --- test/pytorch_mnist_image/Dockerfile | 18 -- test/pytorch_mnist_image/download_dataset.py | 21 --- test/pytorch_mnist_image/entrypoint.sh | 3 - test/pytorch_mnist_image/mnist.py | 159 ------------------ .../mnist_pip_requirements.txt | 3 - 5 files changed, 204 deletions(-) delete mode 100644 test/pytorch_mnist_image/Dockerfile delete mode 100644 test/pytorch_mnist_image/download_dataset.py delete mode 100755 test/pytorch_mnist_image/entrypoint.sh delete mode 100644 test/pytorch_mnist_image/mnist.py delete mode 100644 test/pytorch_mnist_image/mnist_pip_requirements.txt diff --git a/test/pytorch_mnist_image/Dockerfile b/test/pytorch_mnist_image/Dockerfile deleted file mode 100644 index adbbd7cbc..000000000 --- a/test/pytorch_mnist_image/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# Build the manager binary -FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime - -WORKDIR /test -COPY entrypoint.sh entrypoint.sh - -# Install MNIST requirements -COPY mnist_pip_requirements.txt requirements.txt -RUN pip install --requirement requirements.txt - -# Prepare MNIST script -COPY mnist.py mnist.py -COPY download_dataset.py download_dataset.py -RUN torchrun download_dataset.py - -USER 65532:65532 -WORKDIR /workdir -ENTRYPOINT ["/test/entrypoint.sh"] diff --git a/test/pytorch_mnist_image/download_dataset.py b/test/pytorch_mnist_image/download_dataset.py deleted file mode 100644 index 698f68f67..000000000 --- a/test/pytorch_mnist_image/download_dataset.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2022 IBM, Red Hat -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from torchvision.datasets import MNIST - -PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") -MNIST(PATH_DATASETS, train=True, download=True) -MNIST(PATH_DATASETS, train=False, download=True) diff --git a/test/pytorch_mnist_image/entrypoint.sh b/test/pytorch_mnist_image/entrypoint.sh deleted file mode 100755 index 7487b377a..000000000 --- a/test/pytorch_mnist_image/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -torchrun /test/mnist.py diff --git a/test/pytorch_mnist_image/mnist.py b/test/pytorch_mnist_image/mnist.py deleted file mode 100644 index 134d56188..000000000 --- a/test/pytorch_mnist_image/mnist.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2022 IBM, Red Hat -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import torch -import requests -from pytorch_lightning import LightningModule, Trainer -from pytorch_lightning.callbacks.progress import TQDMProgressBar -from torch import nn -from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split -from torchmetrics import Accuracy -from torchvision import transforms -from torchvision.datasets import MNIST - -PATH_WORKDIR = os.environ.get("PATH_WORKDIR", ".") -PATH_DATASETS = os.environ.get("PATH_DATASETS", "/test") -BATCH_SIZE = 256 if torch.cuda.is_available() else 64 -# %% - -print("prior to running the trainer") -print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) -print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) - -class LitMNIST(LightningModule): - def __init__(self, data_dir=PATH_WORKDIR, hidden_size=64, learning_rate=2e-4): - - super().__init__() - - # Set our init args as class attributes - self.data_dir = data_dir - self.hidden_size = hidden_size - self.learning_rate = learning_rate - - # Hardcode some dataset specific attributes - self.num_classes = 10 - self.dims = (1, 28, 28) - channels, width, height = self.dims - self.transform = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,)), - ] - ) - - # Define PyTorch model - self.model = nn.Sequential( - nn.Flatten(), - nn.Linear(channels * width * height, hidden_size), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(hidden_size, hidden_size), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(hidden_size, self.num_classes), - ) - - self.val_accuracy = Accuracy() - self.test_accuracy = Accuracy() - - def forward(self, x): - x = self.model(x) - return F.log_softmax(x, dim=1) - - def training_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - return loss - - def validation_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - preds = torch.argmax(logits, dim=1) - self.val_accuracy.update(preds, y) - - # Calling self.log will surface up scalars for you in TensorBoard - self.log("val_loss", loss, prog_bar=True) - self.log("val_acc", self.val_accuracy, prog_bar=True) - - def test_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - preds = torch.argmax(logits, dim=1) - self.test_accuracy.update(preds, y) - - # Calling self.log will surface up scalars for you in TensorBoard - self.log("test_loss", loss, prog_bar=True) - self.log("test_acc", self.test_accuracy, prog_bar=True) - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) - return optimizer - - #################### - # DATA RELATED HOOKS - #################### - - def prepare_data(self): - MNIST(PATH_DATASETS, train=True, download=True) - MNIST(PATH_DATASETS, train=False, download=True) - - def setup(self, stage=None): - - # Assign train/val datasets for use in dataloaders - if stage == "fit" or stage is None: - mnist_full = MNIST(PATH_DATASETS, train=True, transform=self.transform) - self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) - - # Assign test dataset for use in dataloader(s) - if stage == "test" or stage is None: - self.mnist_test = MNIST( - PATH_DATASETS, train=False, transform=self.transform - ) - - def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) - - def val_dataloader(self): - return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) - - def test_dataloader(self): - return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) - - -# Init DataLoader from MNIST Dataset - -model = LitMNIST() - -print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) -print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) - -# Initialize a trainer -trainer = Trainer( - accelerator="auto", - # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=5, - callbacks=[TQDMProgressBar(refresh_rate=20)], - num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), - devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), - strategy="ddp", -) - -# Train the model ⚡ -trainer.fit(model) diff --git a/test/pytorch_mnist_image/mnist_pip_requirements.txt b/test/pytorch_mnist_image/mnist_pip_requirements.txt deleted file mode 100644 index 87edeef27..000000000 --- a/test/pytorch_mnist_image/mnist_pip_requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytorch_lightning==1.5.10 -torchmetrics==0.9.1 -torchvision==0.12.0 From 7ed1749c206f86011f93753b06df644e78b99760 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 3 Jun 2024 13:10:13 +0200 Subject: [PATCH 247/369] Delete Pytorch MNIST sample image GitHub workflow --- .github/workflows/mnist-job-test-image.yml | 36 ---------------------- 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/mnist-job-test-image.yml diff --git a/.github/workflows/mnist-job-test-image.yml b/.github/workflows/mnist-job-test-image.yml deleted file mode 100644 index 02c4b5c7e..000000000 --- a/.github/workflows/mnist-job-test-image.yml +++ /dev/null @@ -1,36 +0,0 @@ -# This workflow will build the MNIST job test image and push it to the project-codeflare image registry - -name: MNIST Job Test Image - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - 'test/pytorch_mnist_image/**' - -jobs: - push: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set Go - uses: actions/setup-go@v5 - with: - go-version-file: './go.mod' - - - name: Login to Quay.io - id: podman-login-quay - run: | - podman login --username ${{ secrets.QUAY_ID }} --password ${{ secrets.QUAY_TOKEN }} quay.io - - - name: Image Build and Push - run: | - make image-mnist-job-test-push - - - name: Logout from Quay.io - if: always() && steps.podman-login-quay.outcome == 'success' - run: | - podman logout quay.io From c31ff4cb8408e7c8d6d4d455ba9c692615e70327 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 6 Jun 2024 12:14:25 +0100 Subject: [PATCH 248/369] CertGeneratorImage to reference registry.redhat.io --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index d78d82564..774676010 100644 --- a/main.go +++ b/main.go @@ -142,7 +142,7 @@ func main() { RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", MTLSEnabled: ptr.To(true), - CertGeneratorImage: "registry.access.redhat.com/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", + CertGeneratorImage: "registry.redhat.io/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", }, AppWrapper: &config.AppWrapperConfiguration{ Enabled: ptr.To(false), From 9d68d623f95119241642022f7446adca9b63a814 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Thu, 6 Jun 2024 13:40:41 +0000 Subject: [PATCH 249/369] Update dependency versions for release v1.4.3 --- README.md | 4 ++-- config/manager/params.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 887fd6718..ad599adcc 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.4.2](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.2) | -| CodeFlare-SDK | [v0.16.2](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.2) | +| CodeFlare Operator | [v1.4.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.3) | +| CodeFlare-SDK | [v0.16.3](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.3) | | AppWrapper | [v0.12.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.12.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2) | diff --git a/config/manager/params.env b/config/manager/params.env index 0824a531b..a8d821de8 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.2 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.3 namespace=opendatahub From 60c4cc52a01abee9e74d1bf707114961f5e87c6c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 12 Jun 2024 11:05:53 +0100 Subject: [PATCH 250/369] Test auto-merge commit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 203b02d17..8525ca93b 100644 --- a/Makefile +++ b/Makefile @@ -292,7 +292,7 @@ $(OPENSHIFT-GOIMPORTS): $(LOCALBIN) OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION) .PHONY: install-operator-sdk -install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome +install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome. $(OPERATOR_SDK): $(LOCALBIN) curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output $(LOCALBIN)/operator-sdk chmod +x $(OPERATOR_SDK) From 0cdfd7c3854d6f64466ef34e25e875b5bf59433a Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 10 Jun 2024 20:17:10 -0400 Subject: [PATCH 251/369] Upgrade to AppWrapper v0.13.0 --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 19 ++++++++++++++++++- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 8525ca93b..96732863b 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller - APPWRAPPER_VERSION ?= v0.12.0 + APPWRAPPER_VERSION ?= v0.13.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 994e183b3..96bf81803 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.12.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.13.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index f8cbd4e6d..5eaf8c019 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -125,7 +125,7 @@ spec: type: object type: array podSets: - description: PodSets contained in the Component + description: DeclaredPodSets for the Component (optional for known PodCreating GVKs) items: description: AppWrapperPodSet describes an homogeneous set of pods properties: @@ -243,10 +243,27 @@ spec: name: description: Name is the name of the Component type: string + podSets: + description: PodSets is the validated PodSets for the Component (either from AppWrapperComponent.DeclaredPodSets or inferred by the controller) + items: + description: AppWrapperPodSet describes an homogeneous set of pods + properties: + path: + description: Path is the path Component.Template to the PodTemplateSpec for this PodSet + type: string + replicas: + description: Replicas is the number of pods in this PodSet + format: int32 + type: integer + required: + - path + type: object + type: array required: - apiVersion - kind - name + - podSets type: object type: array conditions: diff --git a/go.mod b/go.mod index 2fd0a7aa8..daa82682c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.12.0 + github.com/project-codeflare/appwrapper v0.13.0 github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0 github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index 8901492a0..eaf6c3d1d 100644 --- a/go.sum +++ b/go.sum @@ -242,8 +242,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.12.0 h1:QMxryBPK6ir3VK6Qx4NWOA05/s4xU6uDHI/nXmLznvw= -github.com/project-codeflare/appwrapper v0.12.0/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= +github.com/project-codeflare/appwrapper v0.13.0 h1:2Br8BPsdHEstw5x0KKAyEbVQJPIspA0/xqbje1dx9OI= +github.com/project-codeflare/appwrapper v0.13.0/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0 h1:3Vz7D9/TwzrBNujHQZGb4L6UKu3siAWwVP4Bj3ByUrU= github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From 7676ba1110482d7dbb9a453ff084ffd5dfb1983e Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 17 Jun 2024 09:54:51 +0100 Subject: [PATCH 252/369] Updated e2e cert generator image --- config/e2e/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 538b7ad5d..a09450396 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -7,6 +7,6 @@ data: kuberay: rayDashboardOAuthEnabled: false ingressDomain: "kind" - certGeneratorImage: quay.io/project-codeflare/ray:latest-py39-cu118 + certGeneratorImage: quay.io/project-codeflare/ray:2.20.0-py39-cu118 appwrapper: enabled: true From 4fd90f930f68b252fa4dc4abf394888e425a7a4e Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 18 Jun 2024 09:25:18 +0100 Subject: [PATCH 253/369] Updated codeflare-common dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index daa82682c..2a08a96b6 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.13.0 - github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0 + github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 diff --git a/go.sum b/go.sum index eaf6c3d1d..077e3160f 100644 --- a/go.sum +++ b/go.sum @@ -244,8 +244,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.13.0 h1:2Br8BPsdHEstw5x0KKAyEbVQJPIspA0/xqbje1dx9OI= github.com/project-codeflare/appwrapper v0.13.0/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= -github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0 h1:3Vz7D9/TwzrBNujHQZGb4L6UKu3siAWwVP4Bj3ByUrU= -github.com/project-codeflare/codeflare-common v0.0.0-20240528061920-68eadc29b5b0/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= +github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f h1:KCpu3tYn4wqpSW3ce8WcMU3rkNy+0cfFTlG/ttbM2F4= +github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From cbbee9d3a574c246632dad1aad23ae4014f299e2 Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 18 Jun 2024 10:46:43 -0400 Subject: [PATCH 254/369] update to appwrapper v0.13.1 --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 96732863b..05247ed2e 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller - APPWRAPPER_VERSION ?= v0.13.0 + APPWRAPPER_VERSION ?= v0.13.1 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 96bf81803..a19661fd8 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.13.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.13.1 diff --git a/go.mod b/go.mod index 2a08a96b6..d9726ae6d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.13.0 + github.com/project-codeflare/appwrapper v0.13.1 github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index 077e3160f..0a3469657 100644 --- a/go.sum +++ b/go.sum @@ -242,8 +242,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.13.0 h1:2Br8BPsdHEstw5x0KKAyEbVQJPIspA0/xqbje1dx9OI= -github.com/project-codeflare/appwrapper v0.13.0/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= +github.com/project-codeflare/appwrapper v0.13.1 h1:2aD6Q2USBrZUtzbSNEVdtmlVkc7bbpwTeqAWoAG0ylo= +github.com/project-codeflare/appwrapper v0.13.1/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f h1:KCpu3tYn4wqpSW3ce8WcMU3rkNy+0cfFTlG/ttbM2F4= github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From 3212bdc93b4ac1ee1476cd9abfbc53d3c3795ca8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 21 Jun 2024 17:34:48 +0000 Subject: [PATCH 255/369] Update dependency versions for release v1.4.4 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ad599adcc..431154655 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.4.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.3) | -| CodeFlare-SDK | [v0.16.3](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.3) | -| AppWrapper | [v0.12.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.12.0) | +| CodeFlare Operator | [v1.4.4](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.4) | +| CodeFlare-SDK | [v0.16.4](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.4) | +| AppWrapper | [v0.13.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.13.1) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2) | diff --git a/config/manager/params.env b/config/manager/params.env index a8d821de8..9360ed048 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.3 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.4 namespace=opendatahub From 5af348af76c290672562494016565c2f156a05f0 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 24 Jun 2024 16:35:47 -0400 Subject: [PATCH 256/369] use correct apiGroup for rayclsuters in rbac annotation for appwrappers. --- config/rbac/role.yaml | 12 ------------ pkg/controllers/appwrapper_controller.go | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 454881679..02693cce0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -99,18 +99,6 @@ rules: - patch - update - watch -- apiGroups: - - cluster.ray.io - resources: - - rayclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - config.openshift.io resources: diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index 11684f5bf..6ce476f94 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -30,7 +30,7 @@ package controllers //+kubebuilder:rbac:groups=scheduling.sigs.k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=scheduling.x-k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=kubeflow.org,resources=pytorchjobs,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=cluster.ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete // permissions needed by Kueue's generic reconciller // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=list;get;watch From 2e8e35fe1e9169970f6e6e541fc1cb196e3c9d34 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 24 Jun 2024 16:59:16 -0400 Subject: [PATCH 257/369] add rayjobs to appwrapper rbacs --- config/rbac/role.yaml | 12 ++++++++++++ pkg/controllers/appwrapper_controller.go | 1 + 2 files changed, 13 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 02693cce0..073b261aa 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -242,6 +242,18 @@ rules: - get - patch - update +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index 6ce476f94..cab8ef375 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -31,6 +31,7 @@ package controllers //+kubebuilder:rbac:groups=scheduling.x-k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=kubeflow.org,resources=pytorchjobs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;create;update;patch;delete // permissions needed by Kueue's generic reconciller // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=list;get;watch From 0f957ede5138e17bf9b1a90c9010035b9e709d83 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 25 Jun 2024 13:15:50 -0400 Subject: [PATCH 258/369] Add example dev workflow to CONTRIBUTING.md Signed-off-by: Kevin --- CONTRIBUTING.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d7bf8d35..f516998df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Here are a few things to go over before getting started with CodeFlare Operator ## Environment setup The following should be installed in your working environment: - - Go 1.20.x + - Go 1.21.x - [Download release](https://go.dev/dl/) - [Install Instructions](https://go.dev/doc/install) - [Operator SDK](https://sdk.operatorframework.io/docs/installation/) @@ -53,3 +53,29 @@ To write and inspect unit tests: - Populate the [.vscode/launch.json](https://github.com/project-codeflare/codeflare-operator/tree/main/.vscode/launch.json) file with the location of your Kubernetes config file and desired namespace. - In VSCode on the activity bar click `Run and Debug` or `CTRL + SHIFT + D` to start a local debugging session of the CodeFlare Operator. The operator should be running as intended. + + +## Example dev workflow + +I've made changes to `pkg/controllers/raycluster_controller.go` and `pkg/controllers/raycluster_controller.go`. I've +written unit tests and would like to test my changes using the unit tests as well as on an OpenShift cluster which I +have access to. + +1. Ensure the unit tests you've written are passing and you haven't introduced any regressions + 1. run `make test-unit` +1. build and push image + 1. `make image-build -e IMG=` + 1. `make image-push -e IMG=` +1. Login to your OpenShift cluster via `oc login --token=... --server=...` +1. deploy ODH/RHOAI if necessary + 1. for the latest releases + 1. run `make delete-all-in-one -e USE_RHOAI=` **if** you would like to run on a fresh ODH/RHOAI deployment + 1. run `make all-in-one -e USE_RHOAI=` to deploy new instance of ODH/RHOAI + 1. otherwise follow the dev guides found in + [ODH](https://github.com/opendatahub-io/opendatahub-operator?tab=readme-ov-file#deployment) and + [RHOAI](https://gitlab.cee.redhat.com/data-hub/olminstall) (Red Hat internal only) + 1. run `make install-nfd-operator`, `make install-service-mesh-operator`, `make install-ai-platform-operator`, `make install-nvidia-operator` to ensure all dependent operators are also installed +1. navigate to the DataScienceCluster resource and set `.spec.components.codeflare.managementState` to `Removed` +1. run `make deploy -e IMG=` + +Your dev image should now be deployed in the cluster From e4ff1607343175f781242177faacc4e141df95c1 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Fri, 14 Jun 2024 12:27:15 +0100 Subject: [PATCH 259/369] Add auto-merge workflow on release and adjust release process --- .../{odh-fork-sync.yml => upstream-downstream-sync.yml} | 8 ++++---- README.md | 8 +++----- 2 files changed, 7 insertions(+), 9 deletions(-) rename .github/workflows/{odh-fork-sync.yml => upstream-downstream-sync.yml} (51%) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/upstream-downstream-sync.yml similarity index 51% rename from .github/workflows/odh-fork-sync.yml rename to .github/workflows/upstream-downstream-sync.yml index 22f42f000..cd78dbab5 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/upstream-downstream-sync.yml @@ -1,4 +1,4 @@ -name: Call sync on OpenDataHub CFO fork sync +name: Upstream to Downstream - Release Auto-Merge on: release: types: @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - sync-fork: + trigger-auto-sync: runs-on: ubuntu-latest steps: - - name: Sync-fork + - name: call-sync-workflow run: | - gh workflow run sync-fork.yaml --repo github.com/opendatahub-io/codeflare-operator --ref main + gh workflow run auto-merge-sync.yaml --repo github.com/red-hat-data-services/codeflare-operator --ref main env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: diff --git a/README.md b/README.md index 431154655..6c9a16a49 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,9 @@ For ODH tests additional environment variables are needed: 2. Once all jobs within the action are completed, verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. 3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. 4. Once PR is merged, announce the new release in slack and mail lists, if any. -5. Release automation should open a PR with changes in [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Review the changes proposed by automation. If all the changes are correct then manually cherrypick all `CARRY` and `PATCH` commits from the current main branch, push the result to a dedicated branch and ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the `main` branch (branch protection has to be temporarily disabled). -6. Build ODH/CFO image by triggering [Build and Push action](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/build-and-push.yaml) -7. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. -8. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. -9. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. +5. Release automation should auto-merge changes to [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Verify the [workflow](https://github.com/project-codeflare/codeflare-operator/actions/workflows/project-codeflare-release.yml) ran successfully and review the new merge-commit and commit history. Same for the [Red Hat CodeFlare Operator repo](https://github.com/red-hat-data-services/codeflare-operator), while also ensuring changes are in the latest `rhoai` release branch. - If the auto-merge fails, conflicts must be resolved and force pushed manually to each downstream repository and release branch. +6. In ODH/CFO verify that the [Build and Push action](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/build-and-push.yaml) was triggered and ran successfully. +7. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. ### Releases involving part of the stack From 869bed07462aada3d28b7c96aa794ce9a61037f7 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 26 Jun 2024 16:19:11 -0400 Subject: [PATCH 260/369] Update to AppWrapper v0.20.1 and Kueue v0.7.0 --- CONTRIBUTING.md | 2 +- Dockerfile | 6 +- Makefile | 12 +- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 54 +++---- go.sum | 183 +++++++++++------------ 6 files changed, 128 insertions(+), 131 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f516998df..03b835204 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Here are a few things to go over before getting started with CodeFlare Operator ## Environment setup The following should be installed in your working environment: - - Go 1.21.x + - Go 1.22.x - [Download release](https://go.dev/dl/) - [Install Instructions](https://go.dev/doc/install) - [Operator SDK](https://sdk.operatorframework.io/docs/installation/) diff --git a/Dockerfile b/Dockerfile index b2bf71dcc..e354480ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Build the manager binary -# BEGIN -- workaround lack of go-toolset for golang 1.21 +# BEGIN -- workaround lack of go-toolset for golang 1.22 -ARG GOLANG_IMAGE=golang:1.21 +ARG GOLANG_IMAGE=golang:1.22 ARG GOARCH=amd64 @@ -20,7 +20,7 @@ RUN dnf upgrade -y && dnf install -y \ ENV PATH=/usr/local/go/bin:$PATH COPY --from=golang /usr/local/go /usr/local/go -# END -- workaround lack of go-toolset for golang 1.21 +# END -- workaround lack of go-toolset for golang 1.22 WORKDIR /workspace # Copy the Go Modules manifests diff --git a/Makefile b/Makefile index 05247ed2e..39e2bbe82 100644 --- a/Makefile +++ b/Makefile @@ -12,13 +12,13 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller - APPWRAPPER_VERSION ?= v0.13.1 - APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper - # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) - APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} +APPWRAPPER_VERSION ?= v0.20.1 +APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper +# Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) +APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} - # KUEUE_VERSION defines the default version of Kueue (used for testing) - KUEUE_VERSION ?= v0.6.2 +# KUEUE_VERSION defines the default version of Kueue (used for testing) +KUEUE_VERSION ?= v0.7.0 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index a19661fd8..1c75d91db 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.13.1 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.20.1 diff --git a/go.mod b/go.mod index d9726ae6d..6d99a5642 100644 --- a/go.mod +++ b/go.mod @@ -1,28 +1,28 @@ module github.com/project-codeflare/codeflare-operator -go 1.21 +go 1.22.2 require ( - github.com/onsi/ginkgo/v2 v2.16.0 - github.com/onsi/gomega v1.31.1 + github.com/onsi/ginkgo/v2 v2.19.0 + github.com/onsi/gomega v1.33.1 github.com/open-policy-agent/cert-controller v0.10.1 github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.13.1 + github.com/project-codeflare/appwrapper v0.20.1 github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f - github.com/ray-project/kuberay/ray-operator v1.1.0 - go.uber.org/zap v1.26.0 + github.com/ray-project/kuberay/ray-operator v1.1.1 + go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - k8s.io/api v0.29.2 + k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.29.2 - k8s.io/apimachinery v0.29.2 + k8s.io/apimachinery v0.30.1 k8s.io/client-go v11.0.0+incompatible - k8s.io/component-base v0.29.2 - k8s.io/klog/v2 v2.110.1 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b - sigs.k8s.io/controller-runtime v0.17.0 - sigs.k8s.io/kueue v0.6.2 + k8s.io/component-base v0.29.5 + k8s.io/klog/v2 v2.120.1 + k8s.io/utils v0.0.0-20240102154912-e7106e64919e + sigs.k8s.io/controller-runtime v0.17.3 + sigs.k8s.io/kueue v0.7.0 sigs.k8s.io/yaml v1.4.0 ) @@ -46,22 +46,22 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect @@ -78,7 +78,7 @@ require ( github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -86,22 +86,22 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.17.0 // indirect + golang.org/x/tools v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.29.2 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - sigs.k8s.io/jobset v0.3.1 // indirect + k8s.io/apiserver v0.29.5 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + sigs.k8s.io/jobset v0.5.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 0a3469657..3fb2fb656 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,8 @@ github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCv github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -61,9 +61,8 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -85,8 +84,8 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -108,8 +107,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -127,11 +126,11 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= -github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= @@ -179,10 +178,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeflow/common v0.4.7 h1:zz6QS4k2u2FY838M/FjOtwjJq39MRZVZcvPahRYL97M= -github.com/kubeflow/common v0.4.7/go.mod h1:43MAof/uhpJA2C0urynqatE3oKFQc7m2HLmJty7waqY= -github.com/kubeflow/mpi-operator v0.4.0 h1:PS4jLoMuRyrk/DHuYkI0D46sQQYpQt375HjOV4KVMFs= -github.com/kubeflow/mpi-operator v0.4.0/go.mod h1:/A4mTy/RYh2UIgaGUiXUaW70eThjsogu80WbbcZpuMg= +github.com/kubeflow/mpi-operator v0.5.0 h1:XvBwyXXQ9103DNMa22sxsaQlaktvaT2LY/g0UniGn5U= +github.com/kubeflow/mpi-operator v0.5.0/go.mod h1:SeZQJW8KJxSTWD++eQYKRFpoDg1v8yrdC6fjx2/3mG0= github.com/kubeflow/training-operator v1.7.0 h1:Zh61GlOWrlRi4UFOtJeV+/5REo/OndhwQ25KYd0llzc= github.com/kubeflow/training-operator v1.7.0/go.mod h1:BZCLX1h06wY3YSeSZZcGYAqI9/nVi7isVCRkfgZe9nE= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -213,15 +210,15 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= -github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= @@ -242,21 +239,21 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.13.1 h1:2aD6Q2USBrZUtzbSNEVdtmlVkc7bbpwTeqAWoAG0ylo= -github.com/project-codeflare/appwrapper v0.13.1/go.mod h1:sH9j/rXX6WIlZzFXUOuqK5pagASPZNhuCtdFK+3BDkw= +github.com/project-codeflare/appwrapper v0.20.1 h1:xrhsTyjA2OF+ueASk0IUCcqFUSDYcjrycYAL3qAL3l4= +github.com/project-codeflare/appwrapper v0.20.1/go.mod h1:bsqp0mPcj661tOdyFlpF8jrdEB3TA7Pcyl+a+XGy1rY= github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f h1:KCpu3tYn4wqpSW3ce8WcMU3rkNy+0cfFTlG/ttbM2F4= github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/ray-project/kuberay/ray-operator v1.1.0 h1:kCbPZG6SDv11A1F1VR0oG1oCTjrKUuZiMDsyJqmNr3g= -github.com/ray-project/kuberay/ray-operator v1.1.0/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= +github.com/ray-project/kuberay/ray-operator v1.1.1 h1:mVOA1ddS9aAsPvhhHrpf0ZXgTzccIAyTbeYeDqtcfAk= +github.com/ray-project/kuberay/ray-operator v1.1.1/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -281,36 +278,36 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= -go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= -go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= -go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= -go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= -go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.etcd.io/etcd/api/v3 v3.5.11 h1:B54KwXbWDHyD3XYAwprxNzTe7vlhR69LuBgZnMVvS7E= +go.etcd.io/etcd/api/v3 v3.5.11/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.11 h1:bT2xVspdiCj2910T0V+/KHcVKjkUrCZVtk8J2JF2z1A= +go.etcd.io/etcd/client/pkg/v3 v3.5.11/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/v3 v3.5.11 h1:ajWtgoNSZJ1gmS8k+icvPtqsqEav+iUorF7b0qozgUU= +go.etcd.io/etcd/client/v3 v3.5.11/go.mod h1:a6xQUEqFJ8vztO1agJh/KQKOMfFI8og52ZconzcDJwE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -319,14 +316,14 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= @@ -359,8 +356,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= @@ -371,8 +368,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -399,12 +396,12 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -412,8 +409,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -430,8 +427,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -446,17 +443,17 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -496,51 +493,51 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= -k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ= -k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ= +k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.29.5 h1:223C+JkRnGmudEU00GfpX6quDSrzwwP0DuXOYTyUYb0= +k8s.io/apiserver v0.29.5/go.mod h1:zN9xdatz5g7XwL1Xoz9hD4QQON1GN0c+1kV5e/NHejM= k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= -k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= +k8s.io/component-base v0.29.5 h1:Ptj8AzG+p8c2a839XriHwxakDpZH9uvIgYz+o1agjg8= +k8s.io/component-base v0.29.5/go.mod h1:9nBUoPxW/yimISIgAG7sJDrUGJlu7t8HnDafIrOdU8Q= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= k8s.io/kube-aggregator v0.28.3/go.mod h1:5DyLevbRTcWnT1f9b+lB3BfbXC1w7gDa/OtB6kKInCw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/metrics v0.29.1 h1:qutc3aIPMCniMuEApuLaeYX47rdCn8eycVDx7R6wMlQ= -k8s.io/metrics v0.29.1/go.mod h1:JrbV2U71+v7d/9qb90UVKL8r0uJ6Z2Hy4V7mDm05cKs= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/metrics v0.29.5 h1:5AZBM4pMiq+SHi2/RP1OuL90In3f07BrL610zunKoxA= +k8s.io/metrics v0.29.5/go.mod h1:Cx7JPEEbjdDpEQ2A1QyXSBd6z5bxlwaSf9rgFJ/adfQ= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= -sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= -sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= -sigs.k8s.io/jobset v0.3.1 h1:GGUH6CNDgsVOip+oz6uiMYg5Tt50wHTmIgmeaTolQ2g= -sigs.k8s.io/jobset v0.3.1/go.mod h1:x3pw53wKIxq5fFz7/WacV1Exd9fTHa530szVXBdo9OE= +sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= +sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/jobset v0.5.1 h1:hUuPYfxRiIxWvxQmQKyDp+aQnFKBrCKIAdE1iVx9Mgo= +sigs.k8s.io/jobset v0.5.1/go.mod h1:Vg99rj/6OoGvy1uvywGEHOcVLCWWJYkJtisKqdWzcFw= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kueue v0.6.2 h1:wSX6DY/BNCIaza9R8TyhRxwzSrY02EkyxfXXeHbV758= -sigs.k8s.io/kueue v0.6.2/go.mod h1:TQ1lIc+2rdvObH7SszaxdCGTMYbp9XupKnCj0ZZDgEw= +sigs.k8s.io/kueue v0.7.0 h1:Rwg2Ce/0kjZwdov1XdsAFb11QQtOOHx7HCTwFSMd8wc= +sigs.k8s.io/kueue v0.7.0/go.mod h1:tjzIB8Y1vWwBJRWJsXByRIx89PGUq5/mzeLnYHZoFtk= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= From 6bc0bc76b8003c30c13fa129869e40e119916a0e Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 27 Jun 2024 21:29:50 -0400 Subject: [PATCH 261/369] Update to AppWrapper v0.20.2 --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 39e2bbe82..8f9af5740 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.20.1 +APPWRAPPER_VERSION ?= v0.20.2 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 1c75d91db..6a441a927 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.20.1 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.20.2 diff --git a/go.mod b/go.mod index 6d99a5642..26925dbb5 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.20.1 + github.com/project-codeflare/appwrapper v0.20.2 github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 3fb2fb656..8151eaee7 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.20.1 h1:xrhsTyjA2OF+ueASk0IUCcqFUSDYcjrycYAL3qAL3l4= -github.com/project-codeflare/appwrapper v0.20.1/go.mod h1:bsqp0mPcj661tOdyFlpF8jrdEB3TA7Pcyl+a+XGy1rY= +github.com/project-codeflare/appwrapper v0.20.2 h1:0+kLqVZUqSaLf0OaoQQoD3greKr8untX2jjYFAfRQ+U= +github.com/project-codeflare/appwrapper v0.20.2/go.mod h1:mGGvStJ1dVQTDnLdTttuYMem/okn4aVa+k93MDjrAtE= github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f h1:KCpu3tYn4wqpSW3ce8WcMU3rkNy+0cfFTlG/ttbM2F4= github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From 90ba84a962881d1d5a7059ae3feeac9d031c33db Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 28 Jun 2024 10:53:38 +0200 Subject: [PATCH 262/369] Update TLS certificate generation container image --- config/e2e/config.yaml | 2 +- go.mod | 4 +++- go.sum | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index a09450396..474e02deb 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -7,6 +7,6 @@ data: kuberay: rayDashboardOAuthEnabled: false ingressDomain: "kind" - certGeneratorImage: quay.io/project-codeflare/ray:2.20.0-py39-cu118 + certGeneratorImage: quay.io/rhoai/ray:2.23.0-py39-cu121 appwrapper: enabled: true diff --git a/go.mod b/go.mod index 26925dbb5..cb20dbb48 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.20.2 - github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f + github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 @@ -66,6 +66,7 @@ require ( github.com/imdario/mergo v0.3.16 // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kubeflow/training-operator v1.7.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -73,6 +74,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect diff --git a/go.sum b/go.sum index 8151eaee7..675cfcd07 100644 --- a/go.sum +++ b/go.sum @@ -164,6 +164,8 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -200,6 +202,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -241,8 +245,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.20.2 h1:0+kLqVZUqSaLf0OaoQQoD3greKr8untX2jjYFAfRQ+U= github.com/project-codeflare/appwrapper v0.20.2/go.mod h1:mGGvStJ1dVQTDnLdTttuYMem/okn4aVa+k93MDjrAtE= -github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f h1:KCpu3tYn4wqpSW3ce8WcMU3rkNy+0cfFTlG/ttbM2F4= -github.com/project-codeflare/codeflare-common v0.0.0-20240617130731-0c3f3b3c0e5f/go.mod h1:tlPi2e1HZQuf7AAFc7keWdVUNcxV+Gfh6Ss4KAQs1O0= +github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e h1:juFd1dQyioeMxbVE6F0YD25ozm/jiqJE+MpDhu8p22k= +github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From fa186c257f07d8c50f8c0e9408f772b10d6be0d8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 28 Jun 2024 16:05:37 +0200 Subject: [PATCH 263/369] test: Upgrade pytorch_lightning to fix invalid metadata --- test/e2e/mnist_pip_requirements.txt | 2 +- test/e2e/mnist_rayjob_raycluster_test.go | 2 +- test/odh/resources/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/mnist_pip_requirements.txt b/test/e2e/mnist_pip_requirements.txt index 87edeef27..4c9d5fcb8 100644 --- a/test/e2e/mnist_pip_requirements.txt +++ b/test/e2e/mnist_pip_requirements.txt @@ -1,3 +1,3 @@ -pytorch_lightning==1.5.10 +pytorch_lightning==1.9.5 torchmetrics==0.9.1 torchvision==0.12.0 diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index b7f039d91..5bf5e9324 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -313,7 +313,7 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC Entrypoint: "python /home/ray/jobs/mnist.py", RuntimeEnvYAML: ` pip: - - pytorch_lightning==1.5.10 + - pytorch_lightning==1.9.5 - torchmetrics==0.9.1 - torchvision==0.12.0 env_vars: diff --git a/test/odh/resources/requirements.txt b/test/odh/resources/requirements.txt index cf9fb5889..a58df2950 100644 --- a/test/odh/resources/requirements.txt +++ b/test/odh/resources/requirements.txt @@ -1,6 +1,6 @@ {{.PipIndexUrl}} {{.PipTrustedHost}} -pytorch_lightning==1.5.10 +pytorch_lightning==1.9.5 ray_lightning torchmetrics==0.9.1 torchvision==0.12.0 From 077121e00379aff63d110b3b102627a88901fbc8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 28 Jun 2024 16:54:29 +0000 Subject: [PATCH 264/369] Update dependency versions for release v1.5.0 --- README.md | 8 ++++---- config/manager/params.env | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6c9a16a49..5b186e6c0 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.4.4](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.4) | -| CodeFlare-SDK | [v0.16.4](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.4) | -| AppWrapper | [v0.13.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.13.1) | +| CodeFlare Operator | [v1.5.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.5.0) | +| CodeFlare-SDK | [v0.17.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.17.0) | +| AppWrapper | [v0.20.2](https://github.com/project-codeflare/appwrapper/releases/tag/v0.20.2) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | -| Kueue | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2) | +| Kueue | [v0.7.0](https://github.com/opendatahub-io/kueue/releases/tag/v0.7.0) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 9360ed048..f591a8906 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.4 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.5.0 namespace=opendatahub From 4673aeb0fd9fdeadb796868207a62ccc31cc3646 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 16 Apr 2024 16:12:42 -0400 Subject: [PATCH 265/369] add watches for owned resources and CRBs Signed-off-by: Kevin --- config/rbac/role.yaml | 14 ++++ main.go | 42 +++++++++++- pkg/controllers/raycluster_controller.go | 81 +++++++++++++++++------- pkg/controllers/support.go | 14 ++-- 4 files changed, 119 insertions(+), 32 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 073b261aa..7009a4210 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -113,7 +113,9 @@ rules: - create - delete - get + - list - patch + - watch - apiGroups: - "" resources: @@ -122,8 +124,10 @@ rules: - create - delete - get + - list - patch - update + - watch - apiGroups: - "" resources: @@ -132,8 +136,10 @@ rules: - create - delete - get + - list - patch - update + - watch - apiGroups: - dscinitialization.opendatahub.io resources: @@ -204,8 +210,10 @@ rules: - create - delete - get + - list - patch - update + - watch - apiGroups: - networking.k8s.io resources: @@ -214,8 +222,10 @@ rules: - create - delete - get + - list - patch - update + - watch - apiGroups: - ray.io resources: @@ -262,8 +272,10 @@ rules: - create - delete - get + - list - patch - update + - watch - apiGroups: - route.openshift.io resources: @@ -273,8 +285,10 @@ rules: - create - delete - get + - list - patch - update + - watch - apiGroups: - scheduling.k8s.io resources: diff --git a/main.go b/main.go index 774676010..9c0d0437d 100644 --- a/main.go +++ b/main.go @@ -36,10 +36,13 @@ import ( "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -49,12 +52,14 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" + clientcache "k8s.io/client-go/tools/cache" retrywatch "k8s.io/client-go/tools/watch" configv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/klog/v2" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -166,6 +171,38 @@ func main() { kubeConfig.QPS = ptr.Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) setupLog.V(2).Info("REST client", "qps", kubeConfig.QPS, "burst", kubeConfig.Burst) + selector, err := labels.Parse(controllers.RayClusterNameLabel) + exitOnError(err, "unable to parse label selector") + + cacheOpts := cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Label: selector, + }, + &corev1.Service{}: { + Label: selector, + }, + &corev1.ServiceAccount{}: { + Label: selector, + }, + &networkingv1.Ingress{}: { + Label: selector, + }, + &networkingv1.NetworkPolicy{}: { + Label: selector, + }, + &rbacv1.ClusterRoleBinding{}: { + Label: selector, + }, + }, + } + + if isOpenShift(ctx, kubeClient.DiscoveryClient) { + cacheOpts.ByObject[&routev1.Route{}] = cache.ByObject{ + Label: selector, + } + } + mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ @@ -179,6 +216,7 @@ func main() { LeaseDuration: &cfg.LeaderElection.LeaseDuration.Duration, RetryPeriod: &cfg.LeaderElection.RetryPeriod.Duration, RenewDeadline: &cfg.LeaderElection.RenewDeadline.Duration, + Cache: cacheOpts, }) exitOnError(err, "unable to create manager") @@ -451,7 +489,7 @@ func waitForAPI(ctx context.Context, mgr ctrl.Manager, apiName string, action fu // Wait for the API to become available then invoke action setupLog.Info(fmt.Sprintf("API %v not available, setting up retry watcher", apiName)) - retryWatcher, err := retrywatch.NewRetryWatcher(crdList.ResourceVersion, &cache.ListWatch{ + retryWatcher, err := retrywatch.NewRetryWatcher(crdList.ResourceVersion, &clientcache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { return crdClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) }, diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index bf58426bc..d0c3cbabf 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -34,6 +34,7 @@ import ( rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -48,6 +49,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" routev1 "github.com/openshift/api/route/v1" routev1ac "github.com/openshift/client-go/route/applyconfigurations/route/v1" @@ -78,6 +81,8 @@ const ( CAPrivateKeyKey = "ca.key" CACertKey = "ca.crt" + + RayClusterNameLabel = "ray.openshift.ai/cluster-name" ) var ( @@ -88,16 +93,16 @@ var ( // +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes;routes/custom-host,verbs=get;create;update;patch;delete -// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get -// +kubebuilder:rbac:groups=core,resources=services,verbs=get;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;create;update;patch;delete -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes;routes/custom-host,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;patch;delete;get;watch +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create; // +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create; // +kubebuilder:rbac:groups=dscinitialization.opendatahub.io,resources=dscinitializations,verbs=get;list;watch -// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;create;update;patch;delete;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -301,7 +306,7 @@ func crbNameFromCluster(cluster *rayv1.RayCluster) string { func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration { return rbacv1ac.ClusterRoleBinding( crbNameFromCluster(cluster)). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name, "ray.openshift.ai/cluster-namespace": cluster.Namespace}). WithSubjects( rbacv1ac.Subject(). WithKind("ServiceAccount"). @@ -322,14 +327,14 @@ func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string { func desiredServiceAccount(cluster *rayv1.RayCluster) *corev1ac.ServiceAccountApplyConfiguration { return corev1ac.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithAnnotations(map[string]string{ "serviceaccounts.openshift.io/oauth-redirectreference.first": "" + `{"kind":"OAuthRedirectReference","apiVersion":"v1",` + `"reference":{"kind":"Route","name":"` + dashboardNameFromCluster(cluster) + `"}}`, }). WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } @@ -343,7 +348,7 @@ func rayClientNameFromCluster(cluster *rayv1.RayCluster) string { func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfiguration { return routev1ac.Route(dashboardNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithSpec(routev1ac.RouteSpec(). WithTo(routev1ac.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). WithPort(routev1ac.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))). @@ -353,7 +358,7 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfigu ), ). WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } @@ -367,7 +372,7 @@ func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string { func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfiguration { return corev1ac.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithAnnotations(map[string]string{"service.beta.openshift.io/serving-cert-secret-name": oauthServiceTLSSecretName(cluster)}). WithSpec( corev1ac.ServiceSpec(). @@ -381,7 +386,7 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfig WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), ). WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } @@ -397,10 +402,10 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac. cookieSecret := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) return corev1ac.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithStringData(map[string]string{"cookie_secret": cookieSecret}). WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } @@ -410,7 +415,7 @@ func caSecretNameFromCluster(cluster *rayv1.RayCluster) string { func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.SecretApplyConfiguration { return corev1ac.Secret(caSecretNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithData(map[string][]byte{ CAPrivateKeyKey: key, CACertKey: cert, @@ -419,7 +424,8 @@ func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.Secr WithUID(cluster.UID). WithName(cluster.Name). WithKind(cluster.Kind). - WithAPIVersion(cluster.APIVersion)) + WithAPIVersion(cluster.APIVersion). + WithController(true)) } func generateCACertificate() ([]byte, []byte, error) { @@ -466,7 +472,7 @@ func generateCACertificate() ([]byte, []byte, error) { } func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration { return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithSpec(networkingv1ac.NetworkPolicySpec(). WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})). WithIngress( @@ -477,7 +483,7 @@ func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.Netw ), ). WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { @@ -488,7 +494,7 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001))) } return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithSpec(networkingv1ac.NetworkPolicySpec(). WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). WithIngress( @@ -534,7 +540,7 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf ), ). WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } @@ -548,8 +554,35 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } r.CookieSalt = string(b) - return ctrl.NewControllerManagedBy(mgr). + // despite ownership, we need to check for labels because we can't use + controller := ctrl.NewControllerManagedBy(mgr). Named(controllerName). For(&rayv1.RayCluster{}). - Complete(r) + Owns(&corev1.ServiceAccount{}). + Owns(&corev1.Service{}). + Owns(&corev1.Secret{}). + Owns(&networkingv1.Ingress{}). + Owns(&networkingv1.NetworkPolicy{}). + Watches(&rbacv1.ClusterRoleBinding{}, handler.EnqueueRequestsFromMapFunc( + func(c context.Context, o client.Object) []reconcile.Request { + name, ok := o.GetLabels()[RayClusterNameLabel] + if !ok { + return []reconcile.Request{} + } + namespace, ok := o.GetLabels()["ray.openshift.ai/cluster-namespace"] + if !ok { + return []reconcile.Request{} + } + return []reconcile.Request{{ + NamespacedName: client.ObjectKey{ + Name: name, + Namespace: namespace, + }}} + }), + ) + if r.IsOpenShift { + controller.Owns(&routev1.Route{}) + } + + return controller.Complete(r) } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index d208b52d3..24285ff1b 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -20,20 +20,20 @@ func serviceNameFromCluster(cluster *rayv1.RayCluster) string { func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithSpec(routeapply.RouteSpec(). WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)). WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))). WithTLS(routeapply.TLSConfig().WithTermination("passthrough")), ). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithAnnotations(map[string]string{ "nginx.ingress.kubernetes.io/rewrite-target": "/", "nginx.ingress.kubernetes.io/ssl-redirect": "true", @@ -43,7 +43,8 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net WithAPIVersion(cluster.APIVersion). WithKind(cluster.Kind). WithName(cluster.Name). - WithUID(cluster.UID)). + WithUID(cluster.UID). + WithController(true)). WithSpec(networkingv1ac.IngressSpec(). WithIngressClassName("nginx"). WithRules(networkingv1ac.IngressRule(). @@ -68,12 +69,13 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace). - WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithOwnerReferences(v1.OwnerReference(). WithAPIVersion(cluster.APIVersion). WithKind(cluster.Kind). WithName(cluster.Name). - WithUID(cluster.UID)). + WithUID(cluster.UID). + WithController(true)). WithSpec(networkingv1ac.IngressSpec(). WithRules(networkingv1ac.IngressRule(). WithHost(ingressHost). // Full Hostname From 223021f784b635a5c92644f78595bb1c25b8555c Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 20 Jun 2024 12:08:21 +0200 Subject: [PATCH 266/369] [RHOAIENG-9004] Adjust existing test and workflow for GPU testing --- .github/workflows/e2e_tests.yaml | 22 ++++-- Dockerfile | 2 +- config/e2e/patch_resources.yaml | 3 + test/e2e/mnist.py | 10 +-- test/e2e/mnist_pytorch_appwrapper_test.go | 12 +++- test/e2e/mnist_rayjob_raycluster_test.go | 84 ++++++++++++++--------- test/e2e/setup.sh | 6 +- 7 files changed, 92 insertions(+), 47 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index d84ce09c2..23a044fe0 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -27,7 +27,7 @@ concurrency: jobs: kubernetes-e2e: - runs-on: ubuntu-20.04-4core + runs-on: ubuntu-20.04-4core-gpu steps: - name: Checkout code @@ -52,9 +52,15 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup NVidia GPU environment for KinD + uses: ./common/github-actions/nvidia-gpu-setup + - name: Setup and start KinD cluster uses: ./common/github-actions/kind + - name: Install NVidia GPU operator for KinD + uses: ./common/github-actions/nvidia-gpu-operator + - name: Deploy CodeFlare stack id: deploy run: | @@ -62,23 +68,25 @@ jobs: make setup-e2e echo Deploying CodeFlare operator - IMG="${REGISTRY_ADDRESS}"/codeflare-operator - make image-push -e IMG="${IMG}" + IMG=localhost/codeflare-operator:test + make image-build -e IMG="${IMG}" + podman save -o cfo.tar ${IMG} + kind load image-archive cfo.tar --name cluster --verbosity 1000 make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager - name: Run e2e tests run: | - export CODEFLARE_TEST_TIMEOUT_SHORT=1m - export CODEFLARE_TEST_TIMEOUT_MEDIUM=5m - export CODEFLARE_TEST_TIMEOUT_LONG=10m + export CODEFLARE_TEST_TIMEOUT_SHORT=3m + export CODEFLARE_TEST_TIMEOUT_MEDIUM=10m + export CODEFLARE_TEST_TIMEOUT_LONG=20m export CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING=30m export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 60m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' diff --git a/Dockerfile b/Dockerfile index e354480ea..72e842059 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # BEGIN -- workaround lack of go-toolset for golang 1.22 -ARG GOLANG_IMAGE=golang:1.22 +ARG GOLANG_IMAGE=docker.io/library/golang:1.22 ARG GOARCH=amd64 diff --git a/config/e2e/patch_resources.yaml b/config/e2e/patch_resources.yaml index 3da1d5f3a..c3e91ae23 100644 --- a/config/e2e/patch_resources.yaml +++ b/config/e2e/patch_resources.yaml @@ -1,2 +1,5 @@ - op: remove path: /spec/template/spec/containers/0/resources +- op: replace + path: /spec/template/spec/containers/0/imagePullPolicy + value: IfNotPresent diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index 5ac266652..5a89a8b38 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -20,7 +20,7 @@ from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split, RandomSampler +from torch.utils.data import DataLoader, random_split from torchmetrics import Accuracy from torchvision import transforms from torchvision.datasets import MNIST @@ -36,6 +36,9 @@ print("MNIST_DATASET_URL: is ", os.getenv("MNIST_DATASET_URL")) MNIST_DATASET_URL = os.getenv("MNIST_DATASET_URL") +print("ACCELERATOR: is ", os.getenv("ACCELERATOR")) +ACCELERATOR = os.getenv("ACCELERATOR") + class LitMNIST(LightningModule): def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): @@ -158,7 +161,7 @@ def setup(self, stage=None): ) def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE, sampler=RandomSampler(self.mnist_train, num_samples=1000)) + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) def val_dataloader(self): return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) @@ -176,13 +179,12 @@ def test_dataloader(self): # Initialize a trainer trainer = Trainer( - accelerator="auto", + accelerator=ACCELERATOR, # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs max_epochs=3, callbacks=[TQDMProgressBar(refresh_rate=20)], num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), - replace_sampler_ddp=False, strategy="ddp", ) diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go index e918b15b1..94239f57c 100644 --- a/test/e2e/mnist_pytorch_appwrapper_test.go +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -30,10 +30,17 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +func TestMnistPyTorchAppWrapperCpu(t *testing.T) { + runMnistPyTorchAppWrapper(t, "cpu") +} + +func TestMnistPyTorchAppWrapperGpu(t *testing.T) { + runMnistPyTorchAppWrapper(t, "gpu") +} + // Trains the MNIST dataset as a batch Job in an AppWrapper, and asserts successful completion of the training job. -func TestMNISTPyTorchAppWrapper(t *testing.T) { +func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { test := With(t) - test.T().Parallel() // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() @@ -85,6 +92,7 @@ func TestMNISTPyTorchAppWrapper(t *testing.T) { {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, {Name: "PIP_INDEX_URL", Value: GetPipIndexURL()}, {Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()}, + {Name: "ACCELERATOR", Value: accelerator}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 5bf5e9324..0f2490c21 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -18,6 +18,7 @@ package e2e import ( "crypto/tls" + "fmt" "net/http" "net/url" "testing" @@ -36,9 +37,17 @@ import ( // Trains the MNIST dataset as a RayJob, executed by a Ray cluster // directly managed by Kueue, and asserts successful completion of the training job. -func TestMNISTRayJobRayCluster(t *testing.T) { + +func TestMnistRayJobRayClusterCpu(t *testing.T) { + runMnistRayJobRayCluster(t, "cpu", 0) +} + +func TestMnistRayJobRayClusterGpu(t *testing.T) { + runMnistRayJobRayCluster(t, "gpu", 1) +} + +func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int) { test := With(t) - test.T().Parallel() // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() @@ -51,7 +60,7 @@ func TestMNISTRayJobRayCluster(t *testing.T) { test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // Create RayCluster and assign it to the localqueue - rayCluster := constructRayCluster(test, namespace, mnist) + rayCluster := constructRayCluster(test, namespace, mnist, numberOfGpus) AssignToLocalQueue(rayCluster, localQueue) rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) @@ -62,7 +71,7 @@ func TestMNISTRayJobRayCluster(t *testing.T) { Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) // Create RayJob - rayJob := constructRayJob(test, namespace, rayCluster) + rayJob := constructRayJob(test, namespace, rayCluster, accelerator, numberOfGpus) rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) @@ -88,10 +97,17 @@ func TestMNISTRayJobRayCluster(t *testing.T) { To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) } +func TestMnistRayJobRayClusterAppWrapperCpu(t *testing.T) { + runMnistRayJobRayClusterAppWrapper(t, "cpu", 0) +} + +func TestMnistRayJobRayClusterAppWrapperGpu(t *testing.T) { + runMnistRayJobRayClusterAppWrapper(t, "gpu", 1) +} + // Same as TestMNISTRayJobRayCluster, except the RayCluster is wrapped in an AppWrapper -func TestMNISTRayJobRayClusterAppWrapper(t *testing.T) { +func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, numberOfGpus int) { test := With(t) - test.T().Parallel() // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() @@ -104,7 +120,7 @@ func TestMNISTRayJobRayClusterAppWrapper(t *testing.T) { test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // Create RayCluster, wrap in AppWrapper and assign to localqueue - rayCluster := constructRayCluster(test, namespace, mnist) + rayCluster := constructRayCluster(test, namespace, mnist, numberOfGpus) aw := &mcadv1beta2.AppWrapper{ TypeMeta: metav1.TypeMeta{ APIVersion: mcadv1beta2.GroupVersion.String(), @@ -140,7 +156,7 @@ func TestMNISTRayJobRayClusterAppWrapper(t *testing.T) { Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) // Create RayJob - rayJob := constructRayJob(test, namespace, rayCluster) + rayJob := constructRayJob(test, namespace, rayCluster, accelerator, numberOfGpus) rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) @@ -183,7 +199,7 @@ func constructMNISTConfigMap(test Test, namespace *corev1.Namespace) *corev1.Con } } -func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.ConfigMap) *rayv1.RayCluster { +func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.ConfigMap, numberOfGpus int) *rayv1.RayCluster { return &rayv1.RayCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1.GroupVersion.String(), @@ -236,24 +252,6 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf corev1.ResourceMemory: resource.MustParse("2G"), }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "mnist", - MountPath: "/home/ray/jobs", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "mnist", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: mnist.Name, - }, - }, - }, }, }, }, @@ -282,11 +280,31 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("256Mi"), + corev1.ResourceMemory: resource.MustParse("1G"), + "nvidia.com/gpu": resource.MustParse(fmt.Sprint(numberOfGpus)), }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("2G"), + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("4G"), + "nvidia.com/gpu": resource.MustParse(fmt.Sprint(numberOfGpus)), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "mnist", + MountPath: "/home/ray/jobs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "mnist", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: mnist.Name, + }, }, }, }, @@ -299,7 +317,7 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf } } -func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayCluster) *rayv1.RayJob { +func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayCluster, accelerator string, numberOfGpus int) *rayv1.RayJob { return &rayv1.RayJob{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1.GroupVersion.String(), @@ -320,6 +338,7 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `" PIP_INDEX_URL: "` + GetPipIndexURL() + `" PIP_TRUSTED_HOST: "` + GetPipTrustedHost() + `" + ACCELERATOR: "` + accelerator + `" `, ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, @@ -336,6 +355,9 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC }, }, }, + EntrypointNumCpus: 2, + // Using EntrypointNumGpus doesn't seem to work properly on KinD cluster with GPU, EntrypointNumCpus seems reliable + EntrypointNumGpus: float32(numberOfGpus), }, } } diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 8d0bb2eee..a7f442e55 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -84,12 +84,14 @@ metadata: spec: namespaceSelector: {} # match all. resourceGroups: - - coveredResources: ["cpu","memory"] + - coveredResources: ["cpu","memory", "nvidia.com/gpu"] flavors: - name: "default-flavor" resources: - name: "cpu" nominalQuota: 4 - name: "memory" - nominalQuota: "4G" + nominalQuota: "20G" + - name: "nvidia.com/gpu" + nominalQuota: "1" EOF From 024771d439c5dbe769985c6a432afb955424a4ad Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 2 Jul 2024 12:49:02 +0200 Subject: [PATCH 267/369] Skip CPU tests for PR check --- .github/workflows/e2e_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 23a044fe0..284769e25 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -86,7 +86,7 @@ jobs: echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 60m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 60m -v -skip "^Test.*Cpu$" ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' From 1eb260e42e2bef84f9f2e617ef38a86596e59615 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 2 Jul 2024 14:15:51 +0200 Subject: [PATCH 268/369] Upload KinD logs for PR check when Deploy CodeFlare stack step fails --- .github/workflows/e2e_tests.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 284769e25..816536b6c 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -56,6 +56,7 @@ jobs: uses: ./common/github-actions/nvidia-gpu-setup - name: Setup and start KinD cluster + id: kind-install uses: ./common/github-actions/kind - name: Install NVidia GPU operator for KinD @@ -83,7 +84,6 @@ jobs: export CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING=30m export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail go test -timeout 60m -v -skip "^Test.*Cpu$" ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt @@ -92,26 +92,26 @@ jobs: if: always() && steps.deploy.outcome == 'success' run: | echo "Printing CodeFlare operator logs" - kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${TEMP_DIR}/codeflare-operator.log - name: Print Kueue operator logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing Kueue operator logs" KUEUE_CONTROLLER_POD=$(kubectl get pods -n kueue-system | grep kueue-controller | awk '{print $1}') - kubectl logs -n kueue-system --tail -1 ${KUEUE_CONTROLLER_POD} | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kueue.log + kubectl logs -n kueue-system --tail -1 ${KUEUE_CONTROLLER_POD} | tee ${TEMP_DIR}/kueue.log - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing KubeRay operator logs" - kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${TEMP_DIR}/kuberay.log - name: Export all KinD pod logs uses: ./common/github-actions/kind-export-logs - if: always() && steps.deploy.outcome == 'success' + if: always() && steps.kind-install.outcome == 'success' with: - output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} + output-directory: ${TEMP_DIR} - name: Upload logs uses: actions/upload-artifact@v4 @@ -120,7 +120,7 @@ jobs: name: logs retention-days: 10 path: | - ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log + ${{ env.TEMP_DIR }}/**/*.log - name: Post notification about failure to a Slack channel in case of push event if: failure() && github.event_name == 'push' From 677417b8b951c9a210351e65e0492b9c7c7b9c47 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Tue, 2 Jul 2024 11:02:26 -0400 Subject: [PATCH 269/369] Move Anish to emeritus_approvers Signed-off-by: Anish Asthana --- OWNERS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/OWNERS b/OWNERS index 955704b3e..81f43a551 100644 --- a/OWNERS +++ b/OWNERS @@ -1,13 +1,12 @@ approvers: - - anishasthana - astefanutti - ChristianZaccaria - jbusche - kpostoffice - sutaakar - tedhtchang + reviewers: - - anishasthana - astefanutti - Bobbins228 - ChristianZaccaria @@ -17,3 +16,6 @@ reviewers: - kpostoffice - sutaakar - tedhtchang + +emeritus_approvers: + - anishasthana # 2024-07-02 From 3b5a5d2f0c183be25e3c491c6b91b19ca91c7b35 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 10 Jul 2024 16:18:41 -0400 Subject: [PATCH 270/369] print operator configuration in setupLog --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 9c0d0437d..e1adab9f7 100644 --- a/main.go +++ b/main.go @@ -166,6 +166,7 @@ func main() { namespace := namespaceOrDie() exitOnError(loadIntoOrCreate(ctx, kubeClient, namespace, configMapName, cfg), "unable to initialise configuration") + setupLog.Info("Successfully configured operator", "config", *cfg) kubeConfig.Burst = int(ptr.Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst))) kubeConfig.QPS = ptr.Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) From a8b65915d117da2f9d47c72f857766820a284717 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 10 Jul 2024 10:50:49 -0400 Subject: [PATCH 271/369] make oauth image and cert generator configurable via env vars Signed-off-by: Kevin --- .github/workflows/project-codeflare-release.yml | 13 ++++++++++++- config/e2e/config.yaml | 1 - config/e2e/patch_resources.yaml | 5 +++++ main.go | 1 - pkg/config/config.go | 2 -- pkg/controllers/raycluster_webhook.go | 6 +++--- pkg/controllers/raycluster_webhook_test.go | 8 ++++---- pkg/controllers/support.go | 14 ++++++++++++++ 8 files changed, 38 insertions(+), 12 deletions(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 93f7b6e07..d946d5c55 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -98,7 +98,18 @@ jobs: steps: - name: Release CodeFlare operator run: | - gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field kueue-version=${{ github.event.inputs.kueue-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ + --ref ${{ github.ref }} \ + --field is-stable=${{ github.event.inputs.is-stable }} \ + --field version=${{ github.event.inputs.operator-version }} \ + --field replaces=${{ github.event.inputs.replaces }} \ + --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} \ + --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} \ + --field kuberay-version=${{ github.event.inputs.kuberay-version }} \ + --field kueue-version=${{ github.event.inputs.kueue-version }} \ + --field quay-organization=${{ github.event.inputs.quay-organization }} \ + --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} \ + --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 474e02deb..c2efba71d 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -7,6 +7,5 @@ data: kuberay: rayDashboardOAuthEnabled: false ingressDomain: "kind" - certGeneratorImage: quay.io/rhoai/ray:2.23.0-py39-cu121 appwrapper: enabled: true diff --git a/config/e2e/patch_resources.yaml b/config/e2e/patch_resources.yaml index c3e91ae23..345e475d1 100644 --- a/config/e2e/patch_resources.yaml +++ b/config/e2e/patch_resources.yaml @@ -3,3 +3,8 @@ - op: replace path: /spec/template/spec/containers/0/imagePullPolicy value: IfNotPresent +- op: add + path: /spec/template/spec/containers/0/env/- + value: + name: CERT_GENERATOR_IMAGE + value: quay.io/rhoai/ray:2.23.0-py39-cu121 diff --git a/main.go b/main.go index e1adab9f7..77e0c90a0 100644 --- a/main.go +++ b/main.go @@ -147,7 +147,6 @@ func main() { RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", MTLSEnabled: ptr.To(true), - CertGeneratorImage: "registry.redhat.io/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", }, AppWrapper: &config.AppWrapperConfiguration{ Enabled: ptr.To(false), diff --git a/pkg/config/config.go b/pkg/config/config.go index 49749dfec..ddbff27ce 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -50,8 +50,6 @@ type KubeRayConfiguration struct { IngressDomain string `json:"ingressDomain"` MTLSEnabled *bool `json:"mTLSEnabled,omitempty"` - - CertGeneratorImage string `json:"certGeneratorImage"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index dab128115..4e4b259f6 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -231,7 +231,7 @@ func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.Err func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container { return corev1.Container{ Name: oauthProxyContainerName, - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Image: OAuthProxyImage, Ports: []corev1.ContainerPort{ {ContainerPort: 8443, Name: "oauth-proxy"}, }, @@ -349,7 +349,7 @@ func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayCo initContainerHead := corev1.Container{ Name: "create-cert", - Image: config.CertGeneratorImage, + Image: CertGeneratorImage, Command: []string{ "sh", "-c", @@ -363,7 +363,7 @@ func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayCo func rayWorkerInitContainer(config *config.KubeRayConfiguration) corev1.Container { initContainerWorker := corev1.Container{ Name: "create-cert", - Image: config.CertGeneratorImage, + Image: CertGeneratorImage, Command: []string{ "sh", "-c", diff --git a/pkg/controllers/raycluster_webhook_test.go b/pkg/controllers/raycluster_webhook_test.go index 44927309d..d8e4f8c43 100644 --- a/pkg/controllers/raycluster_webhook_test.go +++ b/pkg/controllers/raycluster_webhook_test.go @@ -243,7 +243,7 @@ func TestValidateCreate(t *testing.T) { Containers: []corev1.Container{ { Name: oauthProxyContainerName, - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Image: OAuthProxyImage, Ports: []corev1.ContainerPort{ {ContainerPort: 8443, Name: "oauth-proxy"}, }, @@ -363,7 +363,7 @@ func TestValidateUpdate(t *testing.T) { Containers: []corev1.Container{ { Name: oauthProxyContainerName, - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Image: OAuthProxyImage, Ports: []corev1.ContainerPort{ {ContainerPort: 8443, Name: "oauth-proxy"}, }, @@ -414,7 +414,7 @@ func TestValidateUpdate(t *testing.T) { InitContainers: []corev1.Container{ { Name: "create-cert", - Image: "", + Image: "registry.redhat.io/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", Command: []string{ "sh", "-c", @@ -490,7 +490,7 @@ func TestValidateUpdate(t *testing.T) { InitContainers: []corev1.Container{ { Name: "create-cert", - Image: "", + Image: "registry.redhat.io/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", Command: []string{ "sh", "-c", diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 24285ff1b..61ba32d2a 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -1,6 +1,8 @@ package controllers import ( + "os" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" @@ -14,6 +16,18 @@ import ( routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" ) +var ( + CertGeneratorImage = getEnv("CERT_GENERATOR_IMAGE", "registry.redhat.io/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328") + OAuthProxyImage = getEnv("OAUTH_PROXY_IMAGE", "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366") +) + +func getEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + func serviceNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-head-svc" } From 03d5bee1b09ef6d5dfeac302d510431108020001 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 10 Jul 2024 10:30:12 -0400 Subject: [PATCH 272/369] Update AppWrappers to v0.21.1 Highlights: * Update to Kueue 0.7.1 * Remove ChildAdmissionController only needed by Kueue 0.6 * Flexible configuration of Kueue's GenericJob Reconciller * Correctly handle wrapped resources that use generateName Full Changelog: https://github.com/project-codeflare/appwrapper/compare/v0.20.2...v0.21.1 --- Makefile | 3 +-- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 13 +++++++++---- go.sum | 15 ++++++++------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 8f9af5740..a48170bb4 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,8 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.20.2 +APPWRAPPER_VERSION ?= v0.21.1 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper -# Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version) APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the default version of Kueue (used for testing) diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 6a441a927..4a83e2ee6 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.20.2 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.21.1 diff --git a/go.mod b/go.mod index cb20dbb48..3c32d1d8e 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,11 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.20.2 + github.com/project-codeflare/appwrapper v0.21.1 github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.29.2 k8s.io/apimachinery v0.30.1 @@ -22,7 +22,7 @@ require ( k8s.io/klog/v2 v2.120.1 k8s.io/utils v0.0.0-20240102154912-e7106e64919e sigs.k8s.io/controller-runtime v0.17.3 - sigs.k8s.io/kueue v0.7.0 + sigs.k8s.io/kueue v0.7.1 sigs.k8s.io/yaml v1.4.0 ) @@ -34,13 +34,18 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 +// These replace directives support the backlevel go version required by ODH build +replace github.com/project-codeflare/appwrapper v0.21.1 => github.com/project-codeflare/appwrapper v0.21.2-0.20240712173553-5b007c947b37 + +replace sigs.k8s.io/kueue v0.7.1 => github.com/opendatahub-io/kueue v0.7.0-odh-test + require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect diff --git a/go.sum b/go.sum index 675cfcd07..fa550784a 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,9 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -229,6 +230,8 @@ github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opendatahub-io/kueue v0.7.0-odh-test h1:QZg/iBs/+kbX7zIxktAi9j5yOezEgXq9BCLSvHJ+BIU= +github.com/opendatahub-io/kueue v0.7.0-odh-test/go.mod h1:uTAgWTJVrIkm1FPbp6BnhZhr29zlUcgzb5z1m5rFw9k= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= @@ -243,8 +246,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.20.2 h1:0+kLqVZUqSaLf0OaoQQoD3greKr8untX2jjYFAfRQ+U= -github.com/project-codeflare/appwrapper v0.20.2/go.mod h1:mGGvStJ1dVQTDnLdTttuYMem/okn4aVa+k93MDjrAtE= +github.com/project-codeflare/appwrapper v0.21.2-0.20240712173553-5b007c947b37 h1:x4qdbN98B9gtaU7pseJWABZzwoDawXLC5QMlx0idXxc= +github.com/project-codeflare/appwrapper v0.21.2-0.20240712173553-5b007c947b37/go.mod h1:gKjO+iRtMIdBvIBYmN+VciL9kzWmkfwgk/+24wCLhSM= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e h1:juFd1dQyioeMxbVE6F0YD25ozm/jiqJE+MpDhu8p22k= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -329,8 +332,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -540,8 +543,6 @@ sigs.k8s.io/jobset v0.5.1/go.mod h1:Vg99rj/6OoGvy1uvywGEHOcVLCWWJYkJtisKqdWzcFw= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kueue v0.7.0 h1:Rwg2Ce/0kjZwdov1XdsAFb11QQtOOHx7HCTwFSMd8wc= -sigs.k8s.io/kueue v0.7.0/go.mod h1:tjzIB8Y1vWwBJRWJsXByRIx89PGUq5/mzeLnYHZoFtk= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= From 26a12761e97416db6609d6a1acf899570b971c19 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 11 Jul 2024 10:19:10 +0200 Subject: [PATCH 273/369] Generate AppWrapper name to provide unique workloads --- test/e2e/mnist_pytorch_appwrapper_test.go | 34 ++++++++++++++--------- test/e2e/mnist_rayjob_raycluster_test.go | 20 ++++++++----- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go index 94239f57c..ab196959c 100644 --- a/test/e2e/mnist_pytorch_appwrapper_test.go +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -75,14 +75,20 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { Kind: "Job", }, ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, + GenerateName: "mnist", + Namespace: namespace.Name, }, Spec: batchv1.JobSpec{ Completions: Ptr(int32(1)), Parallelism: Ptr(int32(1)), Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + }, + }, Containers: []corev1.Container{ { Name: "job", @@ -139,9 +145,9 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { Kind: "AppWrapper", }, ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, + GenerateName: "mnist", + Namespace: namespace.Name, + Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, }, Spec: mcadv1beta2.AppWrapperSpec{ Components: []mcadv1beta2.AppWrapperComponent{ @@ -158,16 +164,18 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { unstruct := unstructured.Unstructured{Object: awMap} _, err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.GenerateName) - test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning))) + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.GenerateName) + test.Eventually(AppWrappers(test, namespace), TestTimeoutMedium). + Should(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) test.T().Logf("Waiting for AppWrapper %s/%s to complete", job.Namespace, job.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( - Or( - WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperSucceeded)), - WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperFailed)), + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).Should( + ContainElement( + Or( + WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperSucceeded)), + WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperFailed)), + ), )) } diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 0f2490c21..aae9e1d0c 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -127,9 +127,9 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number Kind: "AppWrapper", }, ObjectMeta: metav1.ObjectMeta{ - Name: rayCluster.Name, - Namespace: namespace.Name, - Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, + GenerateName: rayCluster.Name, + Namespace: namespace.Name, + Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, }, Spec: mcadv1beta2.AppWrapperSpec{ Components: []mcadv1beta2.AppWrapperComponent{ @@ -145,11 +145,11 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number unstruct := unstructured.Unstructured{Object: awMap} _, err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.GenerateName) - test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning))) + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.GenerateName) + test.Eventually(AppWrappers(test, namespace), TestTimeoutMedium). + Should(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) test.T().Logf("Waiting for RayCluster %s/%s to be running", rayCluster.Namespace, rayCluster.Name) test.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). @@ -266,6 +266,12 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + }, + }, Containers: []corev1.Container{ { Name: "ray-worker", From 749b8469a679cec8a716fb501bdfda7512cc58ef Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 18 Jul 2024 17:04:46 +0200 Subject: [PATCH 274/369] Adjust e2e tests to verify deletion of resources --- test/e2e/mnist_pytorch_appwrapper_test.go | 23 ++++++++++--- test/e2e/mnist_rayjob_raycluster_test.go | 40 +++++++++++++---------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go index ab196959c..ab1426588 100644 --- a/test/e2e/mnist_pytorch_appwrapper_test.go +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -145,7 +145,7 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { Kind: "AppWrapper", }, ObjectMeta: metav1.ObjectMeta{ - GenerateName: "mnist", + GenerateName: "mnist-", Namespace: namespace.Name, Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, }, @@ -162,15 +162,17 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { awMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) test.Expect(err).NotTo(HaveOccurred()) unstruct := unstructured.Unstructured{Object: awMap} - _, err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + unstructp, err := test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructp.Object, aw) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.GenerateName) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) - test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.GenerateName) + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) test.Eventually(AppWrappers(test, namespace), TestTimeoutMedium). Should(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) - test.T().Logf("Waiting for AppWrapper %s/%s to complete", job.Namespace, job.Name) + test.T().Logf("Waiting for AppWrapper %s/%s to complete", aw.Namespace, aw.Name) test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).Should( ContainElement( Or( @@ -178,4 +180,15 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperFailed)), ), )) + + // Assert the AppWrapper has completed successfully + test.Expect(AppWrappers(test, namespace)(test)). + To(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperSucceeded)))) + + test.T().Logf("Deleting AppWrapper %s/%s", aw.Namespace, aw.Name) + err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.T().Logf("Waiting for AppWrapper %s/%s to be deleted", aw.Namespace, aw.Name) + test.Eventually(AppWrappers(test, namespace), TestTimeoutShort).Should(BeEmpty()) } diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index aae9e1d0c..b1408ed49 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -81,13 +81,6 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) - // Wait for Ray job id to be available, this value is needed for writing logs in defer - test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutShort). - Should(WithTransform(RayJobId, Not(BeEmpty()))) - - // Retrieving the job logs once it has completed or timed out - defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) - test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) @@ -95,6 +88,15 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) + + WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) + + test.T().Logf("Deleting RayCluster %s/%s", rayCluster.Namespace, rayCluster.Name) + err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Delete(test.Ctx(), rayCluster.Name, metav1.DeleteOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.T().Logf("Waiting for RayCluster %s/%s to be deleted", rayCluster.Namespace, rayCluster.Name) + test.Eventually(RayClusters(test, namespace.Name), TestTimeoutShort).Should(BeEmpty()) } func TestMnistRayJobRayClusterAppWrapperCpu(t *testing.T) { @@ -143,11 +145,13 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number awMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) test.Expect(err).NotTo(HaveOccurred()) unstruct := unstructured.Unstructured{Object: awMap} - _, err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + unstructp, err := test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructp.Object, aw) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.GenerateName) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) - test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.GenerateName) + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) test.Eventually(AppWrappers(test, namespace), TestTimeoutMedium). Should(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) @@ -166,13 +170,6 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) - // Wait for Ray job id to be available, this value is needed for writing logs in defer - test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutShort). - Should(WithTransform(RayJobId, Not(BeEmpty()))) - - // Retrieving the job logs once it has completed or timed out - defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) - test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) @@ -180,6 +177,15 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) + + WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) + + test.T().Logf("Deleting AppWrapper %s/%s", aw.Namespace, aw.Name) + err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.T().Logf("Waiting for AppWrapper %s/%s to be deleted", aw.Namespace, aw.Name) + test.Eventually(AppWrappers(test, namespace), TestTimeoutShort).Should(BeEmpty()) } func constructMNISTConfigMap(test Test, namespace *corev1.Namespace) *corev1.ConfigMap { From 09625927448d579c3b5d97a477f0c317d41c2c89 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 19 Jul 2024 12:54:53 +0200 Subject: [PATCH 275/369] Remove ODH integration tests from CFO repo --- test/odh/environment.go | 49 ------ test/odh/notebook.go | 95 ------------ test/odh/ray_test.go | 129 --------------- test/odh/resources/custom-nb-small.yaml | 165 -------------------- test/odh/resources/mnist.py | 190 ----------------------- test/odh/resources/mnist_mcad_mini.ipynb | 95 ------------ test/odh/resources/mnist_ray_mini.ipynb | 146 ----------------- test/odh/resources/requirements.txt | 6 - test/odh/support.go | 34 ---- test/odh/template.go | 40 ----- 10 files changed, 949 deletions(-) delete mode 100644 test/odh/environment.go delete mode 100644 test/odh/notebook.go delete mode 100644 test/odh/ray_test.go delete mode 100644 test/odh/resources/custom-nb-small.yaml delete mode 100644 test/odh/resources/mnist.py delete mode 100644 test/odh/resources/mnist_mcad_mini.ipynb delete mode 100644 test/odh/resources/mnist_ray_mini.ipynb delete mode 100644 test/odh/resources/requirements.txt delete mode 100644 test/odh/support.go delete mode 100644 test/odh/template.go diff --git a/test/odh/environment.go b/test/odh/environment.go deleted file mode 100644 index 0a087c9d5..000000000 --- a/test/odh/environment.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "os" - - . "github.com/project-codeflare/codeflare-common/support" -) - -const ( - // The environment variable for namespace where ODH is installed to. - odhNamespaceEnvVar = "ODH_NAMESPACE" - // The environment variable for ODH Notebook ImageStream name - notebookImageStreamName = "NOTEBOOK_IMAGE_STREAM_NAME" -) - -func GetOpenDataHubNamespace() string { - return lookupEnvOrDefault(odhNamespaceEnvVar, "opendatahub") -} - -func GetNotebookImageStreamName(t Test) string { - isName, ok := os.LookupEnv(notebookImageStreamName) - if !ok { - t.T().Fatalf("Expected environment variable %s not found, please use this environment variable to specify what ImageStream to use for Notebook.", notebookImageStreamName) - } - return isName -} - -func lookupEnvOrDefault(key, value string) string { - if v, ok := os.LookupEnv(key); ok { - return v - } - return value -} diff --git a/test/odh/notebook.go b/test/odh/notebook.go deleted file mode 100644 index 70bdda82c..000000000 --- a/test/odh/notebook.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "bytes" - - gomega "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/yaml" - - imagev1 "github.com/openshift/api/image/v1" -) - -const recommendedTagAnnotation = "opendatahub.io/workbench-image-recommended" - -var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"} - -type NotebookProps struct { - IngressDomain string - OpenShiftApiUrl string - KubernetesBearerToken string - Namespace string - OpenDataHubNamespace string - ImageStreamName string - ImageStreamTag string - RayImage string - NotebookConfigMapName string - NotebookConfigMapFileName string - NotebookPVC string -} - -func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string) { - // Create PVC for Notebook - notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce) - - // Retrieve ImageStream tag for - is := GetImageStream(test, GetOpenDataHubNamespace(), GetNotebookImageStreamName(test)) - recommendedTagName := getRecommendedImageStreamTag(test, is) - - // Read the Notebook CR from resources and perform replacements for custom values using go template - notebookProps := NotebookProps{ - IngressDomain: GetOpenShiftIngressDomain(test), - OpenShiftApiUrl: GetOpenShiftApiUrl(test), - KubernetesBearerToken: notebookToken, - Namespace: namespace.Name, - OpenDataHubNamespace: GetOpenDataHubNamespace(), - ImageStreamName: GetNotebookImageStreamName(test), - ImageStreamTag: recommendedTagName, - RayImage: GetRayImage(), - NotebookConfigMapName: jupyterNotebookConfigMapName, - NotebookConfigMapFileName: jupyterNotebookConfigMapFileName, - NotebookPVC: notebookPVC.Name, - } - notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml") - test.Expect(err).NotTo(gomega.HaveOccurred()) - - parsedNotebookTemplate := ParseTemplate(test, notebookTemplate, notebookProps) - - // Create Notebook CR - notebookCR := &unstructured.Unstructured{} - err = yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(parsedNotebookTemplate), 8192).Decode(notebookCR) - test.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{}) - test.Expect(err).NotTo(gomega.HaveOccurred()) -} - -func getRecommendedImageStreamTag(test Test, is *imagev1.ImageStream) (tagName string) { - for _, tag := range is.Spec.Tags { - if tag.Annotations[recommendedTagAnnotation] == "true" { - return tag.Name - } - } - test.T().Fatalf("tag with annotation '%s' not found in ImageStream %s", recommendedTagAnnotation, is.Name) - return -} diff --git a/test/odh/ray_test.go b/test/odh/ray_test.go deleted file mode 100644 index 14856919d..000000000 --- a/test/odh/ray_test.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" - - rbacv1 "k8s.io/api/rbac/v1" -) - -func TestRay(t *testing.T) { - test := With(t) - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - jupyterNotebookConfigMapFileName := "mnist_ray_mini.ipynb" - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // MNIST Ray Notebook - jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"), - "mnist.py": readMnistPy(test), - "requirements.txt": readRequirementsTxt(test), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, - }, - } - - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, sa, clusterRole) - token := CreateToken(test, namespace.Name, sa) - - // Create Notebook CR - createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) - - // Make sure the RayCluster is created and running - test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong). - Should( - And( - HaveLen(1), - ContainElement(WithTransform(RayClusterState, Equal(rayv1.Ready))), - ), - ) - - // Make sure the RayCluster finishes and is deleted - test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong). - Should(HaveLen(0)) -} - -func readRequirementsTxt(test Test) []byte { - // Read the requirements.txt from resources and perform replacements for custom values using go template - props := struct { - PipIndexUrl string - PipTrustedHost string - }{ - PipIndexUrl: "--index " + GetPipIndexURL(), - } - - // Provide trusted host only if defined - if len(GetPipTrustedHost()) > 0 { - props.PipTrustedHost = "--trusted-host " + GetPipTrustedHost() - } - - template, err := files.ReadFile("resources/requirements.txt") - test.Expect(err).NotTo(HaveOccurred()) - - return ParseTemplate(test, template, props) -} - -func readMnistPy(test Test) []byte { - // Read the mnist.py from resources and perform replacements for custom values using go template - props := struct { - MnistDatasetURL string - }{ - MnistDatasetURL: GetMnistDatasetURL(), - } - template, err := files.ReadFile("resources/mnist.py") - test.Expect(err).NotTo(HaveOccurred()) - - return ParseTemplate(test, template, props) -} diff --git a/test/odh/resources/custom-nb-small.yaml b/test/odh/resources/custom-nb-small.yaml deleted file mode 100644 index 791a2d98a..000000000 --- a/test/odh/resources/custom-nb-small.yaml +++ /dev/null @@ -1,165 +0,0 @@ -# This template maybe used to spin up a custom notebook image -# i.e.: sed s/{{.IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f - -# resources generated: -# pod/jupyter-nb-kube-3aadmin-0 -# service/jupyter-nb-kube-3aadmin -# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com) -# service/jupyter-nb-kube-3aadmin-tls -apiVersion: kubeflow.org/v1 -kind: Notebook -metadata: - annotations: - notebooks.opendatahub.io/inject-oauth: "true" - notebooks.opendatahub.io/last-image-selection: codeflare-notebook:latest - notebooks.opendatahub.io/last-size-selection: Small - notebooks.opendatahub.io/oauth-logout-url: https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home - opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{.Namespace}}.{{.IngressDomain}}/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin - opendatahub.io/username: kube:admin - generation: 1 - labels: - app: jupyter-nb-kube-3aadmin - opendatahub.io/dashboard: "true" - opendatahub.io/odh-managed: "true" - opendatahub.io/user: kube-3aadmin - name: jupyter-nb-kube-3aadmin - namespace: {{.Namespace}} -spec: - template: - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: nvidia.com/gpu.present - operator: NotIn - values: - - "true" - weight: 1 - containers: - - env: - - name: NOTEBOOK_ARGS - value: |- - --ServerApp.port=8888 - --ServerApp.token='' - --ServerApp.password='' - --ServerApp.base_url=/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin - --ServerApp.quit_button=False - --ServerApp.tornado_settings={"user":"kube-3aadmin","hub_host":"https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}","hub_prefix":"/notebookController/kube-3aadmin"} - - name: JUPYTER_IMAGE - value: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} - - name: JUPYTER_NOTEBOOK_PORT - value: "8888" - - name: OCP_SERVER - value: {{.OpenShiftApiUrl}} - - name: OCP_TOKEN - value: {{.KubernetesBearerToken}} - image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} - command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} -p ray_image {{.RayImage}} && sleep infinity"] - # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ] - imagePullPolicy: Always - # livenessProbe: - # failureThreshold: 3 - # httpGet: - # path: /notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin/api - # port: notebook-port - # scheme: HTTP - # initialDelaySeconds: 10 - # periodSeconds: 5 - # successThreshold: 1 - # timeoutSeconds: 1 - name: jupyter-nb-kube-3aadmin - ports: - - containerPort: 8888 - name: notebook-port - protocol: TCP - resources: - limits: - cpu: "2" - memory: 3Gi - requests: - cpu: "1" - memory: 3Gi - volumeMounts: - - mountPath: /opt/app-root/src - name: jupyterhub-nb-kube-3aadmin-pvc - - mountPath: /opt/app-root/notebooks - name: {{.NotebookConfigMapName}} - workingDir: /opt/app-root/src - - args: - - --provider=openshift - - --https-address=:8443 - - --http-address= - - --openshift-service-account=jupyter-nb-kube-3aadmin - - --cookie-secret-file=/etc/oauth/config/cookie_secret - - --cookie-expire=24h0m0s - - --tls-cert=/etc/tls/private/tls.crt - - --tls-key=/etc/tls/private/tls.key - - --upstream=http://localhost:8888 - - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --skip-auth-regex=^(?:/notebook/$(NAMESPACE)/jupyter-nb-kube-3aadmin)?/api$ - - --email-domain=* - - --skip-provider-button - - --openshift-sar={"verb":"get","resource":"notebooks","resourceAPIGroup":"kubeflow.org","resourceName":"jupyter-nb-kube-3aadmin","namespace":"$(NAMESPACE)"} - - --logout-url=https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.redhat.io/openshift4/ose-oauth-proxy:v4.10 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - httpGet: - path: /oauth/healthz - port: oauth-proxy - scheme: HTTPS - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 1 - name: oauth-proxy - ports: - - containerPort: 8443 - name: oauth-proxy - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /oauth/healthz - port: oauth-proxy - scheme: HTTPS - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - cpu: 100m - memory: 64Mi - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - mountPath: /etc/oauth/config - name: oauth-config - - mountPath: /etc/tls/private - name: tls-certificates - enableServiceLinks: false - serviceAccountName: jupyter-nb-kube-3aadmin - volumes: - - name: jupyterhub-nb-kube-3aadmin-pvc - persistentVolumeClaim: - claimName: {{.NotebookPVC}} - - name: oauth-config - secret: - defaultMode: 420 - secretName: jupyter-nb-kube-3aadmin-oauth-config - - name: tls-certificates - secret: - defaultMode: 420 - secretName: jupyter-nb-kube-3aadmin-tls - - name: {{.NotebookConfigMapName}} - configMap: - name: {{.NotebookConfigMapName}} diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py deleted file mode 100644 index 85d420f48..000000000 --- a/test/odh/resources/mnist.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2022 IBM, Red Hat -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import torch -import requests -from pytorch_lightning import LightningModule, Trainer -from pytorch_lightning.callbacks.progress import TQDMProgressBar -from torch import nn -from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split, RandomSampler -from torchmetrics import Accuracy -from torchvision import transforms -from torchvision.datasets import MNIST - -PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") -BATCH_SIZE = 256 if torch.cuda.is_available() else 64 -# %% - -print("prior to running the trainer") -print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) -print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) - -MNIST_DATASET_URL = "{{.MnistDatasetURL}}" -print("MNIST_DATASET_URL: is ", MNIST_DATASET_URL) - -class LitMNIST(LightningModule): - def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): - - super().__init__() - - # Set our init args as class attributes - self.data_dir = data_dir - self.hidden_size = hidden_size - self.learning_rate = learning_rate - - # Hardcode some dataset specific attributes - self.num_classes = 10 - self.dims = (1, 28, 28) - channels, width, height = self.dims - self.transform = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,)), - ] - ) - - # Define PyTorch model - self.model = nn.Sequential( - nn.Flatten(), - nn.Linear(channels * width * height, hidden_size), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(hidden_size, hidden_size), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(hidden_size, self.num_classes), - ) - - self.val_accuracy = Accuracy() - self.test_accuracy = Accuracy() - - def forward(self, x): - x = self.model(x) - return F.log_softmax(x, dim=1) - - def training_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - return loss - - def validation_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - preds = torch.argmax(logits, dim=1) - self.val_accuracy.update(preds, y) - - # Calling self.log will surface up scalars for you in TensorBoard - self.log("val_loss", loss, prog_bar=True) - self.log("val_acc", self.val_accuracy, prog_bar=True) - - def test_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - preds = torch.argmax(logits, dim=1) - self.test_accuracy.update(preds, y) - - # Calling self.log will surface up scalars for you in TensorBoard - self.log("test_loss", loss, prog_bar=True) - self.log("test_acc", self.test_accuracy, prog_bar=True) - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) - return optimizer - - #################### - # DATA RELATED HOOKS - #################### - - def prepare_data(self): - datasetFiles = [ - "t10k-images-idx3-ubyte", - "t10k-labels-idx1-ubyte", - "train-images-idx3-ubyte", - "train-labels-idx1-ubyte" - ] - - # Create required folder structure - downloadLocation = os.path.join(PATH_DATASETS, "MNIST", "raw") - os.makedirs(downloadLocation, exist_ok=True) - print(f"{downloadLocation} folder_path created!") - - for file in datasetFiles: - print(f"Downloading MNIST dataset {file}... to path : {downloadLocation}") - response = requests.get(f"{MNIST_DATASET_URL}{file}", stream=True) - filePath = os.path.join(downloadLocation, file) - - #to download dataset file - try: - if response.status_code == 200: - open(filePath, 'wb').write(response.content) - print(f"{file}: Downloaded and saved zipped file to path - {filePath}") - else: - print(f"Failed to download file {file}") - except Exception as e: - print(e) - print(f"Downloaded MNIST dataset to... {downloadLocation}") - - MNIST(self.data_dir, train=True, download=True) - MNIST(self.data_dir, train=False, download=True) - - def setup(self, stage=None): - - # Assign train/val datasets for use in dataloaders - if stage == "fit" or stage is None: - mnist_full = MNIST(self.data_dir, train=True, transform=self.transform) - self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) - - # Assign test dataset for use in dataloader(s) - if stage == "test" or stage is None: - self.mnist_test = MNIST( - self.data_dir, train=False, transform=self.transform - ) - - def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE, sampler=RandomSampler(self.mnist_train, num_samples=1000)) - - def val_dataloader(self): - return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) - - def test_dataloader(self): - return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) - - -# Init DataLoader from MNIST Dataset - -model = LitMNIST() - -print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) -print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) - -# Initialize a trainer -trainer = Trainer( - accelerator="auto", - # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=3, - callbacks=[TQDMProgressBar(refresh_rate=20)], - num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), - devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), - replace_sampler_ddp=False, - strategy="ddp", -) - -# Train the model ⚡ -trainer.fit(model) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb deleted file mode 100644 index 341eb12ce..000000000 --- a/test/odh/resources/mnist_mcad_mini.ipynb +++ /dev/null @@ -1,95 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk.job.jobs import DDPJobDefinition\n", - "from time import sleep" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47ca5c15", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "#parameters\n", - "namespace = \"default\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26b21373", - "metadata": {}, - "outputs": [], - "source": [ - "job = DDPJobDefinition(name=\"mnistjob\", script=\"/test/mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/demo-images:pytorch-mnist-v0.0.1\").submit()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d24e9f95", - "metadata": {}, - "outputs": [], - "source": [ - "finished = False\n", - "while not finished:\n", - " sleep(1)\n", - " try:\n", - " finished = (\"Epoch 2: 100%\" in job.logs())\n", - " except:\n", - " finished = False" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f078b7cd", - "metadata": {}, - "outputs": [], - "source": [ - "job.cancel()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/test/odh/resources/mnist_ray_mini.ipynb b/test/odh/resources/mnist_ray_mini.ipynb deleted file mode 100644 index 0d8fcc53a..000000000 --- a/test/odh/resources/mnist_ray_mini.ipynb +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration\n", - "from codeflare_sdk.job.jobs import DDPJobDefinition\n", - "from time import sleep" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30888aed", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "#parameters\n", - "namespace = \"default\"\n", - "ray_image = \"has to be specified\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f4bc870-091f-4e11-9642-cba145710159", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Create our cluster and submit appwrapper\n", - "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False, image=ray_image))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Bring up the cluster\n", - "cluster.up()\n", - "cluster.wait_ready()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df71c1ed", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "cluster.status()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "cluster.details()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47ca5c15", - "metadata": {}, - "outputs": [], - "source": [ - "job = DDPJobDefinition(name=\"mnisttest\", script=\"mnist.py\", workspace=\"file:///opt/app-root/notebooks/..data\", scheduler_args={\"requirements\": \"/opt/app-root/notebooks/requirements.txt\"}).submit(cluster)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f63a178a", - "metadata": {}, - "outputs": [], - "source": [ - "finished = False\n", - "while not finished:\n", - " sleep(1)\n", - " status = job.status()\n", - " finished = (str(status.state) == \"SUCCEEDED\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6b099777", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.down()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/test/odh/resources/requirements.txt b/test/odh/resources/requirements.txt deleted file mode 100644 index a58df2950..000000000 --- a/test/odh/resources/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -{{.PipIndexUrl}} -{{.PipTrustedHost}} -pytorch_lightning==1.9.5 -ray_lightning -torchmetrics==0.9.1 -torchvision==0.12.0 diff --git a/test/odh/support.go b/test/odh/support.go deleted file mode 100644 index d828ed950..000000000 --- a/test/odh/support.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "embed" - - "github.com/onsi/gomega" - "github.com/project-codeflare/codeflare-common/support" -) - -//go:embed resources/* -var files embed.FS - -func ReadFile(t support.Test, fileName string) []byte { - t.T().Helper() - file, err := files.ReadFile(fileName) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return file -} diff --git a/test/odh/template.go b/test/odh/template.go deleted file mode 100644 index 3ff4da17f..000000000 --- a/test/odh/template.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "bytes" - "html/template" - - "github.com/onsi/gomega" - "github.com/project-codeflare/codeflare-common/support" -) - -func ParseTemplate(t support.Test, inputTemplate []byte, props interface{}) []byte { - t.T().Helper() - - // Parse input template - parsedTemplate, err := template.New("template").Parse(string(inputTemplate)) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - // Filter template and store results to the buffer - buffer := new(bytes.Buffer) - err = parsedTemplate.Execute(buffer, props) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - return buffer.Bytes() -} From 6ccf23ef6fc5299758ce0172684bd6e1059ecf0a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 23 Jul 2024 09:07:41 +0200 Subject: [PATCH 276/369] Upload KinD pod logs even when CFO deployment fails --- .github/workflows/e2e_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 816536b6c..be3104b22 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -115,7 +115,7 @@ jobs: - name: Upload logs uses: actions/upload-artifact@v4 - if: always() && steps.deploy.outcome == 'success' + if: always() && steps.kind-install.outcome == 'success' with: name: logs retention-days: 10 From bce69097bbf377921506fb77f8732d13c1b77d14 Mon Sep 17 00:00:00 2001 From: Ignas Baranauskas Date: Wed, 24 Jul 2024 11:07:02 +0100 Subject: [PATCH 277/369] Add a check for head pod imagePullSecrets --- pkg/controllers/raycluster_controller.go | 52 +++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index d0c3cbabf..5691a92f4 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -213,6 +213,10 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{RequeueAfter: requeueTime}, err } + if err := r.deleteHeadPodIfMissingImagePullSecrets(ctx, cluster); err != nil { + return ctrl.Result{RequeueAfter: requeueTime}, err + } + _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth ClusterRoleBinding") @@ -470,6 +474,7 @@ func generateCACertificate() ([]byte, []byte, error) { return privateKeyPem, certPem, nil } + func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration { return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace). WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). @@ -486,6 +491,7 @@ func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.Netw metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), ) } + func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{ networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), @@ -544,6 +550,49 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf ) } +func (r *RayClusterReconciler) deleteHeadPodIfMissingImagePullSecrets(ctx context.Context, cluster *rayv1.RayCluster) error { + serviceAccount, err := r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Get(ctx, oauthServiceAccountNameFromCluster(cluster), metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get OAuth ServiceAccount: %w", err) + } + + headPod, err := getHeadPod(ctx, r, cluster) + if err != nil { + return fmt.Errorf("failed to get head pod: %w", err) + } + + if headPod == nil { + return nil + } + + missingSecrets := map[string]bool{} + for _, secret := range serviceAccount.ImagePullSecrets { + missingSecrets[secret.Name] = true + } + for _, secret := range headPod.Spec.ImagePullSecrets { + delete(missingSecrets, secret.Name) + } + if len(missingSecrets) > 0 { + if err := r.kubeClient.CoreV1().Pods(headPod.Namespace).Delete(ctx, headPod.Name, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete head pod: %w", err) + } + } + return nil +} + +func getHeadPod(ctx context.Context, r *RayClusterReconciler, cluster *rayv1.RayCluster) (*corev1.Pod, error) { + podList, err := r.kubeClient.CoreV1().Pods(cluster.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("ray.io/node-type=head,ray.io/cluster=%s", cluster.Name), + }) + if err != nil { + return nil, err + } + if len(podList.Items) > 0 { + return &podList.Items[0], nil + } + return nil, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { r.kubeClient = kubernetes.NewForConfigOrDie(mgr.GetConfig()) @@ -577,7 +626,8 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { NamespacedName: client.ObjectKey{ Name: name, Namespace: namespace, - }}} + }, + }} }), ) if r.IsOpenShift { From 133db9408abef5bee19c000d43bbde049b95ee5d Mon Sep 17 00:00:00 2001 From: Ignas Baranauskas Date: Thu, 25 Jul 2024 11:21:00 +0100 Subject: [PATCH 278/369] Unit test for head pod imagePullSecrets --- pkg/controllers/raycluster_controller_test.go | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go index 94958b334..e0a7e969c 100644 --- a/pkg/controllers/raycluster_controller_test.go +++ b/pkg/controllers/raycluster_controller_test.go @@ -33,7 +33,7 @@ import ( var _ = Describe("RayCluster controller", func() { Context("RayCluster controller test", func() { - var rayClusterName = "test-raycluster" + rayClusterName := "test-raycluster" var namespaceName string BeforeEach(func(ctx SpecContext) { By("Creating a namespace for running the tests.") @@ -145,6 +145,53 @@ var _ = Describe("RayCluster controller", func() { }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) }) + It("should delete the head pod if missing image pull secrets", func(ctx SpecContext) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) + Expect(err).To(Not(HaveOccurred())) + + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + + headPodName := "head-pod" + headPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: headPodName, + Namespace: namespaceName, + Labels: map[string]string{ + "ray.io/node-type": "head", + "ray.io/cluster": foundRayCluster.Name, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "head-container", + Image: "busybox", + }, + }, + }, + } + _, err = k8sClient.CoreV1().Pods(namespaceName).Create(ctx, headPod, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + Eventually(func() (*corev1.Pod, error) { + return k8sClient.CoreV1().Pods(namespaceName).Get(ctx, headPodName, metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + + sa, err := k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + Expect(err).To(Not(HaveOccurred())) + + sa.ImagePullSecrets = append(sa.ImagePullSecrets, corev1.LocalObjectReference{Name: "test-image-pull-secret"}) + _, err = k8sClient.CoreV1().ServiceAccounts(namespaceName).Update(ctx, sa, metav1.UpdateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + Eventually(func() error { + _, err := k8sClient.CoreV1().Pods(namespaceName).Get(ctx, headPodName, metav1.GetOptions{}) + return err + }).WithTimeout(time.Second * 10).Should(Satisfy(errors.IsNotFound)) + }) + It("should remove CRB when the RayCluster is deleted", func(ctx SpecContext) { foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) @@ -157,7 +204,6 @@ var _ = Describe("RayCluster controller", func() { return err }).WithTimeout(time.Second * 10).Should(Satisfy(errors.IsNotFound)) }) - }) }) From 2cd228eda78c6f264e58e934b723563c87d5a785 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 23 Jul 2024 09:32:17 +0200 Subject: [PATCH 279/369] Make sure that negative unit tests use own fork of RayCluster CR --- pkg/controllers/raycluster_webhook_test.go | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/controllers/raycluster_webhook_test.go b/pkg/controllers/raycluster_webhook_test.go index d8e4f8c43..8e0c375a1 100644 --- a/pkg/controllers/raycluster_webhook_test.go +++ b/pkg/controllers/raycluster_webhook_test.go @@ -309,16 +309,15 @@ func TestValidateCreate(t *testing.T) { test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to ValidateCreate function") }) - // Negative Test: Invalid RayCluster with EnableIngress set to true - invalidRayCluster := validRayCluster.DeepCopy() - t.Run("Negative: Expected errors on call to ValidateCreate function due to EnableIngress set to True", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = support.Ptr(true) _, err := rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateCreate function due to EnableIngress set to True") }) t.Run("Negative: Expected errors on call to ValidateCreate function due to manipulated OAuth Proxy Container", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for i, headContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { if headContainer.Name == oauthProxyContainerName { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[i].Args = []string{"--invalid-arg"} @@ -330,6 +329,7 @@ func TestValidateCreate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateCreate function due to manipulated OAuth Proxy Volume", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { if headVolume.Name == oauthProxyVolumeName { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" @@ -341,6 +341,7 @@ func TestValidateCreate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateCreate function due to manipulated head group service account name", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = "invalid-service-account-name" _, err = rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateCreate function due to manipulated head group service account name") @@ -535,17 +536,15 @@ func TestValidateUpdate(t *testing.T) { test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to ValidateUpdate function") }) - // Negative Test Cases - trueBool := true - invalidRayCluster := validRayCluster.DeepCopy() - t.Run("Negative: Expected errors on call to ValidateUpdate function due to EnableIngress set to True", func(t *testing.T) { - invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = &trueBool + invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = support.Ptr(true) _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to EnableIngress set to True") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Container", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for i, headContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { if headContainer.Name == oauthProxyContainerName { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[i].Args = []string{"--invalid-arg"} @@ -557,6 +556,7 @@ func TestValidateUpdate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Volume", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { if headVolume.Name == oauthProxyVolumeName { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" @@ -568,12 +568,14 @@ func TestValidateUpdate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated head group service account name", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = "invalid-service-account-name" _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated head group service account name") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Init Container in the head group", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for i, headInitContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers { if headInitContainer.Name == "create-cert" { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers[i].Command = []string{"manipulated command"} @@ -585,6 +587,7 @@ func TestValidateUpdate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Init Container in the worker group", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { for i, workerInitContainer := range workerGroup.Template.Spec.InitContainers { if workerInitContainer.Name == "create-cert" { @@ -598,6 +601,7 @@ func TestValidateUpdate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Volume in the head group", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { if headVolume.Name == "ca-vol" { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" @@ -609,6 +613,7 @@ func TestValidateUpdate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Volume in the worker group", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { for i, workerVolume := range workerGroup.Template.Spec.Volumes { if workerVolume.Name == "ca-vol" { @@ -622,6 +627,7 @@ func TestValidateUpdate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated env vars in the head group", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for i, headEnvVar := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env { if headEnvVar.Name == "RAY_USE_TLS" { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env[i].Value = "invalid-value" @@ -633,6 +639,7 @@ func TestValidateUpdate(t *testing.T) { }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated env vars in the worker group", func(t *testing.T) { + invalidRayCluster := validRayCluster.DeepCopy() for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { for i, workerEnvVar := range workerGroup.Template.Spec.Containers[0].Env { if workerEnvVar.Name == "RAY_USE_TLS" { From 41aa3638b7f8e272581f991018da85d3a42d68e2 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 19 Jul 2024 17:27:28 -0400 Subject: [PATCH 280/369] Update AppWrappers to v0.22.0 Highlights: + Annotations to mark exit codes as being terminal + Autopilot: auto-injection of Node anti-affinities + Autopilot: Automatic reset/resume of AppWrappers using unhealthy resources + Emit an event when an unhealthy appwrapper is being reset Full Changelog: https://github.com/project-codeflare/appwrapper/compare/v0.21.1...v0.22.0 --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/rbac/role.yaml | 8 ++++++++ go.mod | 4 ++-- go.sum | 4 ++-- pkg/controllers/appwrapper_controller.go | 3 +++ 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a48170bb4..03612e837 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.21.1 +APPWRAPPER_VERSION ?= v0.22.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 4a83e2ee6..d8c3d2e5c 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.21.1 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.22.0 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7009a4210..879d6bc80 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -14,6 +14,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index 3c32d1d8e..a36f1c4b3 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.21.1 + github.com/project-codeflare/appwrapper v0.22.0 github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 @@ -35,7 +35,7 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // These replace directives support the backlevel go version required by ODH build -replace github.com/project-codeflare/appwrapper v0.21.1 => github.com/project-codeflare/appwrapper v0.21.2-0.20240712173553-5b007c947b37 +replace github.com/project-codeflare/appwrapper v0.22.0 => github.com/project-codeflare/appwrapper v0.22.1-0.20240719212005-aab106b2126e replace sigs.k8s.io/kueue v0.7.1 => github.com/opendatahub-io/kueue v0.7.0-odh-test diff --git a/go.sum b/go.sum index fa550784a..33662f832 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.21.2-0.20240712173553-5b007c947b37 h1:x4qdbN98B9gtaU7pseJWABZzwoDawXLC5QMlx0idXxc= -github.com/project-codeflare/appwrapper v0.21.2-0.20240712173553-5b007c947b37/go.mod h1:gKjO+iRtMIdBvIBYmN+VciL9kzWmkfwgk/+24wCLhSM= +github.com/project-codeflare/appwrapper v0.22.1-0.20240719212005-aab106b2126e h1:cIsCTtAZaT2fsQG/QGUm4/wvJnobYawCPZwTwVE2DGo= +github.com/project-codeflare/appwrapper v0.22.1-0.20240719212005-aab106b2126e/go.mod h1:gKjO+iRtMIdBvIBYmN+VciL9kzWmkfwgk/+24wCLhSM= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e h1:juFd1dQyioeMxbVE6F0YD25ozm/jiqJE+MpDhu8p22k= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index cab8ef375..f15a4ef2e 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -41,3 +41,6 @@ package controllers // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/finalizers,verbs=update // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch + +// permission to watch nodes for Autopilot integration +//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch From 365862006fa06e5327178c5bd4625d47ceea91d5 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 26 Jul 2024 16:26:36 +0000 Subject: [PATCH 281/369] Update dependency versions for release v1.6.0 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5b186e6c0..0ce51b59f 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.5.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.5.0) | -| CodeFlare-SDK | [v0.17.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.17.0) | -| AppWrapper | [v0.20.2](https://github.com/project-codeflare/appwrapper/releases/tag/v0.20.2) | +| CodeFlare Operator | [v1.6.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.6.0) | +| CodeFlare-SDK | [v0.18.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.18.0) | +| AppWrapper | [v0.22.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.22.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.7.0](https://github.com/opendatahub-io/kueue/releases/tag/v0.7.0) | diff --git a/config/manager/params.env b/config/manager/params.env index f591a8906..351479ef6 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.5.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.6.0 namespace=opendatahub From c6e4ff09733ea5ec9370c7036bf392e1ae9aebdb Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 31 Jul 2024 12:25:16 -0400 Subject: [PATCH 282/369] Update AppWrapper to v0.23.0 Highlights: + Optional dynamic adjustment of slack quota + Distinguish NoSchedule and NoExecute Autopilot labels Full Changelog: https://github.com/project-codeflare/appwrapper/compare/v0.22.0...v0.23.0 --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 4 ++-- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 03612e837..927abec60 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.22.0 +APPWRAPPER_VERSION ?= v0.23.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index d8c3d2e5c..b0e9570f9 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.22.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.23.0 diff --git a/go.mod b/go.mod index a36f1c4b3..08c87aa4d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.22.0 + github.com/project-codeflare/appwrapper v0.23.0 github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 @@ -35,7 +35,7 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // These replace directives support the backlevel go version required by ODH build -replace github.com/project-codeflare/appwrapper v0.22.0 => github.com/project-codeflare/appwrapper v0.22.1-0.20240719212005-aab106b2126e +replace github.com/project-codeflare/appwrapper v0.23.0 => github.com/project-codeflare/appwrapper v0.23.1-0.20240731154950-ad486fb7e7ee replace sigs.k8s.io/kueue v0.7.1 => github.com/opendatahub-io/kueue v0.7.0-odh-test diff --git a/go.sum b/go.sum index 33662f832..7e6ff93bf 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.22.1-0.20240719212005-aab106b2126e h1:cIsCTtAZaT2fsQG/QGUm4/wvJnobYawCPZwTwVE2DGo= -github.com/project-codeflare/appwrapper v0.22.1-0.20240719212005-aab106b2126e/go.mod h1:gKjO+iRtMIdBvIBYmN+VciL9kzWmkfwgk/+24wCLhSM= +github.com/project-codeflare/appwrapper v0.23.1-0.20240731154950-ad486fb7e7ee h1:kl/nuApg9116e1jSmDdUSaNmxG2Q/OpubEqI+YVfizk= +github.com/project-codeflare/appwrapper v0.23.1-0.20240731154950-ad486fb7e7ee/go.mod h1:6PNty9c0IOonWrJ80j6VL4uTijixtJ3OXSdFJBCDZZE= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e h1:juFd1dQyioeMxbVE6F0YD25ozm/jiqJE+MpDhu8p22k= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From ca81c466e8a4d63770a24724364d48676bb38a42 Mon Sep 17 00:00:00 2001 From: Olivier Tardieu Date: Tue, 30 Jul 2024 15:55:40 -0400 Subject: [PATCH 283/369] Downgrade StatusReasonConflict errors to debug messages --- go.mod | 2 +- main.go | 2 +- pkg/controllers/support.go | 40 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 08c87aa4d..c4ccc596c 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/project-codeflare/codeflare-operator go 1.22.2 require ( + github.com/go-logr/logr v1.4.2 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/open-policy-agent/cert-controller v0.10.1 @@ -51,7 +52,6 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect diff --git a/main.go b/main.go index 77e0c90a0..b0b2d0c5f 100644 --- a/main.go +++ b/main.go @@ -116,7 +116,7 @@ func main() { zapOptions.BindFlags(flag.CommandLine) flag.Parse() - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) + ctrl.SetLogger(controllers.FilteredLogger(zap.New(zap.UseFlagOptions(&zapOptions)))) klog.SetLogger(ctrl.Log) setupLog.Info("Build info", diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 61ba32d2a..f3f5fbec6 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -3,11 +3,13 @@ package controllers import ( "os" + "github.com/go-logr/logr" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation/field" v1 "k8s.io/client-go/applyconfigurations/meta/v1" @@ -172,3 +174,41 @@ func withEnvVarName(name string) compare[corev1.EnvVar] { return e1.Name == name } } + +// logSink implements a log sink with an error log filter +type logSink struct { + sink logr.LogSink +} + +func (l logSink) Init(info logr.RuntimeInfo) { + l.sink.Init(info) +} + +func (l logSink) Enabled(level int) bool { + return l.sink.Enabled(level) +} +func (l logSink) Info(level int, msg string, keysAndValues ...any) { + l.sink.Info(level, msg, keysAndValues...) +} + +func (l logSink) Error(err error, msg string, keysAndValues ...any) { + // downgrade StatusReasonConflict errors to debug messages + if errors.IsConflict(err) { + l.sink.Info(1, msg, append(keysAndValues, "error", err.Error())...) + } else { + l.sink.Error(err, msg, keysAndValues...) + } +} + +func (l logSink) WithValues(keysAndValues ...any) logr.LogSink { + return logSink{l.sink.WithValues(keysAndValues...)} +} + +func (l logSink) WithName(name string) logr.LogSink { + return logSink{l.sink.WithName(name)} +} + +// FilteredLogger returns a copy of the logger with an error log filter +func FilteredLogger(logger logr.Logger) logr.Logger { + return logger.WithSink(logSink{logger.GetSink()}) +} From 280b7662e9c56902ba557e40f08e7e6a036e937d Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 6 Aug 2024 14:33:57 -0400 Subject: [PATCH 284/369] keepin it DRY with a helper function 4 owner ref Signed-off-by: Kevin --- pkg/controllers/raycluster_controller.go | 31 ++++++------------------ pkg/controllers/support.go | 27 +++++++++------------ 2 files changed, 19 insertions(+), 39 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 5691a92f4..ba1cb17c4 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -337,9 +337,7 @@ func desiredServiceAccount(cluster *rayv1.RayCluster) *corev1ac.ServiceAccountAp `{"kind":"OAuthRedirectReference","apiVersion":"v1",` + `"reference":{"kind":"Route","name":"` + dashboardNameFromCluster(cluster) + `"}}`, }). - WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), - ) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func dashboardNameFromCluster(cluster *rayv1.RayCluster) string { @@ -361,9 +359,7 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfigu WithTermination(routev1.TLSTerminationReencrypt), ), ). - WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), - ) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string { @@ -389,9 +385,7 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfig ). WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), ). - WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), - ) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string { @@ -408,9 +402,7 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac. return corev1ac.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithStringData(map[string]string{"cookie_secret": cookieSecret}). - WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), - ) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func caSecretNameFromCluster(cluster *rayv1.RayCluster) string { @@ -424,12 +416,7 @@ func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.Secr CAPrivateKeyKey: key, CACertKey: cert, }). - WithOwnerReferences(metav1ac.OwnerReference(). - WithUID(cluster.UID). - WithName(cluster.Name). - WithKind(cluster.Kind). - WithAPIVersion(cluster.APIVersion). - WithController(true)) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func generateCACertificate() ([]byte, []byte, error) { @@ -487,9 +474,7 @@ func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.Netw ), ), ). - WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), - ) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { @@ -545,9 +530,7 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf ), ), ). - WithOwnerReferences( - metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), - ) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func (r *RayClusterReconciler) deleteHeadPodIfMissingImagePullSecrets(ctx context.Context, cluster *rayv1.RayCluster) error { diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index f3f5fbec6..b2fec8b8c 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -42,9 +42,7 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))). WithTLS(routeapply.TLSConfig().WithTermination("passthrough")), ). - WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion).WithController(true), - ) + WithOwnerReferences(ownerRefForRayCluster(cluster)) } func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { @@ -55,12 +53,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net "nginx.ingress.kubernetes.io/ssl-redirect": "true", "nginx.ingress.kubernetes.io/ssl-passthrough": "true", }). - WithOwnerReferences(v1.OwnerReference(). - WithAPIVersion(cluster.APIVersion). - WithKind(cluster.Kind). - WithName(cluster.Name). - WithUID(cluster.UID). - WithController(true)). + WithOwnerReferences(ownerRefForRayCluster(cluster)). WithSpec(networkingv1ac.IngressSpec(). WithIngressClassName("nginx"). WithRules(networkingv1ac.IngressRule(). @@ -86,12 +79,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). - WithOwnerReferences(v1.OwnerReference(). - WithAPIVersion(cluster.APIVersion). - WithKind(cluster.Kind). - WithName(cluster.Name). - WithUID(cluster.UID). - WithController(true)). + WithOwnerReferences(ownerRefForRayCluster(cluster)). WithSpec(networkingv1ac.IngressSpec(). WithRules(networkingv1ac.IngressRule(). WithHost(ingressHost). // Full Hostname @@ -212,3 +200,12 @@ func (l logSink) WithName(name string) logr.LogSink { func FilteredLogger(logger logr.Logger) logr.Logger { return logger.WithSink(logSink{logger.GetSink()}) } + +func ownerRefForRayCluster(cluster *rayv1.RayCluster) *v1.OwnerReferenceApplyConfiguration { + return v1.OwnerReference(). + WithAPIVersion(cluster.APIVersion). + WithKind(cluster.Kind). + WithName(cluster.Name). + WithUID(cluster.UID). + WithController(true) +} From f31a77d1caa4e75f152ed59f67b6f6562fff3096 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 16 Aug 2024 09:57:08 +0000 Subject: [PATCH 285/369] Update dependency versions for release v1.7.0 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0ce51b59f..d5bf35556 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.6.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.6.0) | -| CodeFlare-SDK | [v0.18.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.18.0) | -| AppWrapper | [v0.22.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.22.0) | +| CodeFlare Operator | [v1.7.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.7.0) | +| CodeFlare-SDK | [v0.19.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.19.1) | +| AppWrapper | [v0.23.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.23.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.7.0](https://github.com/opendatahub-io/kueue/releases/tag/v0.7.0) | diff --git a/config/manager/params.env b/config/manager/params.env index 351479ef6..2efa29e44 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.6.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.7.0 namespace=opendatahub From 68087612f0835a3bb15cd9ab9cd7e14e9f8d150f Mon Sep 17 00:00:00 2001 From: Jiri Petrlik Date: Tue, 27 Aug 2024 10:25:06 +0200 Subject: [PATCH 286/369] Add sleep before creating Kueue resources Fixes following errors. Service for webhook is not available immediately: error appears Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "mresourceflavor.kb.io": failed to call webhook: Post "https://kueue-webhook-service.kueue-system.svc:443/mutate-kueue-x-k8s-io-v1beta1-resourceflavor?timeout=10s": dial tcp 10.96.115.38:443: connect: connection refused --- test/e2e/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index a7f442e55..1ee1d90fc 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -68,6 +68,7 @@ do done echo "" +sleep 5 echo Creating Kueue ResourceFlavor and ClusterQueue cat < Date: Wed, 21 Aug 2024 12:26:13 +0100 Subject: [PATCH 287/369] codflare-operator manifests kustomize v5 upgrade --- Makefile | 2 +- config/default/kustomization.yaml | 15 +++++++-------- config/e2e/kustomization.yaml | 2 +- config/manager/kustomization.yaml | 16 ++++++++++------ config/scorecard/kustomization.yaml | 4 ++-- 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 927abec60..8c2b4ba65 100644 --- a/Makefile +++ b/Makefile @@ -240,7 +240,7 @@ GH_CLI ?= $(LOCALBIN)/gh SED ?= /usr/bin/sed ## Tool Versions -KUSTOMIZE_VERSION ?= v4.5.4 +KUSTOMIZE_VERSION ?= v5.4.3 CODEGEN_VERSION ?= v0.27.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 YQ_VERSION ?= v4.35.2 ## latest version that works with go1.20 diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index e4f0687af..74cfe3139 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -9,20 +9,19 @@ namespace: openshift-operators namePrefix: codeflare-operator- # Labels to add to all resources and selectors. -commonLabels: - app.kubernetes.io/name: codeflare-operator - app.kubernetes.io/part-of: codeflare - -bases: +labels: + - pairs: + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare + includeSelectors: true +resources: - ../crd - ../rbac - ../manager - ../webhook + - metrics_service.yaml # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus -resources: - - metrics_service.yaml - patches: - path: manager_webhook_patch.yaml diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index 772370da6..6e8cdf4b7 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,6 +1,6 @@ namespace: openshift-operators -bases: +resources: - config.yaml - ../default diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 847eca176..b67faae70 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -11,11 +11,15 @@ configMapGenerator: configurations: - params.yaml -vars: -- name: codeflare_operator_controller_image - objref: +replacements: +- source: kind: ConfigMap name: stack-config - apiVersion: v1 - fieldref: - fieldpath: data.codeflare-operator-controller-image + version: v1 + fieldPath: data.codeflare-operator-controller-image + targets: + - select: + kind: Deployment + name: manager + fieldPaths: + - spec.template.spec.containers.0.image diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml index 50cd2d084..a9a84a85a 100644 --- a/config/scorecard/kustomization.yaml +++ b/config/scorecard/kustomization.yaml @@ -1,6 +1,6 @@ resources: - bases/config.yaml -patchesJson6902: +patches: - path: patches/basic.config.yaml target: group: scorecard.operatorframework.io @@ -13,4 +13,4 @@ patchesJson6902: version: v1alpha3 kind: Configuration name: config -#+kubebuilder:scaffold:patchesJson6902 +#+kubebuilder:scaffold:patches From 78d633351c77015567a00080a77d420aa8359a6d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 29 Aug 2024 09:22:04 +0200 Subject: [PATCH 288/369] Add varshaprasad96 to OWNERS --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index 81f43a551..14ed5926f 100644 --- a/OWNERS +++ b/OWNERS @@ -5,6 +5,7 @@ approvers: - kpostoffice - sutaakar - tedhtchang + - varshaprasad96 reviewers: - astefanutti @@ -16,6 +17,7 @@ reviewers: - kpostoffice - sutaakar - tedhtchang + - varshaprasad96 emeritus_approvers: - anishasthana # 2024-07-02 From 805752269239ebb548de634b96f7c55412858dd5 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 30 Aug 2024 12:47:35 -0400 Subject: [PATCH 289/369] update appwrappers to v0.24.0 (#611) Features * Upgrade to Kueue 0.8.0 * Use Patch operations to update AppWrapper.Status * Inject AppWrapper finalizer in WebHook Bugs and Regressions * Get dev mode (make run) working again * account for all covered resources when a node is cordoned Other Changes * Logging improvements for development/debugging * Upgrade to Kind v0.24.0 * Unit test improvement * add failing job with retry to end-to-end tests * improve e2e testing of failures * need non-zero reset pause to reliably detect Resetting phase Full Changelog: https://github.com/project-codeflare/appwrapper/compare/v0.23.0...v0.24.0 --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 55 +++++----- go.sum | 128 ++++++++++------------- 4 files changed, 83 insertions(+), 104 deletions(-) diff --git a/Makefile b/Makefile index 8c2b4ba65..a4ff90fa6 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.23.0 +APPWRAPPER_VERSION ?= v0.24.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index b0e9570f9..2c4dd3ef4 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.23.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.24.0 diff --git a/go.mod b/go.mod index c4ccc596c..021816478 100644 --- a/go.mod +++ b/go.mod @@ -10,20 +10,20 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.23.0 + github.com/project-codeflare/appwrapper v0.24.0 github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20231006140011-7918f672742d - k8s.io/api v0.30.1 + golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 + k8s.io/api v0.30.2 k8s.io/apiextensions-apiserver v0.29.2 - k8s.io/apimachinery v0.30.1 + k8s.io/apimachinery v0.30.2 k8s.io/client-go v11.0.0+incompatible - k8s.io/component-base v0.29.5 + k8s.io/component-base v0.29.6 k8s.io/klog/v2 v2.120.1 - k8s.io/utils v0.0.0-20240102154912-e7106e64919e + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.17.3 - sigs.k8s.io/kueue v0.7.1 + sigs.k8s.io/kueue v0.8.0 sigs.k8s.io/yaml v1.4.0 ) @@ -36,26 +36,26 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // These replace directives support the backlevel go version required by ODH build -replace github.com/project-codeflare/appwrapper v0.23.0 => github.com/project-codeflare/appwrapper v0.23.1-0.20240731154950-ad486fb7e7ee +replace github.com/project-codeflare/appwrapper v0.24.0 => github.com/project-codeflare/appwrapper v0.24.1-0.20240828152845-bfaac68eef30 -replace sigs.k8s.io/kueue v0.7.1 => github.com/opendatahub-io/kueue v0.7.0-odh-test +replace sigs.k8s.io/kueue v0.8.0 => github.com/opendatahub-io/kueue v0.7.0-odh-test require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -93,22 +93,21 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.21.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.29.5 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - sigs.k8s.io/jobset v0.5.1 // indirect + k8s.io/apiserver v0.29.6 // indirect + k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect + sigs.k8s.io/jobset v0.5.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 7e6ff93bf..5d4999b87 100644 --- a/go.sum +++ b/go.sum @@ -16,11 +16,11 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -41,13 +41,13 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -70,20 +70,18 @@ github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -174,7 +172,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -246,8 +243,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.23.1-0.20240731154950-ad486fb7e7ee h1:kl/nuApg9116e1jSmDdUSaNmxG2Q/OpubEqI+YVfizk= -github.com/project-codeflare/appwrapper v0.23.1-0.20240731154950-ad486fb7e7ee/go.mod h1:6PNty9c0IOonWrJ80j6VL4uTijixtJ3OXSdFJBCDZZE= +github.com/project-codeflare/appwrapper v0.24.1-0.20240828152845-bfaac68eef30 h1:Yrjh9PSr08CphBREsGwk1fxMv3a79k2WHTp8vdLTdAM= +github.com/project-codeflare/appwrapper v0.24.1-0.20240828152845-bfaac68eef30/go.mod h1:ru5BZAN38Uoxn1XfATbVN0R6i82kFp29BKih4vFOgDo= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e h1:juFd1dQyioeMxbVE6F0YD25ozm/jiqJE+MpDhu8p22k= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -268,23 +265,18 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -292,7 +284,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.11 h1:B54KwXbWDHyD3XYAwprxNzTe7vlhR69LuBgZnMVvS7E= go.etcd.io/etcd/api/v3 v3.5.11/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= go.etcd.io/etcd/client/pkg/v3 v3.5.11 h1:bT2xVspdiCj2910T0V+/KHcVKjkUrCZVtk8J2JF2z1A= @@ -328,12 +319,11 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 h1:Mi0bCswbz+9cXmwFAdxoo5GPFMKONUpua6iUdtQS7lk= +golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -341,7 +331,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -353,7 +342,6 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -362,21 +350,19 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -400,26 +386,23 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -433,9 +416,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -444,8 +426,6 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -473,8 +453,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -500,20 +480,20 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= -k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= +k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= -k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/apiserver v0.29.5 h1:223C+JkRnGmudEU00GfpX6quDSrzwwP0DuXOYTyUYb0= -k8s.io/apiserver v0.29.5/go.mod h1:zN9xdatz5g7XwL1Xoz9hD4QQON1GN0c+1kV5e/NHejM= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.29.6 h1:JxgDbpgahOgqoDOf+zVl2mI+rQcHcLQnK6YhhtsjbNs= +k8s.io/apiserver v0.29.6/go.mod h1:HrQwfPWxhwEa+n8/+5YwSF5yT2WXbeyFjqq6KEXHTX8= k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.29.5 h1:Ptj8AzG+p8c2a839XriHwxakDpZH9uvIgYz+o1agjg8= -k8s.io/component-base v0.29.5/go.mod h1:9nBUoPxW/yimISIgAG7sJDrUGJlu7t8HnDafIrOdU8Q= +k8s.io/component-base v0.29.6 h1:XkVJI67FvBgNb/3kKqvaGKokxUrIR0RrksCPNI+JYCs= +k8s.io/component-base v0.29.6/go.mod h1:kIahZm8aw9lV8Vw17LF89REmeBrv5+QEl3v7HsrmITY= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -526,20 +506,20 @@ k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= k8s.io/kube-aggregator v0.28.3/go.mod h1:5DyLevbRTcWnT1f9b+lB3BfbXC1w7gDa/OtB6kKInCw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/metrics v0.29.5 h1:5AZBM4pMiq+SHi2/RP1OuL90In3f07BrL610zunKoxA= k8s.io/metrics v0.29.5/go.mod h1:Cx7JPEEbjdDpEQ2A1QyXSBd6z5bxlwaSf9rgFJ/adfQ= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= -sigs.k8s.io/jobset v0.5.1 h1:hUuPYfxRiIxWvxQmQKyDp+aQnFKBrCKIAdE1iVx9Mgo= -sigs.k8s.io/jobset v0.5.1/go.mod h1:Vg99rj/6OoGvy1uvywGEHOcVLCWWJYkJtisKqdWzcFw= +sigs.k8s.io/jobset v0.5.2 h1:276q5Pi/ErLYj+GQ0ydEXR6tx3LwBhEzHLQv+k8bYF4= +sigs.k8s.io/jobset v0.5.2/go.mod h1:Vg99rj/6OoGvy1uvywGEHOcVLCWWJYkJtisKqdWzcFw= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= From 7f00118fb291fe1c716845ac36495df9e4345188 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 6 Aug 2024 14:24:23 -0400 Subject: [PATCH 290/369] add hash to end of resource names to avoid name clash also added a version annotation to raycluster for the CFO version Signed-off-by: Kevin --- main.go | 2 +- pkg/controllers/raycluster_controller.go | 60 ++++- pkg/controllers/raycluster_webhook.go | 34 ++- pkg/controllers/raycluster_webhook_test.go | 296 ++++++++++++++++++--- pkg/controllers/support.go | 10 + 5 files changed, 347 insertions(+), 55 deletions(-) diff --git a/main.go b/main.go index b0b2d0c5f..d9e46c221 100644 --- a/main.go +++ b/main.go @@ -248,7 +248,7 @@ func setupRayClusterController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorCo <-certsReady setupLog.Info("Certs ready") - err := controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay) + err := controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay, OperatorVersion) if err != nil { return err } diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index ba1cb17c4..902570bba 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -114,6 +114,12 @@ var ( // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.15.3/pkg/reconcile +func shouldUseOldName(cluster *rayv1.RayCluster) bool { + // hashed name code was added in the same commit as the version annotation + _, ok := cluster.GetAnnotations()[versionAnnotation] + return !ok +} + func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := ctrl.LoggerFrom(ctx) @@ -304,7 +310,10 @@ func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool { } func crbNameFromCluster(cluster *rayv1.RayCluster) string { - return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} + if shouldUseOldName(cluster) { + return cluster.Name + "-" + cluster.Namespace + "-auth" + } + return RCCUniqueName(cluster.Name + "-" + cluster.Namespace + "-auth") } func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration { @@ -326,7 +335,10 @@ func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.Cluster } func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string { - return cluster.Name + "-oauth-proxy" + if shouldUseOldName(cluster) { + return cluster.Name + "-oauth-proxy" + } + return RCCUniqueName(cluster.Name + "-oauth-proxy") } func desiredServiceAccount(cluster *rayv1.RayCluster) *corev1ac.ServiceAccountApplyConfiguration { @@ -363,11 +375,17 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfigu } func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string { - return cluster.Name + "-oauth" + if shouldUseOldName(cluster) { + return cluster.Name + "-oauth" + } + return RCCUniqueName(cluster.Name + "-oauth") } func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string { - return cluster.Name + "-proxy-tls-secret" + if shouldUseOldName(cluster) { + return cluster.Name + "-proxy-tls-secret" + } + return RCCUniqueName(cluster.Name + "-proxy-tls-secret") } func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfiguration { @@ -389,7 +407,10 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfig } func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string { - return cluster.Name + "-oauth-config" + if shouldUseOldName(cluster) { + return cluster.Name + "-oauth-config" + } + return RCCUniqueName(cluster.Name + "-oauth-config") } // desiredOAuthSecret defines the desired OAuth secret object @@ -406,7 +427,10 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac. } func caSecretNameFromCluster(cluster *rayv1.RayCluster) string { - return "ca-secret-" + cluster.Name + if shouldUseOldName(cluster) { + return "ca-secret-" + cluster.Name + } + return RCCUniqueName(cluster.Name + "-ca-secret") } func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.SecretApplyConfiguration { @@ -462,8 +486,17 @@ func generateCACertificate() ([]byte, []byte, error) { return privateKeyPem, certPem, nil } +func workerNWPNameFromCluster(cluster *rayv1.RayCluster) string { + if shouldUseOldName(cluster) { + return cluster.Name + "-workers" + } + return RCCUniqueName(cluster.Name + "-workers") +} + func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration { - return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace). + return networkingv1ac.NetworkPolicy( + workerNWPNameFromCluster(cluster), cluster.Namespace, + ). WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithSpec(networkingv1ac.NetworkPolicySpec(). WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})). @@ -477,6 +510,13 @@ func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.Netw WithOwnerReferences(ownerRefForRayCluster(cluster)) } +func headNWPNameFromCluster(cluster *rayv1.RayCluster) string { + if shouldUseOldName(cluster) { + return cluster.Name + "-head" + } + return RCCUniqueName(cluster.Name + "-head") +} + func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{ networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), @@ -484,7 +524,7 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf if ptr.Deref(cfg.MTLSEnabled, true) { allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001))) } - return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace). + return networkingv1ac.NetworkPolicy(headNWPNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}). WithSpec(networkingv1ac.NetworkPolicySpec(). WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). @@ -619,3 +659,7 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return controller.Complete(r) } + +func RCCUniqueName(s string) string { + return s + "-" + seededHash(controllerName, s) +} diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 4e4b259f6..c630bd5ba 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -38,14 +38,16 @@ const ( oauthProxyContainerName = "oauth-proxy" oauthProxyVolumeName = "proxy-tls-secret" initContainerName = "create-cert" + versionAnnotation = "ray.openshift.ai/version" ) // log is for logging in this package. var rayclusterlog = logf.Log.WithName("raycluster-resource") -func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration) error { +func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration, operatorVersion string) error { rayClusterWebhookInstance := &rayClusterWebhook{ - Config: cfg, + Config: cfg, + OperatorVersion: operatorVersion, } return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). @@ -58,7 +60,8 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf // +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.ray.openshift.ai,admissionReviewVersions=v1 type rayClusterWebhook struct { - Config *config.KubeRayConfiguration + Config *config.KubeRayConfiguration + OperatorVersion string } var _ webhook.CustomDefaulter = &rayClusterWebhook{} @@ -66,15 +69,24 @@ var _ webhook.CustomValidator = &rayClusterWebhook{} // Default implements webhook.Defaulter so a webhook will be registered for the type func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { + logger := ctrl.LoggerFrom(ctx) rayCluster := obj.(*rayv1.RayCluster) + // add annotation to use new names + annotations := rayCluster.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[versionAnnotation] = w.OperatorVersion + rayCluster.SetAnnotations(annotations) + logger.Info("Ray Cluster annotations", "annotations", rayCluster.GetAnnotations()) if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { rayclusterlog.V(2).Info("Adding OAuth sidecar container") rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName)) rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName)) - rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = oauthServiceAccountNameFromCluster(rayCluster) } if ptr.Deref(w.Config.MTLSEnabled, true) { @@ -218,7 +230,7 @@ func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList { func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.ErrorList { var allErrors field.ErrorList - if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != rayCluster.Name+"-oauth-proxy" { + if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != oauthServiceAccountNameFromCluster(rayCluster) { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "headGroupSpec", "template", "spec", "serviceAccountName"), rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName, @@ -241,7 +253,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container { ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ - Name: rayCluster.Name + "-oauth-config", + Name: oauthSecretNameFromCluster(rayCluster), }, Key: "cookie_secret", }, @@ -251,7 +263,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container { Args: []string{ "--https-address=:8443", "--provider=openshift", - "--openshift-service-account=" + rayCluster.Name + "-oauth-proxy", + "--openshift-service-account=" + oauthServiceAccountNameFromCluster(rayCluster), "--upstream=http://localhost:8265", "--tls-cert=/etc/tls/private/tls.crt", "--tls-key=/etc/tls/private/tls.key", @@ -273,7 +285,7 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume { Name: oauthProxyVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: rayCluster.Name + "-proxy-tls-secret", + SecretName: oauthServiceTLSSecretName(rayCluster), }, }, } @@ -329,7 +341,7 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume { Name: "ca-vol", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: `ca-secret-` + rayCluster.Name, + SecretName: caSecretNameFromCluster(rayCluster), }, }, }, @@ -343,9 +355,9 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume { } func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) corev1.Container { - rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + config.IngressDomain + rayClientRoute := rayClientNameFromCluster(rayCluster) + "-" + rayCluster.Namespace + "." + config.IngressDomain // Service name for basic interactive - svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc" + svcDomain := serviceNameFromCluster(rayCluster) + "." + rayCluster.Namespace + ".svc" initContainerHead := corev1.Container{ Name: "create-cert", diff --git a/pkg/controllers/raycluster_webhook_test.go b/pkg/controllers/raycluster_webhook_test.go index 8e0c375a1..d1c798a8b 100644 --- a/pkg/controllers/raycluster_webhook_test.go +++ b/pkg/controllers/raycluster_webhook_test.go @@ -35,7 +35,8 @@ var ( rayClusterName = "test-raycluster" rcWebhook = &rayClusterWebhook{ - Config: &config.KubeRayConfiguration{}, + Config: &config.KubeRayConfiguration{}, + OperatorVersion: "0.0.0", } ) @@ -124,7 +125,7 @@ func TestRayClusterWebhookDefault(t *testing.T) { t.Run("Expected required service account name for the head group", func(t *testing.T) { test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName). - To(Equal(validRayCluster.Name+"-oauth-proxy"), + To(Equal(validRayCluster.Name+"-oauth-proxy-ecaffa62"), "Expected the service account name to be set correctly") }) @@ -230,11 +231,47 @@ func TestRayClusterWebhookDefault(t *testing.T) { func TestValidateCreate(t *testing.T) { test := support.NewTest(t) + minimalRayCluster := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + Namespace: namespace, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "head"}, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + WorkerGroupSpecs: []rayv1.WorkerGroupSpec{ + { + GroupName: "worker-group-1", + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "worker-container-1", + }, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + }, + }, + } validRayCluster := &rayv1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Name: rayClusterName, Namespace: namespace, + Annotations: map[string]string{ + versionAnnotation: "0.0.0", + }, }, Spec: rayv1.RayClusterSpec{ HeadGroupSpec: rayv1.HeadGroupSpec{ @@ -253,7 +290,7 @@ func TestValidateCreate(t *testing.T) { ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ - Name: rayClusterName + "-oauth-config", + Name: rayClusterName + "-oauth-config-01e408a7", }, Key: "cookie_secret", }, @@ -263,7 +300,7 @@ func TestValidateCreate(t *testing.T) { Args: []string{ "--https-address=:8443", "--provider=openshift", - "--openshift-service-account=" + rayClusterName + "-oauth-proxy", + "--openshift-service-account=" + rayClusterName + "-oauth-proxy-ecaffa62", "--upstream=http://localhost:8265", "--tls-cert=/etc/tls/private/tls.crt", "--tls-key=/etc/tls/private/tls.key", @@ -284,12 +321,12 @@ func TestValidateCreate(t *testing.T) { Name: oauthProxyVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: rayClusterName + "-proxy-tls-secret", + SecretName: rayClusterName + "-proxy-tls-secret-3e5a0266", }, }, }, }, - ServiceAccountName: rayClusterName + "-oauth-proxy", + ServiceAccountName: rayClusterName + "-oauth-proxy-ecaffa62", }, }, RayStartParams: map[string]string{}, @@ -298,12 +335,18 @@ func TestValidateCreate(t *testing.T) { } // Create the RayClusters - if _, err := test.Client().Ray().RayV1().RayClusters(namespace).Create(test.Ctx(), validRayCluster, metav1.CreateOptions{}); err != nil { + if _, err := test.Client().Ray().RayV1().RayClusters(namespace).Create(test.Ctx(), minimalRayCluster, metav1.CreateOptions{}); err != nil { test.T().Fatalf("Failed to create RayCluster: %v", err) } + t.Run("Expect updated values in minimal RayCluster", func(t *testing.T) { + err := rcWebhook.Default(test.Ctx(), runtime.Object(minimalRayCluster)) + test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to Default function") + test.Expect(minimalRayCluster.GetAnnotations()[versionAnnotation]).ShouldNot(BeNil(), "Expected version annotation to be set") + }) + // Call to ValidateCreate function is made - warnings, err := rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(validRayCluster)) + warnings, err := rcWebhook.ValidateCreate(test.Ctx(), runtime.Object(minimalRayCluster)) t.Run("Expected no warnings or errors on call to ValidateCreate function", func(t *testing.T) { test.Expect(warnings).Should(BeNil(), "Expected no warnings on call to ValidateCreate function") test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to ValidateCreate function") @@ -351,8 +394,184 @@ func TestValidateCreate(t *testing.T) { func TestValidateUpdate(t *testing.T) { test := support.NewTest(t) + rayClientRoute := "rayclient-" + rayClusterName + "-" + namespace + "." + rcWebhook.Config.IngressDomain + svcDomain := rayClusterName + "-head-svc." + namespace + ".svc" + validRayClusterNewNames := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + Namespace: namespace, + Annotations: map[string]string{ + versionAnnotation: "0.0.0", + }, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: oauthProxyContainerName, + Image: OAuthProxyImage, + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: rayClusterName + "-oauth-config-01e408a7", + }, + Key: "cookie_secret", + }, + }, + }, + { + Name: "MY_POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + {Name: "RAY_USE_TLS", Value: "1"}, + {Name: "RAY_TLS_SERVER_CERT", Value: "/home/ray/workspace/tls/server.crt"}, + {Name: "RAY_TLS_SERVER_KEY", Value: "/home/ray/workspace/tls/server.key"}, + {Name: "RAY_TLS_CA_CERT", Value: "/home/ray/workspace/tls/ca.crt"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + rayClusterName + "-oauth-proxy-ecaffa62", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"" + namespace + "\",\"verb\":\"get\"}}", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: oauthProxyVolumeName, + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "create-cert", + Image: "registry.redhat.io/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "ca-vol", + MountPath: "/home/ray/workspace/ca", + ReadOnly: true, + }, + { + Name: "server-cert", + MountPath: "/home/ray/workspace/tls", + ReadOnly: false, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: oauthProxyVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: rayClusterName + "-proxy-tls-secret-3e5a0266", + }, + }, + }, + { + Name: "ca-vol", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: rayClusterName + "-ca-secret-001432aa", + }, + }, + }, + { + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + ServiceAccountName: rayClusterName + "-oauth-proxy-ecaffa62", + }, + }, + RayStartParams: map[string]string{}, + }, + WorkerGroupSpecs: []rayv1.WorkerGroupSpec{ + { + GroupName: "worker-group-1", + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "worker-container-1", + Env: []corev1.EnvVar{ + { + Name: "MY_POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + {Name: "RAY_USE_TLS", Value: "1"}, + {Name: "RAY_TLS_SERVER_CERT", Value: "/home/ray/workspace/tls/server.crt"}, + {Name: "RAY_TLS_SERVER_KEY", Value: "/home/ray/workspace/tls/server.key"}, + {Name: "RAY_TLS_CA_CERT", Value: "/home/ray/workspace/tls/ca.crt"}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "create-cert", + Image: "registry.redhat.io/ubi9@sha256:770cf07083e1c85ae69c25181a205b7cdef63c11b794c89b3b487d4670b4c328", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: certVolumeMounts(), + }, + }, + Volumes: []corev1.Volume{ + { + Name: "ca-vol", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: rayClusterName + "-ca-secret-001432aa", + }, + }, + }, + { + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + }, + }, + } - validRayCluster := &rayv1.RayCluster{ + validRayClusterOldNames := &rayv1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Name: rayClusterName, Namespace: namespace, @@ -419,7 +638,7 @@ func TestValidateUpdate(t *testing.T) { Command: []string{ "sh", "-c", - `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-` + rayClusterName + `-` + namespace + `.\nDNS.6 = ` + rayClusterName + `-head-svc.` + namespace + `.svc` + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, }, VolumeMounts: []corev1.VolumeMount{ { @@ -448,7 +667,7 @@ func TestValidateUpdate(t *testing.T) { Name: "ca-vol", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: `ca-secret-` + rayClusterName, + SecretName: "ca-secret-" + rayClusterName, }, }, }, @@ -505,7 +724,7 @@ func TestValidateUpdate(t *testing.T) { Name: "ca-vol", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: `ca-secret-` + rayClusterName, + SecretName: "ca-secret-" + rayClusterName, }, }, }, @@ -525,69 +744,76 @@ func TestValidateUpdate(t *testing.T) { } // Create the RayClusters - if _, err := test.Client().Ray().RayV1().RayClusters(namespace).Create(test.Ctx(), validRayCluster, metav1.CreateOptions{}); err != nil { + if _, err := test.Client().Ray().RayV1().RayClusters(namespace).Create(test.Ctx(), validRayClusterNewNames, metav1.CreateOptions{}); err != nil { test.T().Fatalf("Failed to create RayCluster: %v", err) } // Positive Test Case: Valid RayCluster with immutable fields - t.Run("Expected no warnings or errors on call to ValidateUpdate function", func(t *testing.T) { - warnings, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(validRayCluster)) + t.Run("Expected no warnings or errors on call to ValidateUpdate function with version annotation set", func(t *testing.T) { + test.Expect(runtime.Object(validRayClusterNewNames).(*rayv1.RayCluster).Annotations).ShouldNot(BeNil(), "Expected version annotation to be set") + warnings, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(validRayClusterNewNames)) + test.Expect(warnings).Should(BeNil(), "Expected no warnings on call to ValidateUpdate function") + test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to ValidateUpdate function") + }) + + t.Run("Expected no warnings or errors on call to ValidateUpdate function with version annotation unset", func(t *testing.T) { + warnings, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterOldNames), runtime.Object(validRayClusterOldNames)) test.Expect(warnings).Should(BeNil(), "Expected no warnings on call to ValidateUpdate function") test.Expect(err).ShouldNot(HaveOccurred(), "Expected no errors on call to ValidateUpdate function") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to EnableIngress set to True", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() invalidRayCluster.Spec.HeadGroupSpec.EnableIngress = support.Ptr(true) - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to EnableIngress set to True") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Container", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for i, headContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { if headContainer.Name == oauthProxyContainerName { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[i].Args = []string{"--invalid-arg"} break } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Container") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Volume", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { if headVolume.Name == oauthProxyVolumeName { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" break } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated OAuth Proxy Volume") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated head group service account name", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = "invalid-service-account-name" - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated head group service account name") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Init Container in the head group", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for i, headInitContainer := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers { if headInitContainer.Name == "create-cert" { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers[i].Command = []string{"manipulated command"} break } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Init Container in the head group") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Init Container in the worker group", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { for i, workerInitContainer := range workerGroup.Template.Spec.InitContainers { if workerInitContainer.Name == "create-cert" { @@ -596,24 +822,24 @@ func TestValidateUpdate(t *testing.T) { } } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Init Container in the worker group") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Volume in the head group", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for i, headVolume := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes { if headVolume.Name == "ca-vol" { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes[i].Secret.SecretName = "invalid-secret-name" break } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Volume in the head group") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated Volume in the worker group", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { for i, workerVolume := range workerGroup.Template.Spec.Volumes { if workerVolume.Name == "ca-vol" { @@ -622,24 +848,24 @@ func TestValidateUpdate(t *testing.T) { } } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated Volume in the worker group") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated env vars in the head group", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for i, headEnvVar := range invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env { if headEnvVar.Name == "RAY_USE_TLS" { invalidRayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env[i].Value = "invalid-value" break } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated env vars in the head group") }) t.Run("Negative: Expected errors on call to ValidateUpdate function due to manipulated env vars in the worker group", func(t *testing.T) { - invalidRayCluster := validRayCluster.DeepCopy() + invalidRayCluster := validRayClusterNewNames.DeepCopy() for _, workerGroup := range invalidRayCluster.Spec.WorkerGroupSpecs { for i, workerEnvVar := range workerGroup.Template.Spec.Containers[0].Env { if workerEnvVar.Name == "RAY_USE_TLS" { @@ -648,7 +874,7 @@ func TestValidateUpdate(t *testing.T) { } } } - _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayCluster), runtime.Object(invalidRayCluster)) + _, err := rcWebhook.ValidateUpdate(test.Ctx(), runtime.Object(validRayClusterNewNames), runtime.Object(invalidRayCluster)) test.Expect(err).Should(HaveOccurred(), "Expected errors on call to ValidateUpdate function due to manipulated env vars in the worker group") }) } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index b2fec8b8c..9d236848e 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -1,6 +1,8 @@ package controllers import ( + "crypto/sha256" + "fmt" "os" "github.com/go-logr/logr" @@ -209,3 +211,11 @@ func ownerRefForRayCluster(cluster *rayv1.RayCluster) *v1.OwnerReferenceApplyCon WithUID(cluster.UID). WithController(true) } + +var ( + hashLength = 8 +) + +func seededHash(seed string, s string) string { + return fmt.Sprintf("%x", sha256.Sum256([]byte(seed+s)))[:hashLength] +} From 8fa9455d03e26432ef98458761e07543849c4642 Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 3 Sep 2024 17:23:29 -0400 Subject: [PATCH 291/369] RBAC fix to enable slack cluster queue lending limit adjustment The codeflare operator needs permission to read and write clusterqueues to enable the AppWrapper controller to adjust the lending limit of a designated slack cluster queue to reflect cordoned nodes. --- config/rbac/role.yaml | 10 ++++++++++ pkg/controllers/appwrapper_controller.go | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 879d6bc80..598b4dafb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -168,6 +168,16 @@ rules: - patch - update - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - get + - list + - patch + - update + - watch - apiGroups: - kueue.x-k8s.io resources: diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index f15a4ef2e..c58516dad 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -42,5 +42,6 @@ package controllers // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch -// permission to watch nodes for Autopilot integration +// permission to watch nodes and edit clusterqueues for Autopilot integration //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +//+kubebuilder:rbac:groups=kueue.x-k8s.io,resources=clusterqueues,verbs=get;list;watch;update;patch From 73d5bebd601c1422807243c73ce306eceed4a0a5 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 6 Sep 2024 14:10:34 +0000 Subject: [PATCH 292/369] Update dependency versions for release v1.8.0 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d5bf35556..5c7d4fbe1 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.7.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.7.0) | -| CodeFlare-SDK | [v0.19.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.19.1) | -| AppWrapper | [v0.23.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.23.0) | +| CodeFlare Operator | [v1.8.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.8.0) | +| CodeFlare-SDK | [v0.20.2](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.20.2) | +| AppWrapper | [v0.24.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.24.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.7.0](https://github.com/opendatahub-io/kueue/releases/tag/v0.7.0) | diff --git a/config/manager/params.env b/config/manager/params.env index 2efa29e44..9bc685f29 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.7.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.8.0 namespace=opendatahub From e1ce2264766df8a974d082bcb9dfc109c85c8c3a Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 9 Sep 2024 16:09:30 +0100 Subject: [PATCH 293/369] Update cert generator image to Ray 2.35 --- config/e2e/patch_resources.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/e2e/patch_resources.yaml b/config/e2e/patch_resources.yaml index 345e475d1..138170c35 100644 --- a/config/e2e/patch_resources.yaml +++ b/config/e2e/patch_resources.yaml @@ -7,4 +7,4 @@ path: /spec/template/spec/containers/0/env/- value: name: CERT_GENERATOR_IMAGE - value: quay.io/rhoai/ray:2.23.0-py39-cu121 + value: quay.io/modh/ray:2.35.0-py39-cu121 From 84bea74588bf077dd9aed36a4164ab6c3848776d Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 13 Sep 2024 10:20:31 +0100 Subject: [PATCH 294/369] Increasing e2e timeout to 15 minutes --- .github/workflows/e2e_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index be3104b22..7c8365652 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -79,7 +79,7 @@ jobs: - name: Run e2e tests run: | export CODEFLARE_TEST_TIMEOUT_SHORT=3m - export CODEFLARE_TEST_TIMEOUT_MEDIUM=10m + export CODEFLARE_TEST_TIMEOUT_MEDIUM=15m export CODEFLARE_TEST_TIMEOUT_LONG=20m export CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING=30m From 9c10a6a6af0e9ba3521807cf4b6250c190cb524b Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 26 Sep 2024 12:06:58 -0400 Subject: [PATCH 295/369] Update appwrappers to v0.25.0 Changes: + Add age printcolumn + Update to Kueue 0.8.1 + autopilot: add TESTING NoSchedule taint to default config --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 3 ++ go.mod | 36 ++++++++-------- go.sum | 52 +++++++++++++----------- 5 files changed, 53 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index a4ff90fa6..b59d77795 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.24.0 +APPWRAPPER_VERSION ?= v0.25.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 2c4dd3ef4..c81f73836 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.24.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.25.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index 5eaf8c019..17b80415e 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -26,6 +26,9 @@ spec: - jsonPath: .status.conditions[?(@.type=="Unhealthy")].status name: Unhealthy type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date name: v1beta2 schema: openAPIV3Schema: diff --git a/go.mod b/go.mod index 021816478..583e490f9 100644 --- a/go.mod +++ b/go.mod @@ -10,20 +10,20 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.24.0 + github.com/project-codeflare/appwrapper v0.25.0 github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 k8s.io/api v0.30.2 - k8s.io/apiextensions-apiserver v0.29.2 + k8s.io/apiextensions-apiserver v0.29.6 k8s.io/apimachinery v0.30.2 k8s.io/client-go v11.0.0+incompatible k8s.io/component-base v0.29.6 - k8s.io/klog/v2 v2.120.1 + k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 - sigs.k8s.io/controller-runtime v0.17.3 - sigs.k8s.io/kueue v0.8.0 + sigs.k8s.io/controller-runtime v0.17.5 + sigs.k8s.io/kueue v0.8.1 sigs.k8s.io/yaml v1.4.0 ) @@ -35,10 +35,13 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 -// These replace directives support the backlevel go version required by ODH build -replace github.com/project-codeflare/appwrapper v0.24.0 => github.com/project-codeflare/appwrapper v0.24.1-0.20240828152845-bfaac68eef30 +// This replace directive supports the backlevel go version required by the ODH build +replace github.com/project-codeflare/appwrapper v0.25.0 => github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4 -replace sigs.k8s.io/kueue v0.8.0 => github.com/opendatahub-io/kueue v0.7.0-odh-test +// These replace directives deal with the backlevel ODH kueue version +replace sigs.k8s.io/kueue v0.8.1 => github.com/opendatahub-io/kueue v0.7.0-odh-2 + +replace github.com/prometheus/common v0.57.0 => github.com/prometheus/common v0.46.0 require ( github.com/aymerick/douceur v0.2.0 // indirect @@ -50,7 +53,7 @@ require ( github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -73,6 +76,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/kubeflow/training-operator v1.7.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect @@ -84,29 +88,29 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_golang v1.20.2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.46.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/common v0.57.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.29.6 // indirect - k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect + k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect sigs.k8s.io/jobset v0.5.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 5d4999b87..8ba1d5b68 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -170,6 +170,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -182,6 +184,8 @@ github.com/kubeflow/mpi-operator v0.5.0 h1:XvBwyXXQ9103DNMa22sxsaQlaktvaT2LY/g0U github.com/kubeflow/mpi-operator v0.5.0/go.mod h1:SeZQJW8KJxSTWD++eQYKRFpoDg1v8yrdC6fjx2/3mG0= github.com/kubeflow/training-operator v1.7.0 h1:Zh61GlOWrlRi4UFOtJeV+/5REo/OndhwQ25KYd0llzc= github.com/kubeflow/training-operator v1.7.0/go.mod h1:BZCLX1h06wY3YSeSZZcGYAqI9/nVi7isVCRkfgZe9nE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -227,8 +231,8 @@ github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opendatahub-io/kueue v0.7.0-odh-test h1:QZg/iBs/+kbX7zIxktAi9j5yOezEgXq9BCLSvHJ+BIU= -github.com/opendatahub-io/kueue v0.7.0-odh-test/go.mod h1:uTAgWTJVrIkm1FPbp6BnhZhr29zlUcgzb5z1m5rFw9k= +github.com/opendatahub-io/kueue v0.7.0-odh-2 h1:sRPncaqDJ0a6WJiArVuYVDLPWTXm6hosqiBXzb9xMt4= +github.com/opendatahub-io/kueue v0.7.0-odh-2/go.mod h1:uTAgWTJVrIkm1FPbp6BnhZhr29zlUcgzb5z1m5rFw9k= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= @@ -243,19 +247,19 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.24.1-0.20240828152845-bfaac68eef30 h1:Yrjh9PSr08CphBREsGwk1fxMv3a79k2WHTp8vdLTdAM= -github.com/project-codeflare/appwrapper v0.24.1-0.20240828152845-bfaac68eef30/go.mod h1:ru5BZAN38Uoxn1XfATbVN0R6i82kFp29BKih4vFOgDo= +github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4 h1:XYjq50WpGxagELHurCXyiirvdM9OzxTnCMcQC9gebnQ= +github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4/go.mod h1:zDALq3/gn+eiczpD7TBZWWbAVuwcCGDFuN/77oh+CDw= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e h1:juFd1dQyioeMxbVE6F0YD25ozm/jiqJE+MpDhu8p22k= github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/ray-project/kuberay/ray-operator v1.1.1 h1:mVOA1ddS9aAsPvhhHrpf0ZXgTzccIAyTbeYeDqtcfAk= github.com/ray-project/kuberay/ray-operator v1.1.1/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -322,8 +326,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 h1:Mi0bCswbz+9cXmwFAdxoo5GPFMKONUpua6iUdtQS7lk= -golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -353,8 +357,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -453,8 +457,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -482,8 +486,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= -k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= +k8s.io/apiextensions-apiserver v0.29.6 h1:tUu1N6Zt9GT8KVcPF5aGDqfISz1mveM4yFh7eL5bxmE= +k8s.io/apiextensions-apiserver v0.29.6/go.mod h1:iw1EbwZat08I219qrQKoFMHGo7J9KxPqMpVKxCbNbCs= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= @@ -500,14 +504,14 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= k8s.io/kube-aggregator v0.28.3/go.mod h1:5DyLevbRTcWnT1f9b+lB3BfbXC1w7gDa/OtB6kKInCw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA= -k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlLosk/JlfP3xQLNkQlHJeXw= +k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/metrics v0.29.5 h1:5AZBM4pMiq+SHi2/RP1OuL90In3f07BrL610zunKoxA= k8s.io/metrics v0.29.5/go.mod h1:Cx7JPEEbjdDpEQ2A1QyXSBd6z5bxlwaSf9rgFJ/adfQ= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -516,8 +520,8 @@ k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCI k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= -sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= -sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/controller-runtime v0.17.5 h1:1FI9Lm7NiOOmBsgTV36/s2XrEFXnO2C4sbg/Zme72Rw= +sigs.k8s.io/controller-runtime v0.17.5/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= sigs.k8s.io/jobset v0.5.2 h1:276q5Pi/ErLYj+GQ0ydEXR6tx3LwBhEzHLQv+k8bYF4= sigs.k8s.io/jobset v0.5.2/go.mod h1:Vg99rj/6OoGvy1uvywGEHOcVLCWWJYkJtisKqdWzcFw= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= From 59a1d236d8fb2763886c508b2feea6fad2901ea1 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 27 Sep 2024 10:54:26 +0000 Subject: [PATCH 296/369] Update dependency versions for release v1.9.0 --- Makefile | 2 +- README.md | 8 ++++---- config/manager/params.env | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b59d77795..7d761845a 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the default version of Kueue (used for testing) -KUEUE_VERSION ?= v0.7.0 +KUEUE_VERSION ?= v0.8.1 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) diff --git a/README.md b/README.md index 5c7d4fbe1..171d23c1f 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.8.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.8.0) | -| CodeFlare-SDK | [v0.20.2](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.20.2) | -| AppWrapper | [v0.24.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.24.0) | +| CodeFlare Operator | [v1.9.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.9.0) | +| CodeFlare-SDK | [v0.21.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.21.0) | +| AppWrapper | [v0.25.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.25.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | -| Kueue | [v0.7.0](https://github.com/opendatahub-io/kueue/releases/tag/v0.7.0) | +| Kueue | [v0.8.1](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.1) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 9bc685f29..55360f27c 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.8.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.9.0 namespace=opendatahub From f50055554f2feab5b1d1c9980d7c0f977fefd708 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Thu, 26 Sep 2024 16:53:57 +0530 Subject: [PATCH 297/369] Create kueue resources as part of test execution --- go.mod | 2 +- go.sum | 4 +- test/e2e/mnist_pytorch_appwrapper_test.go | 17 +++-- test/e2e/mnist_rayjob_raycluster_test.go | 79 ++++++++++++++++++----- test/e2e/setup.sh | 27 -------- test/e2e/support.go | 12 ---- 6 files changed, 77 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index 583e490f9..7a70df90e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.25.0 - github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e + github.com/project-codeflare/codeflare-common v0.0.0-20240927111823-758dad4e90d0 github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 diff --git a/go.sum b/go.sum index 8ba1d5b68..bbb7e9f0b 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4 h1:XYjq50WpGxagELHurCXyiirvdM9OzxTnCMcQC9gebnQ= github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4/go.mod h1:zDALq3/gn+eiczpD7TBZWWbAVuwcCGDFuN/77oh+CDw= -github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e h1:juFd1dQyioeMxbVE6F0YD25ozm/jiqJE+MpDhu8p22k= -github.com/project-codeflare/codeflare-common v0.0.0-20240628111341-56c962a09b7e/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= +github.com/project-codeflare/codeflare-common v0.0.0-20240927111823-758dad4e90d0 h1:5gfJUhF2GRZIXCUK/aUYTo79Ipo4Ngg9HO8Jgj8zThM= +github.com/project-codeflare/codeflare-common v0.0.0-20240927111823-758dad4e90d0/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go index ab1426588..1919c616b 100644 --- a/test/e2e/mnist_pytorch_appwrapper_test.go +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -28,23 +28,30 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" ) func TestMnistPyTorchAppWrapperCpu(t *testing.T) { - runMnistPyTorchAppWrapper(t, "cpu") + runMnistPyTorchAppWrapper(t, "cpu", 0) } func TestMnistPyTorchAppWrapperGpu(t *testing.T) { - runMnistPyTorchAppWrapper(t, "gpu") + runMnistPyTorchAppWrapper(t, "gpu", 1) } // Trains the MNIST dataset as a batch Job in an AppWrapper, and asserts successful completion of the training job. -func runMnistPyTorchAppWrapper(t *testing.T, accelerator string) { +func runMnistPyTorchAppWrapper(t *testing.T, accelerator string, numberOfGpus int) { test := With(t) - // Create a namespace and localqueue in that namespace + // Create a namespace namespace := test.NewTestNamespace() - localQueue := CreateKueueLocalQueue(test, namespace.Name, "e2e-cluster-queue") + + // Create Kueue resources + resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) + defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) + defer test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Test configuration config := &corev1.ConfigMap{ diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index b1408ed49..c66fb28f3 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -33,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" ) // Trains the MNIST dataset as a RayJob, executed by a Ray cluster @@ -49,9 +50,15 @@ func TestMnistRayJobRayClusterGpu(t *testing.T) { func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int) { test := With(t) - // Create a namespace and localqueue in that namespace + // Create a namespace namespace := test.NewTestNamespace() - localQueue := CreateKueueLocalQueue(test, namespace.Name, "e2e-cluster-queue") + + // Create Kueue resources + resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) + defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) + defer test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Create MNIST training script mnist := constructMNISTConfigMap(test, namespace) @@ -61,7 +68,6 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int // Create RayCluster and assign it to the localqueue rayCluster := constructRayCluster(test, namespace, mnist, numberOfGpus) - AssignToLocalQueue(rayCluster, localQueue) rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name) @@ -78,8 +84,8 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int rayDashboardURL := getRayDashboardURL(test, rayCluster.Namespace, rayCluster.Name) - test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) - rayClient := NewRayClusterClient(rayDashboardURL) + test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL) + rayClient := GetRayClusterClient(test, rayDashboardURL, test.Config().BearerToken) test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). @@ -111,9 +117,15 @@ func TestMnistRayJobRayClusterAppWrapperGpu(t *testing.T) { func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, numberOfGpus int) { test := With(t) - // Create a namespace and localqueue in that namespace + // Create a namespace namespace := test.NewTestNamespace() - localQueue := CreateKueueLocalQueue(test, namespace.Name, "e2e-cluster-queue") + + // Create Kueue resources + resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) + defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) + defer test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Create MNIST training script mnist := constructMNISTConfigMap(test, namespace) @@ -167,8 +179,8 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number rayDashboardURL := getRayDashboardURL(test, rayCluster.Namespace, rayCluster.Name) - test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) - rayClient := NewRayClusterClient(rayDashboardURL) + test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL) + rayClient := GetRayClusterClient(test, rayDashboardURL, test.Config().BearerToken) test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). @@ -374,7 +386,7 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC } } -func getRayDashboardURL(test Test, namespace, rayClusterName string) url.URL { +func getRayDashboardURL(test Test, namespace, rayClusterName string) string { dashboardName := "ray-dashboard-" + rayClusterName if IsOpenShift(test) { @@ -396,10 +408,10 @@ func getRayDashboardURL(test Test, namespace, rayClusterName string) url.URL { return resp.StatusCode, nil }, TestTimeoutShort).Should(Not(Equal(503))) - return url.URL{ - Scheme: "https", - Host: hostname, - } + dashboardUrl, _ := url.Parse("https://" + hostname) + test.T().Logf("Ray-dashboard route : %s\n", dashboardUrl.String()) + + return dashboardUrl.String() } ingress := GetIngress(test, namespace, dashboardName) @@ -408,8 +420,41 @@ func getRayDashboardURL(test Test, namespace, rayClusterName string) url.URL { test.Eventually(Ingress(test, ingress.Namespace, ingress.Name), TestTimeoutShort). Should(WithTransform(LoadBalancerIngresses, HaveLen(1))) - return url.URL{ - Scheme: "http", - Host: ingress.Spec.Rules[0].Host, + hostname := ingress.Spec.Rules[0].Host + dashboardUrl, _ := url.Parse("http://" + hostname) + test.T().Logf("Ray-dashboard route : %s\n", dashboardUrl.String()) + + return dashboardUrl.String() +} + +// Create ClusterQueue +func createClusterQueue(test Test, resourceFlavor *v1beta1.ResourceFlavor, numberOfGpus int) *v1beta1.ClusterQueue { + cqSpec := v1beta1.ClusterQueueSpec{ + NamespaceSelector: &metav1.LabelSelector{}, + ResourceGroups: []v1beta1.ResourceGroup{ + { + CoveredResources: []corev1.ResourceName{corev1.ResourceName("cpu"), corev1.ResourceName("memory"), corev1.ResourceName("nvidia.com/gpu")}, + Flavors: []v1beta1.FlavorQuotas{ + { + Name: v1beta1.ResourceFlavorReference(resourceFlavor.Name), + Resources: []v1beta1.ResourceQuota{ + { + Name: corev1.ResourceCPU, + NominalQuota: resource.MustParse("8"), + }, + { + Name: corev1.ResourceMemory, + NominalQuota: resource.MustParse("12Gi"), + }, + { + Name: corev1.ResourceName("nvidia.com/gpu"), + NominalQuota: resource.MustParse(fmt.Sprint(numberOfGpus)), + }, + }, + }, + }, + }, + }, } + return CreateKueueClusterQueue(test, cqSpec) } diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 1ee1d90fc..71b5b12a0 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -69,30 +69,3 @@ done echo "" sleep 5 -echo Creating Kueue ResourceFlavor and ClusterQueue -cat < Date: Mon, 30 Sep 2024 10:28:52 -0400 Subject: [PATCH 298/369] update codeflare-common version to remove replace for prometheus/common --- go.mod | 6 ++---- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 7a70df90e..e87b9f57d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.25.0 - github.com/project-codeflare/codeflare-common v0.0.0-20240927111823-758dad4e90d0 + github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 @@ -41,8 +41,6 @@ replace github.com/project-codeflare/appwrapper v0.25.0 => github.com/project-co // These replace directives deal with the backlevel ODH kueue version replace sigs.k8s.io/kueue v0.8.1 => github.com/opendatahub-io/kueue v0.7.0-odh-2 -replace github.com/prometheus/common v0.57.0 => github.com/prometheus/common v0.46.0 - require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -88,7 +86,7 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.2 // indirect + github.com/prometheus/client_golang v1.20.4 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.57.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/go.sum b/go.sum index bbb7e9f0b..059a71934 100644 --- a/go.sum +++ b/go.sum @@ -249,15 +249,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4 h1:XYjq50WpGxagELHurCXyiirvdM9OzxTnCMcQC9gebnQ= github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4/go.mod h1:zDALq3/gn+eiczpD7TBZWWbAVuwcCGDFuN/77oh+CDw= -github.com/project-codeflare/codeflare-common v0.0.0-20240927111823-758dad4e90d0 h1:5gfJUhF2GRZIXCUK/aUYTo79Ipo4Ngg9HO8Jgj8zThM= -github.com/project-codeflare/codeflare-common v0.0.0-20240927111823-758dad4e90d0/go.mod h1:unKTw+XoMANTES3WieG016im7rxZ7IR2/ph++L5Vp1Y= -github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= -github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 h1:Eupu9yxaGTddtoxb9SjrYJlokRHEYU5NNVRQmdXSNVs= +github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3/go.mod h1:v7XKwaDoCspsHQlWJNarO7gOpR+iumSS+c1bWs3kJOI= +github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= +github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= -github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/ray-project/kuberay/ray-operator v1.1.1 h1:mVOA1ddS9aAsPvhhHrpf0ZXgTzccIAyTbeYeDqtcfAk= From 5c7c6d2b7a4e17cad47fc47c6e1a533dfdc630d8 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 16 Oct 2024 18:10:46 -0400 Subject: [PATCH 299/369] Update appwrappers to v0.26.0 Changes: + Redesign node monitoring to account for Node deletion + Also log the delta when lendingLimit is updated + Implement spec.managedBy field --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 3 +++ go.mod | 4 ++-- go.sum | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 7d761845a..529841c60 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.25.0 +APPWRAPPER_VERSION ?= v0.26.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index c81f73836..f9579adb2 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.25.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.26.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index 17b80415e..02fa812aa 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -152,6 +152,9 @@ spec: - template type: object type: array + managedBy: + description: ManagedBy is used to indicate the controller or entity that manages the AppWrapper. + type: string suspend: description: Suspend suspends the AppWrapper when set to true type: boolean diff --git a/go.mod b/go.mod index e87b9f57d..8cd893c4d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.25.0 + github.com/project-codeflare/appwrapper v0.26.0 github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 @@ -36,7 +36,7 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // This replace directive supports the backlevel go version required by the ODH build -replace github.com/project-codeflare/appwrapper v0.25.0 => github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4 +replace github.com/project-codeflare/appwrapper v0.26.0 => github.com/project-codeflare/appwrapper v0.26.1-0.20241016213414-d3825c270f61 // These replace directives deal with the backlevel ODH kueue version replace sigs.k8s.io/kueue v0.8.1 => github.com/opendatahub-io/kueue v0.7.0-odh-2 diff --git a/go.sum b/go.sum index 059a71934..34c4da85b 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4 h1:XYjq50WpGxagELHurCXyiirvdM9OzxTnCMcQC9gebnQ= -github.com/project-codeflare/appwrapper v0.25.1-0.20240926155059-30a8af17b8f4/go.mod h1:zDALq3/gn+eiczpD7TBZWWbAVuwcCGDFuN/77oh+CDw= +github.com/project-codeflare/appwrapper v0.26.1-0.20241016213414-d3825c270f61 h1:jSqZDG4RihAkBaQp2fYFbp6EO4EGMv0pxfvO+6C2Cis= +github.com/project-codeflare/appwrapper v0.26.1-0.20241016213414-d3825c270f61/go.mod h1:zDALq3/gn+eiczpD7TBZWWbAVuwcCGDFuN/77oh+CDw= github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 h1:Eupu9yxaGTddtoxb9SjrYJlokRHEYU5NNVRQmdXSNVs= github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3/go.mod h1:v7XKwaDoCspsHQlWJNarO7gOpR+iumSS+c1bWs3kJOI= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= From 067c47bca25ee7f6736b3cd1d714c373f970b30a Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Thu, 17 Oct 2024 14:41:56 +0000 Subject: [PATCH 300/369] Update dependency versions for release v1.10.0 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 171d23c1f..83e17c4ea 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.9.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.9.0) | -| CodeFlare-SDK | [v0.21.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.21.0) | -| AppWrapper | [v0.25.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.25.0) | +| CodeFlare Operator | [v1.10.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.10.0) | +| CodeFlare-SDK | [v0.22.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.22.0) | +| AppWrapper | [v0.26.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.26.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.8.1](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.1) | diff --git a/config/manager/params.env b/config/manager/params.env index 55360f27c..8b226eb77 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.9.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.10.0 namespace=opendatahub From 397963f4d705d4a9ad8930655d5b2b77b8737fdf Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 31 Oct 2024 14:06:54 -0400 Subject: [PATCH 301/369] Add RBAC aggregator cluster roles for appwrappers --- config/rbac/appwrapper_editor_role.yaml | 27 +++++++++++++++++++++++++ config/rbac/appwrapper_viewer_role.yaml | 22 ++++++++++++++++++++ config/rbac/kustomization.yaml | 2 ++ 3 files changed, 51 insertions(+) create mode 100644 config/rbac/appwrapper_editor_role.yaml create mode 100644 config/rbac/appwrapper_viewer_role.yaml diff --git a/config/rbac/appwrapper_editor_role.yaml b/config/rbac/appwrapper_editor_role.yaml new file mode 100644 index 000000000..73fa98691 --- /dev/null +++ b/config/rbac/appwrapper_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit appwrappers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + name: appwrapper-editor-role +rules: +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/status + verbs: + - get diff --git a/config/rbac/appwrapper_viewer_role.yaml b/config/rbac/appwrapper_viewer_role.yaml new file mode 100644 index 000000000..1efbb8599 --- /dev/null +++ b/config/rbac/appwrapper_viewer_role.yaml @@ -0,0 +1,22 @@ +# permissions for end users to view appwrappers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: viewer-role +rules: +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + verbs: + - get + - list + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 166fe7986..29b817651 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -9,3 +9,5 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +- appwrapper_editor_role.yaml +- appwrapper_viewer_role.yaml From 1810758edc05d9acc87bef7ac5551fe497f4fe89 Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 5 Nov 2024 11:42:18 -0500 Subject: [PATCH 302/369] Update appwrappers to 0.27.0 Changes: * Append autopilot anti-affinities to existing matchExpressions array * Add labels to aggregate appwrapper editor/viewer role RBACs * Bump Kueue from 0.8.1 to 0.8.3 --- Makefile | 4 ++-- config/crd/appwrapper/kustomization.yaml | 2 +- go.mod | 8 ++++---- go.sum | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 529841c60..5ae089c59 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,12 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.26.0 +APPWRAPPER_VERSION ?= v0.27.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the default version of Kueue (used for testing) -KUEUE_VERSION ?= v0.8.1 +KUEUE_VERSION ?= v0.8.3 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index f9579adb2..75e9b5310 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.26.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.27.0 diff --git a/go.mod b/go.mod index 8cd893c4d..8e6e0a74d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.26.0 + github.com/project-codeflare/appwrapper v0.27.0 github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 github.com/ray-project/kuberay/ray-operator v1.1.1 go.uber.org/zap v1.27.0 @@ -23,7 +23,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.17.5 - sigs.k8s.io/kueue v0.8.1 + sigs.k8s.io/kueue v0.8.3 sigs.k8s.io/yaml v1.4.0 ) @@ -36,10 +36,10 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // This replace directive supports the backlevel go version required by the ODH build -replace github.com/project-codeflare/appwrapper v0.26.0 => github.com/project-codeflare/appwrapper v0.26.1-0.20241016213414-d3825c270f61 +replace github.com/project-codeflare/appwrapper v0.27.0 => github.com/project-codeflare/appwrapper v0.27.1-0.20241105160939-f71c6326acce // These replace directives deal with the backlevel ODH kueue version -replace sigs.k8s.io/kueue v0.8.1 => github.com/opendatahub-io/kueue v0.7.0-odh-2 +replace sigs.k8s.io/kueue v0.8.3 => github.com/opendatahub-io/kueue v0.7.0-odh-2 require ( github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 34c4da85b..8f6d198be 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.26.1-0.20241016213414-d3825c270f61 h1:jSqZDG4RihAkBaQp2fYFbp6EO4EGMv0pxfvO+6C2Cis= -github.com/project-codeflare/appwrapper v0.26.1-0.20241016213414-d3825c270f61/go.mod h1:zDALq3/gn+eiczpD7TBZWWbAVuwcCGDFuN/77oh+CDw= +github.com/project-codeflare/appwrapper v0.27.1-0.20241105160939-f71c6326acce h1:fFxewNcpKzvxYfJdwmHjJswoupX+5u1o9U3P8Gfh49o= +github.com/project-codeflare/appwrapper v0.27.1-0.20241105160939-f71c6326acce/go.mod h1:CDh8V/IGrfT9UW1CWGZNZQmb/5PNK8VGzZsiUL8ywlo= github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 h1:Eupu9yxaGTddtoxb9SjrYJlokRHEYU5NNVRQmdXSNVs= github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3/go.mod h1:v7XKwaDoCspsHQlWJNarO7gOpR+iumSS+c1bWs3kJOI= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= From d6be0ab6c728a4052ff12951ce4c4515843f6d4b Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Thu, 7 Nov 2024 12:52:10 +0000 Subject: [PATCH 303/369] Update dependency versions for release v1.11.0 --- README.md | 8 ++++---- config/manager/params.env | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 83e17c4ea..6d76b453d 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.10.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.10.0) | -| CodeFlare-SDK | [v0.22.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.22.0) | -| AppWrapper | [v0.26.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.26.0) | +| CodeFlare Operator | [v1.11.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.11.0) | +| CodeFlare-SDK | [v0.23.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.23.1) | +| AppWrapper | [v0.27.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.27.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | -| Kueue | [v0.8.1](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.1) | +| Kueue | [v0.8.3](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.3) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 8b226eb77..7b6fd8c90 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.10.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.11.0 namespace=opendatahub From 1fda820d4acc0687e01cb1a3f9bf06551d281d5b Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 15 Nov 2024 14:37:22 +0100 Subject: [PATCH 304/369] Use Red Hat Go toolset image for CodeFlare operator image build --- Dockerfile | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 72e842059..24d923734 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,6 @@ # Build the manager binary -# BEGIN -- workaround lack of go-toolset for golang 1.22 - -ARG GOLANG_IMAGE=docker.io/library/golang:1.22 - -ARG GOARCH=amd64 - -FROM ${GOLANG_IMAGE} AS golang -FROM registry.access.redhat.com/ubi8/ubi:8.8 AS builder - -# Install system dependencies -RUN dnf upgrade -y && dnf install -y \ - gcc \ - make \ - openssl-devel \ - && dnf clean all && rm -rf /var/cache/yum - -# Install Go -ENV PATH=/usr/local/go/bin:$PATH -COPY --from=golang /usr/local/go /usr/local/go - -# END -- workaround lack of go-toolset for golang 1.22 +FROM registry.access.redhat.com/ubi8/go-toolset:1.22@sha256:780ab5f3874a6e2b1e04bb3719e614e835af3f8ab150922d6e84c2f9fd2bdb27 AS builder WORKDIR /workspace # Copy the Go Modules manifests From cb032efe1f1572eb637c43bd4440a79a3ee2429d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 19 Nov 2024 09:40:48 +0100 Subject: [PATCH 305/369] AppWrapper e2e: Remove creationTimestamp from marshalled template --- test/e2e/mnist_pytorch_appwrapper_test.go | 5 ++++- test/e2e/mnist_rayjob_raycluster_test.go | 5 ++++- test/e2e/support.go | 11 +++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go index 1919c616b..34fb742f0 100644 --- a/test/e2e/mnist_pytorch_appwrapper_test.go +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -145,6 +145,9 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string, numberOfGpus in }, } + raw := Raw(test, job) + raw = RemoveCreationTimestamp(test, raw) + // Create an AppWrapper resource aw := &mcadv1beta2.AppWrapper{ TypeMeta: metav1.TypeMeta{ @@ -159,7 +162,7 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string, numberOfGpus in Spec: mcadv1beta2.AppWrapperSpec{ Components: []mcadv1beta2.AppWrapperComponent{ { - Template: Raw(test, job), + Template: raw, }, }, }, diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index c66fb28f3..b2e724834 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -135,6 +135,9 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number // Create RayCluster, wrap in AppWrapper and assign to localqueue rayCluster := constructRayCluster(test, namespace, mnist, numberOfGpus) + raw := Raw(test, rayCluster) + raw = RemoveCreationTimestamp(test, raw) + aw := &mcadv1beta2.AppWrapper{ TypeMeta: metav1.TypeMeta{ APIVersion: mcadv1beta2.GroupVersion.String(), @@ -148,7 +151,7 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number Spec: mcadv1beta2.AppWrapperSpec{ Components: []mcadv1beta2.AppWrapperComponent{ { - Template: Raw(test, rayCluster), + Template: raw, }, }, }, diff --git a/test/e2e/support.go b/test/e2e/support.go index 8fc862957..dec8915db 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -18,9 +18,12 @@ package e2e import ( "embed" + "strings" "github.com/onsi/gomega" "github.com/project-codeflare/codeflare-common/support" + + "k8s.io/apimachinery/pkg/runtime" ) //go:embed *.py *.txt *.sh @@ -32,3 +35,11 @@ func ReadFile(t support.Test, fileName string) []byte { t.Expect(err).NotTo(gomega.HaveOccurred()) return file } + +func RemoveCreationTimestamp(t support.Test, rawExtension runtime.RawExtension) runtime.RawExtension { + t.T().Helper() + patchedRaw := strings.ReplaceAll(string(rawExtension.Raw), `"metadata":{"creationTimestamp":null},`, "") + return runtime.RawExtension{ + Raw: []byte(patchedRaw), + } +} From 2973dec73b2e5193c08480bc5eaecbd3148a73cf Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 15 Nov 2024 15:19:37 +0100 Subject: [PATCH 306/369] Raise Kueue version in CodeFlare operator --- go.mod | 9 +++++--- go.sum | 71 ++++------------------------------------------------------ 2 files changed, 10 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index 8e6e0a74d..6d93e73b6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.27.0 github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 - github.com/ray-project/kuberay/ray-operator v1.1.1 + github.com/ray-project/kuberay/ray-operator v1.2.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 k8s.io/api v0.30.2 @@ -38,8 +38,11 @@ replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // This replace directive supports the backlevel go version required by the ODH build replace github.com/project-codeflare/appwrapper v0.27.0 => github.com/project-codeflare/appwrapper v0.27.1-0.20241105160939-f71c6326acce -// These replace directives deal with the backlevel ODH kueue version -replace sigs.k8s.io/kueue v0.8.3 => github.com/opendatahub-io/kueue v0.7.0-odh-2 +// This replace directive deal with the backlevel ODH kueue version +replace sigs.k8s.io/kueue v0.8.3 => github.com/opendatahub-io/kueue v0.8.3 + +// This replace directive deal with the unaligned Ray operator version transitively pulled by Kueue 0.8.3 +replace github.com/ray-project/kuberay/ray-operator v1.2.1 => github.com/ray-project/kuberay/ray-operator v1.1.1 require ( github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 8f6d198be..3a119a629 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,9 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 h1:X8MJ0fnN5FPdcGF5Ij2/OW+HgiJrRg3AfHAx1PJtIzM= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -25,10 +19,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= -github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -50,8 +40,6 @@ github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lSh github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -64,8 +52,6 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -108,8 +94,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= -github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -136,17 +120,10 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= @@ -231,8 +208,8 @@ github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opendatahub-io/kueue v0.7.0-odh-2 h1:sRPncaqDJ0a6WJiArVuYVDLPWTXm6hosqiBXzb9xMt4= -github.com/opendatahub-io/kueue v0.7.0-odh-2/go.mod h1:uTAgWTJVrIkm1FPbp6BnhZhr29zlUcgzb5z1m5rFw9k= +github.com/opendatahub-io/kueue v0.8.3 h1:MLkHCmIrQR1KM1IcPiGuoEAT3Y+ZTs7493sMkmSUMow= +github.com/opendatahub-io/kueue v0.8.3/go.mod h1:jzRyUhAXHIpEPjt4pMx79t/Cg1g29GlNZY6wiLJE2YI= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= @@ -262,20 +239,16 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/ray-project/kuberay/ray-operator v1.1.1 h1:mVOA1ddS9aAsPvhhHrpf0ZXgTzccIAyTbeYeDqtcfAk= github.com/ray-project/kuberay/ray-operator v1.1.1/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -288,30 +261,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.11 h1:B54KwXbWDHyD3XYAwprxNzTe7vlhR69LuBgZnMVvS7E= -go.etcd.io/etcd/api/v3 v3.5.11/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= -go.etcd.io/etcd/client/pkg/v3 v3.5.11 h1:bT2xVspdiCj2910T0V+/KHcVKjkUrCZVtk8J2JF2z1A= -go.etcd.io/etcd/client/pkg/v3 v3.5.11/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= -go.etcd.io/etcd/client/v3 v3.5.11 h1:ajWtgoNSZJ1gmS8k+icvPtqsqEav+iUorF7b0qozgUU= -go.etcd.io/etcd/client/v3 v3.5.11/go.mod h1:a6xQUEqFJ8vztO1agJh/KQKOMfFI8og52ZconzcDJwE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= -go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= -go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= -go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= -go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= -go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= -go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= -go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= -go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -434,17 +383,9 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -512,14 +453,10 @@ k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlLosk/JlfP3xQLNkQlHJeXw= k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= -k8s.io/metrics v0.29.5 h1:5AZBM4pMiq+SHi2/RP1OuL90In3f07BrL610zunKoxA= -k8s.io/metrics v0.29.5/go.mod h1:Cx7JPEEbjdDpEQ2A1QyXSBd6z5bxlwaSf9rgFJ/adfQ= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= sigs.k8s.io/controller-runtime v0.17.5 h1:1FI9Lm7NiOOmBsgTV36/s2XrEFXnO2C4sbg/Zme72Rw= sigs.k8s.io/controller-runtime v0.17.5/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= sigs.k8s.io/jobset v0.5.2 h1:276q5Pi/ErLYj+GQ0ydEXR6tx3LwBhEzHLQv+k8bYF4= From dd6851a7ff4b4ba0468d3cdda0bf00a8549fc943 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 20 Nov 2024 15:18:29 +0100 Subject: [PATCH 307/369] Remove AppWrapper workaround for downgrading Go lang version --- go.mod | 5 +---- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6d93e73b6..fd131ca3e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/project-codeflare/codeflare-operator -go 1.22.2 +go 1.22.4 require ( github.com/go-logr/logr v1.4.2 @@ -35,9 +35,6 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 -// This replace directive supports the backlevel go version required by the ODH build -replace github.com/project-codeflare/appwrapper v0.27.0 => github.com/project-codeflare/appwrapper v0.27.1-0.20241105160939-f71c6326acce - // This replace directive deal with the backlevel ODH kueue version replace sigs.k8s.io/kueue v0.8.3 => github.com/opendatahub-io/kueue v0.8.3 diff --git a/go.sum b/go.sum index 3a119a629..0b5d76171 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.27.1-0.20241105160939-f71c6326acce h1:fFxewNcpKzvxYfJdwmHjJswoupX+5u1o9U3P8Gfh49o= -github.com/project-codeflare/appwrapper v0.27.1-0.20241105160939-f71c6326acce/go.mod h1:CDh8V/IGrfT9UW1CWGZNZQmb/5PNK8VGzZsiUL8ywlo= +github.com/project-codeflare/appwrapper v0.27.0 h1:WiWw0Hi4rEXuFIEpm8nq1UqJHgVB6YtGcWzRrhRUTyE= +github.com/project-codeflare/appwrapper v0.27.0/go.mod h1:7FpO90DLv0BAq4rwZtXKS9aRRfkR9RvXsj3pgYF0HtQ= github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 h1:Eupu9yxaGTddtoxb9SjrYJlokRHEYU5NNVRQmdXSNVs= github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3/go.mod h1:v7XKwaDoCspsHQlWJNarO7gOpR+iumSS+c1bWs3kJOI= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= From 390fb1783fd78a598484122b0fa457166319a93d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 27 Nov 2024 13:02:53 +0100 Subject: [PATCH 308/369] Add --cgroup-manager cgroupfs for e2e PR check --- .github/workflows/e2e_tests.yaml | 3 +++ Makefile | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 7c8365652..c6cccde5f 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -75,6 +75,9 @@ jobs: kind load image-archive cfo.tar --name cluster --verbosity 1000 make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager + env: + # cgroup-manager configuration is required for NVidia image used in e2e PR check + IMAGE_BUILD_FLAGS: '--cgroup-manager cgroupfs' - name: Run e2e tests run: | diff --git a/Makefile b/Makefile index 5ae089c59..ba5925a3d 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,10 @@ endif # Image URL to use all building/pushing image targets IMG ?= ${IMAGE_TAG_BASE}:${VERSION} + +# IMAGE_BUILD_FLAGS are the flags passed to the podman operator image build command +IMAGE_BUILD_FLAGS := + # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.24.2 @@ -184,7 +188,7 @@ run: manifests fmt vet ## Run a controller from your host. .PHONY: image-build image-build: test-unit ## Build container image with the manager. - podman build -t ${IMG} . + podman $(IMAGE_BUILD_FLAGS) build -t ${IMG} . .PHONY: image-push image-push: image-build ## Push container image with the manager. From 385d79ec2e167bdd5ffd42ed153bbba003477c24 Mon Sep 17 00:00:00 2001 From: sutaakar Date: Thu, 28 Nov 2024 08:16:47 +0000 Subject: [PATCH 309/369] Update dependency versions for release v1.12.0 --- README.md | 4 ++-- config/manager/params.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6d76b453d..2cb3d8fc1 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.11.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.11.0) | -| CodeFlare-SDK | [v0.23.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.23.1) | +| CodeFlare Operator | [v1.12.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.12.0) | +| CodeFlare-SDK | [v0.24.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.24.0) | | AppWrapper | [v0.27.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.27.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.8.3](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.3) | diff --git a/config/manager/params.env b/config/manager/params.env index 7b6fd8c90..a8879b8da 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.11.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.12.0 namespace=opendatahub From ffb0b991e4039e7b4f896ae6b8c18b13d1f4ba18 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 28 Nov 2024 10:43:10 +0100 Subject: [PATCH 310/369] Minor adjustments to CFO release process --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cb3d8fc1..9920c9d94 100644 --- a/README.md +++ b/README.md @@ -110,9 +110,10 @@ For ODH tests additional environment variables are needed: 2. Once all jobs within the action are completed, verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. 3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. 4. Once PR is merged, announce the new release in slack and mail lists, if any. -5. Release automation should auto-merge changes to [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Verify the [workflow](https://github.com/project-codeflare/codeflare-operator/actions/workflows/project-codeflare-release.yml) ran successfully and review the new merge-commit and commit history. Same for the [Red Hat CodeFlare Operator repo](https://github.com/red-hat-data-services/codeflare-operator), while also ensuring changes are in the latest `rhoai` release branch. - If the auto-merge fails, conflicts must be resolved and force pushed manually to each downstream repository and release branch. +5. Release automation should auto-merge changes to [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Verify the [workflow](https://github.com/red-hat-data-services/codeflare-operator/actions/workflows/auto-merge-sync.yaml) ran successfully and review the new merge-commit and commit history. Same for the [Red Hat CodeFlare Operator repo](https://github.com/red-hat-data-services/codeflare-operator), while also ensuring changes are in the latest `rhoai` release branch. - If the auto-merge fails, conflicts must be resolved and force pushed manually to each downstream repository and release branch. 6. In ODH/CFO verify that the [Build and Push action](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/build-and-push.yaml) was triggered and ran successfully. 7. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. +8. Run [ODH CodeFlare Operator release workflow](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/odh-release.yml) to produce ODH CodeFlare Operator release. ### Releases involving part of the stack From c028863493350bbc566605028ed83a783062f15c Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 16 Dec 2024 11:21:06 +0100 Subject: [PATCH 311/369] E2E tests: Upgrade MNIST test script dependencies --- go.mod | 2 +- go.sum | 4 ++-- test/e2e/mnist.py | 4 ++-- test/e2e/mnist_pip_requirements.txt | 6 +++--- test/e2e/mnist_rayjob_raycluster_test.go | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index fd131ca3e..5833ceb62 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.27.0 - github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 + github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924 github.com/ray-project/kuberay/ray-operator v1.2.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 diff --git a/go.sum b/go.sum index 0b5d76171..f11f3adf1 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.27.0 h1:WiWw0Hi4rEXuFIEpm8nq1UqJHgVB6YtGcWzRrhRUTyE= github.com/project-codeflare/appwrapper v0.27.0/go.mod h1:7FpO90DLv0BAq4rwZtXKS9aRRfkR9RvXsj3pgYF0HtQ= -github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3 h1:Eupu9yxaGTddtoxb9SjrYJlokRHEYU5NNVRQmdXSNVs= -github.com/project-codeflare/codeflare-common v0.0.0-20240930133152-11fd6e3be6b3/go.mod h1:v7XKwaDoCspsHQlWJNarO7gOpR+iumSS+c1bWs3kJOI= +github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924 h1:jM+gYqn8eGmUoeQLGGYxlJgXZ1gbZgB2UtpKU9z0x9s= +github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index 5a89a8b38..97b22fc9c 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -72,8 +72,8 @@ def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): nn.Linear(hidden_size, self.num_classes), ) - self.val_accuracy = Accuracy() - self.test_accuracy = Accuracy() + self.val_accuracy = Accuracy(task="multiclass", num_classes=10) + self.test_accuracy = Accuracy(task="multiclass", num_classes=10) def forward(self, x): x = self.model(x) diff --git a/test/e2e/mnist_pip_requirements.txt b/test/e2e/mnist_pip_requirements.txt index 4c9d5fcb8..9f0543d4d 100644 --- a/test/e2e/mnist_pip_requirements.txt +++ b/test/e2e/mnist_pip_requirements.txt @@ -1,3 +1,3 @@ -pytorch_lightning==1.9.5 -torchmetrics==0.9.1 -torchvision==0.12.0 +pytorch_lightning==2.4.0 +torchmetrics==1.6.0 +torchvision==0.20.1 diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index b2e724834..443da1522 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -358,9 +358,9 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC Entrypoint: "python /home/ray/jobs/mnist.py", RuntimeEnvYAML: ` pip: - - pytorch_lightning==1.9.5 - - torchmetrics==0.9.1 - - torchvision==0.12.0 + - pytorch_lightning==2.4.0 + - torchmetrics==1.6.0 + - torchvision==0.20.1 env_vars: MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `" PIP_INDEX_URL: "` + GetPipIndexURL() + `" From 9c96f449aec20f8b1bb9729d73a5eb0e347dfa59 Mon Sep 17 00:00:00 2001 From: David Grove Date: Sat, 14 Dec 2024 00:01:46 -0500 Subject: [PATCH 312/369] Fix improper controller-runtime cache configuration When AppWrappers are enabled, it is not correct to configure the controller-runtime cache with a filter that only allows services, secrets, etc with the RayCluster label to be cached. This breaks any AppWrapper that contains one of these resource kinds. --- main.go | 54 ++++++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/main.go b/main.go index d9e46c221..044716fee 100644 --- a/main.go +++ b/main.go @@ -171,36 +171,34 @@ func main() { kubeConfig.QPS = ptr.Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) setupLog.V(2).Info("REST client", "qps", kubeConfig.QPS, "burst", kubeConfig.Burst) - selector, err := labels.Parse(controllers.RayClusterNameLabel) - exitOnError(err, "unable to parse label selector") - - cacheOpts := cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &corev1.Secret{}: { - Label: selector, - }, - &corev1.Service{}: { - Label: selector, - }, - &corev1.ServiceAccount{}: { - Label: selector, - }, - &networkingv1.Ingress{}: { - Label: selector, - }, - &networkingv1.NetworkPolicy{}: { - Label: selector, - }, - &rbacv1.ClusterRoleBinding{}: { - Label: selector, - }, - }, - } - - if isOpenShift(ctx, kubeClient.DiscoveryClient) { - cacheOpts.ByObject[&routev1.Route{}] = cache.ByObject{ + cacheOpts := cache.Options{} + if cfg.AppWrapper == nil || !ptr.Deref(cfg.AppWrapper.Enabled, false) { + selector, err := labels.Parse(controllers.RayClusterNameLabel) + exitOnError(err, "unable to parse label selector") + cacheOpts.ByObject = make(map[client.Object]cache.ByObject, 7) + cacheOpts.ByObject[&corev1.Secret{}] = cache.ByObject{ + Label: selector, + } + cacheOpts.ByObject[&corev1.Service{}] = cache.ByObject{ + Label: selector, + } + cacheOpts.ByObject[&corev1.ServiceAccount{}] = cache.ByObject{ + Label: selector, + } + cacheOpts.ByObject[&networkingv1.Ingress{}] = cache.ByObject{ Label: selector, } + cacheOpts.ByObject[&networkingv1.NetworkPolicy{}] = cache.ByObject{ + Label: selector, + } + cacheOpts.ByObject[&rbacv1.ClusterRoleBinding{}] = cache.ByObject{ + Label: selector, + } + if isOpenShift(ctx, kubeClient.DiscoveryClient) { + cacheOpts.ByObject[&routev1.Route{}] = cache.ByObject{ + Label: selector, + } + } } mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ From 8ad47852320a01272e21052321247d09d8e4a272 Mon Sep 17 00:00:00 2001 From: Akram Ben Aissi Date: Mon, 9 Dec 2024 11:30:39 +0100 Subject: [PATCH 313/369] Adds UPDATE to the RayCluster Mutating Webhook so apply can be applied --- config/webhook/manifests.yaml | 1 + pkg/controllers/raycluster_webhook.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 36dbe265c..ccc2a9797 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -40,6 +40,7 @@ webhooks: - v1 operations: - CREATE + - UPDATE resources: - rayclusters sideEffects: None diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index c630bd5ba..5515ca836 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -56,7 +56,7 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf Complete() } -// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create,versions=v1,name=mraycluster.ray.openshift.ai,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.ray.openshift.ai,admissionReviewVersions=v1 // +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.ray.openshift.ai,admissionReviewVersions=v1 type rayClusterWebhook struct { From ed8a71f7ad07b5c8f81df96a54e9d1edde385ff8 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 16 Dec 2024 18:40:23 -0500 Subject: [PATCH 314/369] add e2e tests for AppWrappers with Jobs, Deployments, and Services --- test/e2e/deployment_appwrapper_test.go | 165 ++++++++++++++++++++++ test/e2e/job_appwrapper_test.go | 143 +++++++++++++++++++ test/e2e/mnist_pytorch_appwrapper_test.go | 8 +- test/e2e/mnist_rayjob_raycluster_test.go | 16 ++- test/e2e/support.go | 2 + 5 files changed, 328 insertions(+), 6 deletions(-) create mode 100644 test/e2e/deployment_appwrapper_test.go create mode 100644 test/e2e/job_appwrapper_test.go diff --git a/test/e2e/deployment_appwrapper_test.go b/test/e2e/deployment_appwrapper_test.go new file mode 100644 index 000000000..39d953c7b --- /dev/null +++ b/test/e2e/deployment_appwrapper_test.go @@ -0,0 +1,165 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + mcadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + . "github.com/project-codeflare/codeflare-common/support" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" +) + +// verify that an AppWrapper containing a Deployment and Service can execute successfully +func TestDeploymentAppWrapper(t *testing.T) { + test := With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Create Kueue resources + resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + }() + clusterQueue := createClusterQueue(test, resourceFlavor, 0) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + }() + localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) + + // Deployment + Service (ie, a typical inference setup) + test.T().Logf("AppWrapper containing Deployment and Service") + job := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "deployment", + Namespace: namespace.Name, + Labels: map[string]string{"app": "inference"}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "inference"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "inference"}, + }, + Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + }, + }, + Containers: []corev1.Container{ + { + Name: "job", + Image: "quay.io/project-codeflare/busybox:1.36", + Command: []string{"/bin/sh", "-c", "sleep 600; exit 0"}, + }, + }, + RestartPolicy: corev1.RestartPolicyAlways, + }, + }, + }, + } + + service := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "service", + Namespace: namespace.Name, + Labels: map[string]string{"app": "inference"}, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{"app": "inference"}, + Ports: []corev1.ServicePort{{Port: 8080, Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromInt(8080)}}, + }, + } + + raw1 := Raw(test, job) + raw1 = RemoveCreationTimestamp(test, raw1) + raw2 := Raw(test, service) + raw2 = RemoveCreationTimestamp(test, raw2) + + // Create an AppWrapper resource + aw := &mcadv1beta2.AppWrapper{ + TypeMeta: metav1.TypeMeta{ + APIVersion: mcadv1beta2.GroupVersion.String(), + Kind: "AppWrapper", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "infserver-", + Namespace: namespace.Name, + Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, + }, + Spec: mcadv1beta2.AppWrapperSpec{ + Components: []mcadv1beta2.AppWrapperComponent{ + { + Template: raw1, + }, + { + Template: raw2, + }, + }, + }, + } + + appWrapperResource := mcadv1beta2.GroupVersion.WithResource("appwrappers") + awMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) + test.Expect(err).NotTo(HaveOccurred()) + unstruct := unstructured.Unstructured{Object: awMap} + unstructp, err := test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructp.Object, aw) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) + + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) + test.Eventually(AppWrappers(test, namespace), TestTimeoutMedium). + Should(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) + + // A deployment will not complete; so simply make sure it keeps running for reasonable interval + test.T().Logf("Ensuring the AppWrapper %s/%s continues to run", aw.Namespace, aw.Name) + test.Consistently(AppWrappers(test, namespace), TestTimeoutMedium).Should( + ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) + + test.T().Logf("Deleting AppWrapper %s/%s", aw.Namespace, aw.Name) + err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.T().Logf("Waiting for AppWrapper %s/%s to be deleted", aw.Namespace, aw.Name) + test.Eventually(AppWrappers(test, namespace), TestTimeoutShort).Should(BeEmpty()) +} diff --git a/test/e2e/job_appwrapper_test.go b/test/e2e/job_appwrapper_test.go new file mode 100644 index 000000000..9805db202 --- /dev/null +++ b/test/e2e/job_appwrapper_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + mcadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + . "github.com/project-codeflare/codeflare-common/support" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" +) + +// verify that an AppWrapper containing a batchv1/Job can execute successfully +func TestBatchJobAppWrapper(t *testing.T) { + test := With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Create Kueue resources + resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + }() + clusterQueue := createClusterQueue(test, resourceFlavor, 0) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + }() + localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) + + // Batch Job + test.T().Logf("AppWrapper containing batchv1/Job") + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "batchjob", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + }, + }, + Containers: []corev1.Container{ + { + Name: "job", + Image: "quay.io/project-codeflare/busybox:1.36", + Command: []string{"/bin/sh", "-c", "sleep 20; exit 0"}, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + raw := Raw(test, job) + raw = RemoveCreationTimestamp(test, raw) + + // Create an AppWrapper resource + aw := &mcadv1beta2.AppWrapper{ + TypeMeta: metav1.TypeMeta{ + APIVersion: mcadv1beta2.GroupVersion.String(), + Kind: "AppWrapper", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "batchjob-", + Namespace: namespace.Name, + Labels: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name}, + }, + Spec: mcadv1beta2.AppWrapperSpec{ + Components: []mcadv1beta2.AppWrapperComponent{ + { + Template: raw, + }, + }, + }, + } + + appWrapperResource := mcadv1beta2.GroupVersion.WithResource("appwrappers") + awMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(aw) + test.Expect(err).NotTo(HaveOccurred()) + unstruct := unstructured.Unstructured{Object: awMap} + unstructp, err := test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Create(test.Ctx(), &unstruct, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructp.Object, aw) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created AppWrapper %s/%s successfully", aw.Namespace, aw.Name) + + test.T().Logf("Waiting for AppWrapper %s/%s to be running", aw.Namespace, aw.Name) + test.Eventually(AppWrappers(test, namespace), TestTimeoutMedium). + Should(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) + + test.T().Logf("Waiting for AppWrapper %s/%s to complete", aw.Namespace, aw.Name) + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).Should( + ContainElement( + Or( + WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperSucceeded)), + WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperFailed)), + ), + )) + + // Assert the AppWrapper has completed successfully + test.Expect(AppWrappers(test, namespace)(test)). + To(ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperSucceeded)))) + + test.T().Logf("Deleting AppWrapper %s/%s", aw.Namespace, aw.Name) + err = test.Client().Dynamic().Resource(appWrapperResource).Namespace(namespace.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.T().Logf("Waiting for AppWrapper %s/%s to be deleted", aw.Namespace, aw.Name) + test.Eventually(AppWrappers(test, namespace), TestTimeoutShort).Should(BeEmpty()) +} diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go index 34fb742f0..d2e1b5741 100644 --- a/test/e2e/mnist_pytorch_appwrapper_test.go +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -48,9 +48,13 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string, numberOfGpus in // Create Kueue resources resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) - defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + }() clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) - defer test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + }() localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Test configuration diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 443da1522..0b2b01761 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -55,9 +55,13 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int // Create Kueue resources resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) - defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + }() clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) - defer test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + }() CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Create MNIST training script @@ -122,9 +126,13 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number // Create Kueue resources resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) - defer test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + }() clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) - defer test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + }() localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Create MNIST training script diff --git a/test/e2e/support.go b/test/e2e/support.go index dec8915db..ee1564e26 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -39,6 +39,8 @@ func ReadFile(t support.Test, fileName string) []byte { func RemoveCreationTimestamp(t support.Test, rawExtension runtime.RawExtension) runtime.RawExtension { t.T().Helper() patchedRaw := strings.ReplaceAll(string(rawExtension.Raw), `"metadata":{"creationTimestamp":null},`, "") + patchedRaw = strings.ReplaceAll(patchedRaw, `"metadata":{"creationTimestamp":null,`, `"metadata":{`) + patchedRaw = strings.ReplaceAll(patchedRaw, `"creationTimestamp":null,`, "") return runtime.RawExtension{ Raw: []byte(patchedRaw), } From bea5a9729155540d9776a7c5b3b014483ca8545e Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 17 Dec 2024 11:46:58 -0500 Subject: [PATCH 315/369] Update AppWrappers from 0.27.0 to v0.30.0 Changes: + Define aw as a shortname for appwrapper (PR-288) + Promote AppWrapperLabel to v1beta2 api (PR-282) + Ensure consistent resource status if create errors (PR-273) + Flag config error if admissionGP exceeds warmupGP (PR-278) --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 2 ++ go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ba5925a3d..4da106421 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.27.0 +APPWRAPPER_VERSION ?= v0.30.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 75e9b5310..b00296f1b 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.27.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v0.30.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index 02fa812aa..af1c506fa 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -10,6 +10,8 @@ spec: kind: AppWrapper listKind: AppWrapperList plural: appwrappers + shortNames: + - aw singular: appwrapper scope: Namespaced versions: diff --git a/go.mod b/go.mod index 5833ceb62..75453cd48 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/appwrapper v0.27.0 + github.com/project-codeflare/appwrapper v0.30.0 github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924 github.com/ray-project/kuberay/ray-operator v1.2.1 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index f11f3adf1..f38044ed4 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.27.0 h1:WiWw0Hi4rEXuFIEpm8nq1UqJHgVB6YtGcWzRrhRUTyE= -github.com/project-codeflare/appwrapper v0.27.0/go.mod h1:7FpO90DLv0BAq4rwZtXKS9aRRfkR9RvXsj3pgYF0HtQ= +github.com/project-codeflare/appwrapper v0.30.0 h1:tb9LJ/QmC2xyKdM0oVf+WAz9cKIGt3gllDrRgzySgyo= +github.com/project-codeflare/appwrapper v0.30.0/go.mod h1:7FpO90DLv0BAq4rwZtXKS9aRRfkR9RvXsj3pgYF0HtQ= github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924 h1:jM+gYqn8eGmUoeQLGGYxlJgXZ1gbZgB2UtpKU9z0x9s= github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= From 7de1ad4651977f6f3d4d0b3d74def0ada67b2724 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 20 Dec 2024 10:45:27 +0100 Subject: [PATCH 316/369] Rename GPU GitHub runner to avoid version confusion --- .github/workflows/e2e_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index c6cccde5f..63f5c81e3 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -27,7 +27,7 @@ concurrency: jobs: kubernetes-e2e: - runs-on: ubuntu-20.04-4core-gpu + runs-on: gpu-t4-4-core steps: - name: Checkout code From 1a14e909b3bac728e719b321485c2e306cd273af Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 3 Jan 2025 10:56:03 +0100 Subject: [PATCH 317/369] Migrate GitHub runners to newest Ubuntu --- .github/workflows/olm_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index b13f2f894..d1583cce5 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -19,7 +19,7 @@ concurrency: jobs: kubernetes-olm-upgrade: - runs-on: ubuntu-20.04-4core + runs-on: ubuntu-latest-4core timeout-minutes: 60 env: OLM_VERSION: v0.25.0 From 5e354dcfaf0ec9daffe3fba8798b9f117faf467b Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Mon, 6 Jan 2025 06:46:47 -0800 Subject: [PATCH 318/369] [CARRY]: Fix CVE-2024-45338 - update golang.org/x/net --- go.mod | 8 ++++---- go.sum | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 75453cd48..2f2933948 100644 --- a/go.mod +++ b/go.mod @@ -95,11 +95,11 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index f38044ed4..e71bd4404 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= @@ -303,8 +303,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -314,8 +314,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -340,20 +340,20 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From d459dd6e54b17e4aaf57bd2a6d0654b94ee8add0 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Wed, 8 Jan 2025 15:22:23 +0000 Subject: [PATCH 319/369] Update dependency versions for release v1.13.0 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9920c9d94..a6ce3c307 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.12.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.12.0) | -| CodeFlare-SDK | [v0.24.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.24.0) | -| AppWrapper | [v0.27.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.27.0) | +| CodeFlare Operator | [v1.13.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.13.0) | +| CodeFlare-SDK | [v0.25.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.25.0) | +| AppWrapper | [v0.30.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.30.0) | | KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | | Kueue | [v0.8.3](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.3) | diff --git a/config/manager/params.env b/config/manager/params.env index a8879b8da..9acafde5b 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.12.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.13.0 namespace=opendatahub From 1945b65169b519dc9918d1039bb769a49f336c80 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 9 Jan 2025 12:53:45 +0000 Subject: [PATCH 320/369] Remove auto-merge workflow --- .github/workflows/upstream-downstream-sync.yml | 18 ------------------ README.md | 2 +- 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 .github/workflows/upstream-downstream-sync.yml diff --git a/.github/workflows/upstream-downstream-sync.yml b/.github/workflows/upstream-downstream-sync.yml deleted file mode 100644 index cd78dbab5..000000000 --- a/.github/workflows/upstream-downstream-sync.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Upstream to Downstream - Release Auto-Merge -on: - release: - types: - - published - workflow_dispatch: - -jobs: - trigger-auto-sync: - runs-on: ubuntu-latest - steps: - - name: call-sync-workflow - run: | - gh workflow run auto-merge-sync.yaml --repo github.com/red-hat-data-services/codeflare-operator --ref main - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: - bash diff --git a/README.md b/README.md index a6ce3c307..7c0076fa5 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ For ODH tests additional environment variables are needed: 2. Once all jobs within the action are completed, verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. 3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. 4. Once PR is merged, announce the new release in slack and mail lists, if any. -5. Release automation should auto-merge changes to [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Verify the [workflow](https://github.com/red-hat-data-services/codeflare-operator/actions/workflows/auto-merge-sync.yaml) ran successfully and review the new merge-commit and commit history. Same for the [Red Hat CodeFlare Operator repo](https://github.com/red-hat-data-services/codeflare-operator), while also ensuring changes are in the latest `rhoai` release branch. - If the auto-merge fails, conflicts must be resolved and force pushed manually to each downstream repository and release branch. +5. Trigger the [auto-merge-sync workflow](https://github.com/red-hat-data-services/codeflare-operator/actions/workflows/auto-merge-sync.yaml) and verify it ran successfully. This will sync changes to the [ODH CodeFlare-Operator repo](https://github.com/opendatahub-io/codeflare-operator), and the [Red Hat CodeFlare Operator repo](https://github.com/red-hat-data-services/codeflare-operator). Please review the new merge-commit and commit history, and verify changes are also in the latest `rhoai` release branch. - If the auto-merge fails, conflicts must be resolved and force pushed manually to each downstream repository and release branch. 6. In ODH/CFO verify that the [Build and Push action](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/build-and-push.yaml) was triggered and ran successfully. 7. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. 8. Run [ODH CodeFlare Operator release workflow](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/odh-release.yml) to produce ODH CodeFlare Operator release. From 864bb417d19cbfc2be3ede0fae85d4b63ede8d54 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 17 Jan 2025 13:14:26 +0100 Subject: [PATCH 321/369] Don't delete Ray head Pod when ImagePullSecret is provided --- .github/workflows/e2e_tests.yaml | 2 +- go.mod | 2 +- go.sum | 4 +- pkg/controllers/raycluster_controller.go | 7 +- pkg/controllers/raycluster_controller_test.go | 67 +++++++++++++++++++ test/e2e/mnist_rayjob_raycluster_test.go | 42 +++++++++++- 6 files changed, 116 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 63f5c81e3..634f1484b 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -89,7 +89,7 @@ jobs: export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} set -euo pipefail - go test -timeout 60m -v -skip "^Test.*Cpu$" ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 120m -v -skip "^Test.*Cpu$" ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' diff --git a/go.mod b/go.mod index 2f2933948..80763df0f 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/appwrapper v0.30.0 - github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924 + github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a github.com/ray-project/kuberay/ray-operator v1.2.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 diff --git a/go.sum b/go.sum index e71bd4404..d87400841 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.30.0 h1:tb9LJ/QmC2xyKdM0oVf+WAz9cKIGt3gllDrRgzySgyo= github.com/project-codeflare/appwrapper v0.30.0/go.mod h1:7FpO90DLv0BAq4rwZtXKS9aRRfkR9RvXsj3pgYF0HtQ= -github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924 h1:jM+gYqn8eGmUoeQLGGYxlJgXZ1gbZgB2UtpKU9z0x9s= -github.com/project-codeflare/codeflare-common v0.0.0-20241216183607-222395d38924/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= +github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a h1:1F5xsxncIL5Bpboup8d5osQ8iWy/hzkCTtGSBZM2tQM= +github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 902570bba..55bd34657 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -219,8 +219,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{RequeueAfter: requeueTime}, err } - if err := r.deleteHeadPodIfMissingImagePullSecrets(ctx, cluster); err != nil { - return ctrl.Result{RequeueAfter: requeueTime}, err + if len(cluster.Spec.HeadGroupSpec.Template.Spec.ImagePullSecrets) == 0 { + // Delete head pod only if user doesn't specify own imagePullSecrets and imagePullSecrets from OAuth ServiceAccount are not present in the head Pod + if err := r.deleteHeadPodIfMissingImagePullSecrets(ctx, cluster); err != nil { + return ctrl.Result{RequeueAfter: requeueTime}, err + } } _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) diff --git a/pkg/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go index e0a7e969c..db49adf98 100644 --- a/pkg/controllers/raycluster_controller_test.go +++ b/pkg/controllers/raycluster_controller_test.go @@ -192,6 +192,73 @@ var _ = Describe("RayCluster controller", func() { }).WithTimeout(time.Second * 10).Should(Satisfy(errors.IsNotFound)) }) + It("should not delete the head pod if RayCluster CR provides image pull secrets", func(ctx SpecContext) { + By("creating an instance of the RayCluster CR with imagePullSecret") + rayclusterWithPullSecret := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret-cluster", + Namespace: namespaceName, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "custom-pull-secret"}}, + Containers: []corev1.Container{}, + }, + }, + RayStartParams: map[string]string{}, + }, + }, + } + _, err := rayClient.RayV1().RayClusters(namespaceName).Create(ctx, rayclusterWithPullSecret, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(rayclusterWithPullSecret), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + + headPodName := "head-pod" + headPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: headPodName, + Namespace: namespaceName, + Labels: map[string]string{ + "ray.io/node-type": "head", + "ray.io/cluster": rayclusterWithPullSecret.Name, + }, + }, + Spec: corev1.PodSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{ + {Name: "custom-pull-secret"}, + }, + Containers: []corev1.Container{ + { + Name: "head-container", + Image: "busybox", + }, + }, + }, + } + _, err = k8sClient.CoreV1().Pods(namespaceName).Create(ctx, headPod, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + Eventually(func() (*corev1.Pod, error) { + return k8sClient.CoreV1().Pods(namespaceName).Get(ctx, headPodName, metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + + sa, err := k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(rayclusterWithPullSecret), metav1.GetOptions{}) + Expect(err).To(Not(HaveOccurred())) + + sa.ImagePullSecrets = append(sa.ImagePullSecrets, corev1.LocalObjectReference{Name: "test-image-pull-secret"}) + _, err = k8sClient.CoreV1().ServiceAccounts(namespaceName).Update(ctx, sa, metav1.UpdateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + Consistently(func() (*corev1.Pod, error) { + return k8sClient.CoreV1().Pods(namespaceName).Get(ctx, headPodName, metav1.GetOptions{}) + }).WithTimeout(time.Second * 5).Should(Not(BeNil())) + }) + It("should remove CRB when the RayCluster is deleted", func(ctx SpecContext) { foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 0b2b01761..3c24c55d2 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -211,6 +211,44 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number test.Eventually(AppWrappers(test, namespace), TestTimeoutShort).Should(BeEmpty()) } +// Verifying https://github.com/project-codeflare/codeflare-operator/issues/649 +func TestRayClusterImagePullSecret(t *testing.T) { + test := With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Create Kueue resources + resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) + }() + clusterQueue := createClusterQueue(test, resourceFlavor, 0) + defer func() { + _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) + }() + CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) + + // Create MNIST training script + mnist := constructMNISTConfigMap(test, namespace) + mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) + + // Create RayCluster with imagePullSecret and assign it to the localqueue + rayCluster := constructRayCluster(test, namespace, mnist, 0) + rayCluster.Spec.HeadGroupSpec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "custom-pull-secret"}} + rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name) + + test.T().Logf("Waiting for RayCluster %s/%s to be running", rayCluster.Namespace, rayCluster.Name) + test.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) +} + +// Helper functions + func constructMNISTConfigMap(test Test, namespace *corev1.Namespace) *corev1.ConfigMap { return &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -274,11 +312,11 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), + corev1.ResourceMemory: resource.MustParse("2G"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("2G"), + corev1.ResourceMemory: resource.MustParse("4G"), }, }, }, From 028b67d7e3c753eb01d32d0b981a050b1960a66c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 21 Jan 2025 12:54:43 +0000 Subject: [PATCH 322/369] Upgrade openshift/client-go version --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 80763df0f..a252cffdf 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/onsi/gomega v1.33.1 github.com/open-policy-agent/cert-controller v0.10.1 github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 - github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 - github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c + github.com/openshift/api v0.0.0-20240509165319-a8a5b79cc248 + github.com/openshift/client-go v0.0.0-20240510131258-f646d5f29250 github.com/project-codeflare/appwrapper v0.30.0 github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a github.com/ray-project/kuberay/ray-operator v1.2.1 diff --git a/go.sum b/go.sum index d87400841..4b67a06bd 100644 --- a/go.sum +++ b/go.sum @@ -214,10 +214,10 @@ github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeL github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= github.com/openshift-online/ocm-sdk-go v0.1.411/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y= -github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 h1:e3zIxk67/kiABxGFfFVECqJ4FcQRG5DPF8lgDV9f+MM= -github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= -github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= -github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= +github.com/openshift/api v0.0.0-20240509165319-a8a5b79cc248 h1:W0FcSznRzX/R/5AAfaMG1OIdl++6UtwWi/Uy3M45oQM= +github.com/openshift/api v0.0.0-20240509165319-a8a5b79cc248/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= +github.com/openshift/client-go v0.0.0-20240510131258-f646d5f29250 h1:WQ0e89xebsNeChdYw8QrxggYkFtEQOaLBEKJDkMmcUk= +github.com/openshift/client-go v0.0.0-20240510131258-f646d5f29250/go.mod h1:tjGjTE59N1+5W0YE4Wqf0zJpFLIY4d+EpMihDKOn1oA= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From d88898524359fff120576fed2ad41ead6c60c8f9 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 21 Jan 2025 16:03:51 +0100 Subject: [PATCH 323/369] Upgrade KubeRay to 1.2.2 --- Makefile | 2 +- go.mod | 9 +++------ go.sum | 12 ++++++------ 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 4da106421..eed5ff1ac 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ KUEUE_VERSION ?= v0.8.3 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.1.0 +KUBERAY_VERSION ?= v1.2.2 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/go.mod b/go.mod index a252cffdf..db2f96076 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,11 @@ require ( github.com/onsi/gomega v1.33.1 github.com/open-policy-agent/cert-controller v0.10.1 github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 - github.com/openshift/api v0.0.0-20240509165319-a8a5b79cc248 - github.com/openshift/client-go v0.0.0-20240510131258-f646d5f29250 + github.com/openshift/api v0.0.0-20240904015708-69df64132c91 + github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 github.com/project-codeflare/appwrapper v0.30.0 github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a - github.com/ray-project/kuberay/ray-operator v1.2.1 + github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 k8s.io/api v0.30.2 @@ -38,9 +38,6 @@ replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // This replace directive deal with the backlevel ODH kueue version replace sigs.k8s.io/kueue v0.8.3 => github.com/opendatahub-io/kueue v0.8.3 -// This replace directive deal with the unaligned Ray operator version transitively pulled by Kueue 0.8.3 -replace github.com/ray-project/kuberay/ray-operator v1.2.1 => github.com/ray-project/kuberay/ray-operator v1.1.1 - require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 4b67a06bd..68d592223 100644 --- a/go.sum +++ b/go.sum @@ -214,10 +214,10 @@ github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeL github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= github.com/openshift-online/ocm-sdk-go v0.1.411/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y= -github.com/openshift/api v0.0.0-20240509165319-a8a5b79cc248 h1:W0FcSznRzX/R/5AAfaMG1OIdl++6UtwWi/Uy3M45oQM= -github.com/openshift/api v0.0.0-20240509165319-a8a5b79cc248/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= -github.com/openshift/client-go v0.0.0-20240510131258-f646d5f29250 h1:WQ0e89xebsNeChdYw8QrxggYkFtEQOaLBEKJDkMmcUk= -github.com/openshift/client-go v0.0.0-20240510131258-f646d5f29250/go.mod h1:tjGjTE59N1+5W0YE4Wqf0zJpFLIY4d+EpMihDKOn1oA= +github.com/openshift/api v0.0.0-20240904015708-69df64132c91 h1:PfPpMwHR8iAxQuLpQt+x9f3PAIeSPG2BLQd69p+NQHw= +github.com/openshift/api v0.0.0-20240904015708-69df64132c91/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM= +github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 h1:mLJlN8PdZgI5oxwXctB0/IF/2c7F29R5ru1dCLVzEyk= +github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202/go.mod h1:D3yfotGqRY1LWaHcoSCeadhdyvfrQWYJXui1M+JqKvY= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -237,8 +237,8 @@ github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVho github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/ray-project/kuberay/ray-operator v1.1.1 h1:mVOA1ddS9aAsPvhhHrpf0ZXgTzccIAyTbeYeDqtcfAk= -github.com/ray-project/kuberay/ray-operator v1.1.1/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= +github.com/ray-project/kuberay/ray-operator v1.2.2 h1:wj4qe9SmJfD1ubgEaVPuAsnU/WFDvremzR8j3JslBdk= +github.com/ray-project/kuberay/ray-operator v1.2.2/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From a6794de0f9e3f6dd76ccfb3aa09f96d53f8d9e02 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:43:47 +0000 Subject: [PATCH 324/369] Update dependency versions for release v1.14.0 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7c0076fa5..f4eb541e1 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.13.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.13.0) | -| CodeFlare-SDK | [v0.25.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.25.0) | +| CodeFlare Operator | [v1.14.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.14.0) | +| CodeFlare-SDK | [v0.26.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.26.0) | | AppWrapper | [v0.30.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.30.0) | -| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | +| KubeRay | [v1.2.2](https://github.com/opendatahub-io/kuberay/releases/tag/v1.2.2) | | Kueue | [v0.8.3](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.3) | diff --git a/config/manager/params.env b/config/manager/params.env index 9acafde5b..b28efad19 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.13.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.14.0 namespace=opendatahub From 9519938484cc297821b1133b3f399e101d17df07 Mon Sep 17 00:00:00 2001 From: oksanabaza Date: Tue, 14 Jan 2025 15:24:53 +0000 Subject: [PATCH 325/369] Add automated update step for component metadata file in CodeFlare release workflow --- .github/workflows/project-codeflare-release.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index d946d5c55..e9d5f0b85 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -165,3 +165,20 @@ jobs: echo "Kueue release with version ${{ github.event.inputs.kueue-version }} does not exist. Please select an existing version." exit 1 fi + + generate-component-metadata: + needs: [check-appwrapper-version, check-kuberay-version, check-kueue-version] + runs-on: ubuntu-latest + + steps: + - name: Generate component_metadata.yaml + run: | + cat < config/component_metadata.yaml + releases: + - name: CodeFlare Operator + version: ${{ github.event.inputs.operator-version }} + repoUrl: https://github.com/project-codeflare/codeflare-operator + EOL + + - name: Verify generated file + run: cat config/component_metadata.yaml From ed1339f8462378f4725b0b0a667986c79562b0a7 Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 28 Jan 2025 08:59:00 -0500 Subject: [PATCH 326/369] update codeflare common to pick up fix for nvidia gpu setup --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index db2f96076..b80c6c24e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 github.com/project-codeflare/appwrapper v0.30.0 - github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a + github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 diff --git a/go.sum b/go.sum index 68d592223..45e5d663b 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v0.30.0 h1:tb9LJ/QmC2xyKdM0oVf+WAz9cKIGt3gllDrRgzySgyo= github.com/project-codeflare/appwrapper v0.30.0/go.mod h1:7FpO90DLv0BAq4rwZtXKS9aRRfkR9RvXsj3pgYF0HtQ= -github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a h1:1F5xsxncIL5Bpboup8d5osQ8iWy/hzkCTtGSBZM2tQM= -github.com/project-codeflare/codeflare-common v0.0.0-20250117134355-5748d670cd4a/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= +github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b h1:MOmv/aLx/kcHd7PBErx8XNSTW180s8Slf/uVM0uV4rw= +github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From 33dba993b33d81d2c0fe9da8de0dfa0b61b9ef07 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 16 Jan 2025 13:50:57 -0500 Subject: [PATCH 327/369] bump AppWrapper to 1.0.0 + Update to Kueue 0.10 + Remove deprecated kube-rbac-proxy + Refactor workload controller to prepare to upstream to Kueue --- Makefile | 4 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 24 +---- go.mod | 58 +++++------ go.sum | 117 ++++++++++++----------- 5 files changed, 100 insertions(+), 105 deletions(-) diff --git a/Makefile b/Makefile index eed5ff1ac..eee82d9cb 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,12 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v0.30.0 +APPWRAPPER_VERSION ?= v1.0.0 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the default version of Kueue (used for testing) -KUEUE_VERSION ?= v0.8.3 +KUEUE_VERSION ?= v0.10.0 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index b00296f1b..798f5fbbc 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v0.30.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v1.0.0 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index af1c506fa..744b87949 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.16.5 name: appwrappers.workload.codeflare.dev spec: group: workload.codeflare.dev @@ -178,13 +178,11 @@ spec: description: |- Conditions hold the latest available observations of the Component's current state. - The type of the condition could be: - - ResourcesDeployed: The component is deployed on the cluster items: - description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t // other fields\n\t}" + description: Condition contains details for one aspect of the current state of this API Resource. properties: lastTransitionTime: description: |- @@ -225,12 +223,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -278,17 +271,15 @@ spec: description: |- Conditions hold the latest available observations of the AppWrapper current state. - The type of the condition could be: - - QuotaReserved: The AppWrapper was admitted by Kueue and has quota allocated to it - ResourcesDeployed: The contained resources are deployed (or being deployed) on the cluster - PodsReady: All pods of the contained resources are in the Ready or Succeeded state - Unhealthy: One or more of the contained resources is unhealthy - DeletingResources: The contained resources are in the process of being deleted from the cluster items: - description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t // other fields\n\t}" + description: Condition contains details for one aspect of the current state of this API Resource. properties: lastTransitionTime: description: |- @@ -329,12 +320,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/go.mod b/go.mod index b80c6c24e..d2378e1fe 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,33 @@ module github.com/project-codeflare/codeflare-operator -go 1.22.4 +go 1.23.0 require ( github.com/go-logr/logr v1.4.2 - github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega v1.33.1 - github.com/open-policy-agent/cert-controller v0.10.1 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + github.com/open-policy-agent/cert-controller v0.12.0 github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 - github.com/project-codeflare/appwrapper v0.30.0 + github.com/project-codeflare/appwrapper v1.0.0 github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 - k8s.io/api v0.30.2 - k8s.io/apiextensions-apiserver v0.29.6 - k8s.io/apimachinery v0.30.2 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + k8s.io/api v0.31.4 + k8s.io/apiextensions-apiserver v0.31.2 + k8s.io/apimachinery v0.31.4 k8s.io/client-go v11.0.0+incompatible - k8s.io/component-base v0.29.6 + k8s.io/component-base v0.31.4 k8s.io/klog/v2 v2.130.1 - k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 - sigs.k8s.io/controller-runtime v0.17.5 - sigs.k8s.io/kueue v0.8.3 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/kueue v0.10.0 sigs.k8s.io/yaml v1.4.0 ) -replace k8s.io/client-go => k8s.io/client-go v0.29.2 +replace k8s.io/client-go => k8s.io/client-go v0.31.4 replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 @@ -35,8 +35,7 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 -// This replace directive deal with the backlevel ODH kueue version -replace sigs.k8s.io/kueue v0.8.3 => github.com/opendatahub-io/kueue v0.8.3 +replace sigs.k8s.io/kueue v0.10.0 => github.com/opendatahub-io/kueue v0.10.0-odh require ( github.com/aymerick/douceur v0.2.0 // indirect @@ -47,9 +46,8 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -63,7 +61,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -72,7 +70,8 @@ require ( github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect - github.com/kubeflow/training-operator v1.7.0 // indirect + github.com/kubeflow/mpi-operator v0.6.0 // indirect + github.com/kubeflow/training-operator v1.8.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -83,13 +82,14 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.4 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.57.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.33.0 // indirect @@ -97,16 +97,18 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/time v0.6.0 // indirect + golang.org/x/tools v0.26.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.29.6 // indirect - k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect - sigs.k8s.io/jobset v0.5.2 // indirect + k8s.io/apiserver v0.31.4 // indirect + k8s.io/component-helpers v0.31.4 // indirect + k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect + sigs.k8s.io/jobset v0.7.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.3 // indirect ) diff --git a/go.sum b/go.sum index 45e5d663b..3b5b7025c 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -109,8 +109,8 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -157,10 +157,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeflow/mpi-operator v0.5.0 h1:XvBwyXXQ9103DNMa22sxsaQlaktvaT2LY/g0UniGn5U= -github.com/kubeflow/mpi-operator v0.5.0/go.mod h1:SeZQJW8KJxSTWD++eQYKRFpoDg1v8yrdC6fjx2/3mG0= -github.com/kubeflow/training-operator v1.7.0 h1:Zh61GlOWrlRi4UFOtJeV+/5REo/OndhwQ25KYd0llzc= -github.com/kubeflow/training-operator v1.7.0/go.mod h1:BZCLX1h06wY3YSeSZZcGYAqI9/nVi7isVCRkfgZe9nE= +github.com/kubeflow/mpi-operator v0.6.0 h1:RzzG03oyQIiHe/SH2k19N+Hp4QkxEsG2l9G1T7US+ys= +github.com/kubeflow/mpi-operator v0.6.0/go.mod h1:lu3cj3ESq3SdS29nbHC26TuXiTYYQYTHJ6fcy5Xv20c= +github.com/kubeflow/training-operator v1.8.1 h1:mVvoBSCInbMBX0gGSn4+Ihj3ycwoLO1iEAw+0qPhQfc= +github.com/kubeflow/training-operator v1.8.1/go.mod h1:T6I15h1S09ncH5C6St/QEC7Dy6dpHZA5sPFo+VoJAvE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -193,23 +193,23 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= -github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= -github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= -github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= -github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/open-policy-agent/cert-controller v0.12.0 h1:RKXlBafMcCh+++I1geJetXo77tAjyb4542DQc/+aZIw= +github.com/open-policy-agent/cert-controller v0.12.0/go.mod h1:N5bCFXdAXMYx0PdS6ZQ9lrDQQMz+F6deoChym6VleXw= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a h1:gQtOJ50XFyL2Xh3lDD9zP4KQ2PY4mZKQ9hDcWc81Sp8= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a/go.mod h1:tI7nc6H6os2UYZRvSm9Y7bq4oMoXqhwA0WfnqKpoAgc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opendatahub-io/kueue v0.8.3 h1:MLkHCmIrQR1KM1IcPiGuoEAT3Y+ZTs7493sMkmSUMow= -github.com/opendatahub-io/kueue v0.8.3/go.mod h1:jzRyUhAXHIpEPjt4pMx79t/Cg1g29GlNZY6wiLJE2YI= +github.com/opendatahub-io/kueue v0.10.0-odh h1:egnWIOjXSnaUOzjmXyOM/P1UPTK5OlaltSFf7OJ1qRg= +github.com/opendatahub-io/kueue v0.10.0-odh/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= @@ -222,14 +222,15 @@ github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPf github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v0.30.0 h1:tb9LJ/QmC2xyKdM0oVf+WAz9cKIGt3gllDrRgzySgyo= -github.com/project-codeflare/appwrapper v0.30.0/go.mod h1:7FpO90DLv0BAq4rwZtXKS9aRRfkR9RvXsj3pgYF0HtQ= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/project-codeflare/appwrapper v1.0.0 h1:tKvBXxaIE5RwzUC8YZvxcnbuz4JUbiuLjo1IWO4Mciw= +github.com/project-codeflare/appwrapper v1.0.0/go.mod h1:pVCWURfk9DOa/4ig7z91PJWRMlsDchhQVmu7mB32L48= github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b h1:MOmv/aLx/kcHd7PBErx8XNSTW180s8Slf/uVM0uV4rw= github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= -github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= -github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= @@ -256,6 +257,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -275,8 +278,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -354,8 +357,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -369,8 +372,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -398,14 +401,16 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -425,20 +430,22 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= -k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apiextensions-apiserver v0.29.6 h1:tUu1N6Zt9GT8KVcPF5aGDqfISz1mveM4yFh7eL5bxmE= -k8s.io/apiextensions-apiserver v0.29.6/go.mod h1:iw1EbwZat08I219qrQKoFMHGo7J9KxPqMpVKxCbNbCs= +k8s.io/api v0.31.4 h1:I2QNzitPVsPeLQvexMEsj945QumYraqv9m74isPDKhM= +k8s.io/api v0.31.4/go.mod h1:d+7vgXLvmcdT1BCo79VEgJxHHryww3V5np2OYTr6jdw= +k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0= +k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/apiserver v0.29.6 h1:JxgDbpgahOgqoDOf+zVl2mI+rQcHcLQnK6YhhtsjbNs= -k8s.io/apiserver v0.29.6/go.mod h1:HrQwfPWxhwEa+n8/+5YwSF5yT2WXbeyFjqq6KEXHTX8= -k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= -k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM= +k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apiserver v0.31.4 h1:JbtnTaXVYEAYIHJil6Wd74Wif9sd8jVcBw84kwEmp7o= +k8s.io/apiserver v0.31.4/go.mod h1:JJjoTjZ9PTMLdIFq7mmcJy2B9xLN3HeAUebW6xZyIP0= +k8s.io/client-go v0.31.4 h1:t4QEXt4jgHIkKKlx06+W3+1JOwAFU/2OPiOo7H92eRQ= +k8s.io/client-go v0.31.4/go.mod h1:kvuMro4sFYIa8sulL5Gi5GFqUPvfH2O/dXuKstbaaeg= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.29.6 h1:XkVJI67FvBgNb/3kKqvaGKokxUrIR0RrksCPNI+JYCs= -k8s.io/component-base v0.29.6/go.mod h1:kIahZm8aw9lV8Vw17LF89REmeBrv5+QEl3v7HsrmITY= +k8s.io/component-base v0.31.4 h1:wCquJh4ul9O8nNBSB8N/o8+gbfu3BVQkVw9jAUY/Qtw= +k8s.io/component-base v0.31.4/go.mod h1:G4dgtf5BccwiDT9DdejK0qM6zTK0jwDGEKnCmb9+u/s= +k8s.io/component-helpers v0.31.4 h1:pqokuXozyWVrVBMmx0AMcKqNWqXhR00OZvpAE5hG5NM= +k8s.io/component-helpers v0.31.4/go.mod h1:Ddq5GYRK/1uNoPNgJh9N5osPutvBweQEcIG6b8kcvgQ= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -447,27 +454,27 @@ k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= -k8s.io/kube-aggregator v0.28.3/go.mod h1:5DyLevbRTcWnT1f9b+lB3BfbXC1w7gDa/OtB6kKInCw= +k8s.io/kube-aggregator v0.31.2 h1:Uw1zUP2D/4wiSjKWVVzSOcCGLuW/+IdRwjjC0FJooYU= +k8s.io/kube-aggregator v0.31.2/go.mod h1:41/VIXH+/Qcg9ERNAY6bRF/WQR6xL1wFgYagdHac1X4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlLosk/JlfP3xQLNkQlHJeXw= -k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f h1:bnWtxXWdAl5bVOCEPoNdvMkyj6cTW3zxHuwKIakuV9w= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f/go.mod h1:G0W3eI9gG219NHRq3h5uQaRBl4pj4ZpwzRP5ti8y770= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= -k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.17.5 h1:1FI9Lm7NiOOmBsgTV36/s2XrEFXnO2C4sbg/Zme72Rw= -sigs.k8s.io/controller-runtime v0.17.5/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= -sigs.k8s.io/jobset v0.5.2 h1:276q5Pi/ErLYj+GQ0ydEXR6tx3LwBhEzHLQv+k8bYF4= -sigs.k8s.io/jobset v0.5.2/go.mod h1:Vg99rj/6OoGvy1uvywGEHOcVLCWWJYkJtisKqdWzcFw= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= +sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= +sigs.k8s.io/jobset v0.7.1 h1:m28IaaKrQyZ8qa0Q7jK3U5/6TEW+27QPALjlalLP/0A= +sigs.k8s.io/jobset v0.7.1/go.mod h1:cbBuQ6QrTU88x5PrDqcC5AI28YyTSQCIcRTLNTJoHDE= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.4.3 h1:sCP7Vv3xx/CWIuTPVN38lUPx0uw0lcLfzaiDa8Ja01A= +sigs.k8s.io/structured-merge-diff/v4 v4.4.3/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= From ee5038c880d9ba50a0a0482b93c7786c17ac8bc1 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 22 Jan 2025 10:11:18 -0500 Subject: [PATCH 328/369] update AppWrapper RBACs for Kueue 0.10.0 --- config/rbac/role.yaml | 8 ++++++++ pkg/controllers/appwrapper_controller.go | 3 +++ 2 files changed, 11 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 598b4dafb..1f616d40d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -14,6 +14,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch - apiGroups: - "" resources: diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index c58516dad..b160a7f97 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -42,6 +42,9 @@ package controllers // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch +// permissions needed by Kueue's base webhook +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch + // permission to watch nodes and edit clusterqueues for Autopilot integration //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch //+kubebuilder:rbac:groups=kueue.x-k8s.io,resources=clusterqueues,verbs=get;list;watch;update;patch From 14ac0fc8f190b9c75f3b19de7293393123987e92 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 22 Jan 2025 10:22:03 -0500 Subject: [PATCH 329/369] workaround kuberay changes until a proper fix can be implemented --- go.mod | 3 +++ go.sum | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d2378e1fe..f8fce0d3c 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,9 @@ replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 replace sigs.k8s.io/kueue v0.10.0 => github.com/opendatahub-io/kueue v0.10.0-odh +// Workaround deprecation of cluster.Status.State in ray-operator v1.2.2 +replace github.com/ray-project/kuberay/ray-operator v1.2.2 => github.com/ray-project/kuberay/ray-operator v1.2.1 + require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 3b5b7025c..c68d76c7b 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,8 @@ github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVho github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/ray-project/kuberay/ray-operator v1.2.2 h1:wj4qe9SmJfD1ubgEaVPuAsnU/WFDvremzR8j3JslBdk= -github.com/ray-project/kuberay/ray-operator v1.2.2/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= +github.com/ray-project/kuberay/ray-operator v1.2.1 h1:H7ofodGclghsU2TxbDHs+gvqvsOp5DJ/vAPGySL1DIE= +github.com/ray-project/kuberay/ray-operator v1.2.1/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From 2f499ba664388367a2170f18cc8cc7cc9be62a47 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 27 Jan 2025 11:36:49 -0500 Subject: [PATCH 330/369] Pull in Kueue 0.10.1 --- Makefile | 2 +- go.mod | 4 +--- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index eee82d9cb..9ab3481e8 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the default version of Kueue (used for testing) -KUEUE_VERSION ?= v0.10.0 +KUEUE_VERSION ?= v0.10.1 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) diff --git a/go.mod b/go.mod index f8fce0d3c..6243acde8 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.19.3 - sigs.k8s.io/kueue v0.10.0 + sigs.k8s.io/kueue v0.10.1 sigs.k8s.io/yaml v1.4.0 ) @@ -35,8 +35,6 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 -replace sigs.k8s.io/kueue v0.10.0 => github.com/opendatahub-io/kueue v0.10.0-odh - // Workaround deprecation of cluster.Status.State in ray-operator v1.2.2 replace github.com/ray-project/kuberay/ray-operator v1.2.2 => github.com/ray-project/kuberay/ray-operator v1.2.1 diff --git a/go.sum b/go.sum index c68d76c7b..83697cd32 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,6 @@ github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8ab github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a/go.mod h1:tI7nc6H6os2UYZRvSm9Y7bq4oMoXqhwA0WfnqKpoAgc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opendatahub-io/kueue v0.10.0-odh h1:egnWIOjXSnaUOzjmXyOM/P1UPTK5OlaltSFf7OJ1qRg= -github.com/opendatahub-io/kueue v0.10.0-odh/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= @@ -471,6 +469,8 @@ sigs.k8s.io/jobset v0.7.1/go.mod h1:cbBuQ6QrTU88x5PrDqcC5AI28YyTSQCIcRTLNTJoHDE= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kueue v0.10.1 h1:td+Nae1z9L4IrkqJKOIMsKp2TmwUAvk/hWKhSaFupwk= +sigs.k8s.io/kueue v0.10.1/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.3 h1:sCP7Vv3xx/CWIuTPVN38lUPx0uw0lcLfzaiDa8Ja01A= From 3d3fde06f308f047a77d2980654db593fa407e72 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 27 Jan 2025 13:13:38 -0500 Subject: [PATCH 331/369] Go 1.23 workaround --- Dockerfile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 24d923734..c52366a7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,22 @@ # Build the manager binary +FROM registry-proxy.engineering.redhat.com/rh-osbs/openshift-golang-builder:v1.23@sha256:ca0c771ecd4f606986253f747e2773fe2960a6b5e8e7a52f6a4797b173ac7f56 AS golang -FROM registry.access.redhat.com/ubi8/go-toolset:1.22@sha256:780ab5f3874a6e2b1e04bb3719e614e835af3f8ab150922d6e84c2f9fd2bdb27 AS builder +FROM registry.redhat.io/ubi8/ubi@sha256:fd3bf22d0593e2ed26a1c74ce161c52295711a67de677b5938c87704237e49b0 AS builder +ARG GOLANG_VERSION=1.23.0 + +# Install system dependencies +RUN dnf upgrade -y && dnf install -y \ + gcc \ + make \ + openssl-devel \ + git \ + && dnf clean all && rm -rf /var/cache/yum + +# Install Go +ENV PATH=/usr/local/go/bin:$PATH + +COPY --from=golang /usr/lib/golang /usr/local/go +# End of Go versioning workaround WORKDIR /workspace # Copy the Go Modules manifests From 103580c3cd21047740de0f97af2d5b74184ca820 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 27 Jan 2025 13:29:09 -0500 Subject: [PATCH 332/369] use publicly available images --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c52366a7c..959b0146c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ # Build the manager binary -FROM registry-proxy.engineering.redhat.com/rh-osbs/openshift-golang-builder:v1.23@sha256:ca0c771ecd4f606986253f747e2773fe2960a6b5e8e7a52f6a4797b173ac7f56 AS golang -FROM registry.redhat.io/ubi8/ubi@sha256:fd3bf22d0593e2ed26a1c74ce161c52295711a67de677b5938c87704237e49b0 AS builder +# BEGIN -- workaround lack of go-toolset for golang 1.23 +ARG GOLANG_IMAGE=golang:1.21 +FROM ${GOLANG_IMAGE} AS golang + +FROM registry.access.redhat.com/ubi8/ubi@sha256:fd3bf22d0593e2ed26a1c74ce161c52295711a67de677b5938c87704237e49b0 AS builder ARG GOLANG_VERSION=1.23.0 # Install system dependencies @@ -15,7 +18,7 @@ RUN dnf upgrade -y && dnf install -y \ # Install Go ENV PATH=/usr/local/go/bin:$PATH -COPY --from=golang /usr/lib/golang /usr/local/go +COPY --from=golang /usr/local/go /usr/local/go # End of Go versioning workaround WORKDIR /workspace From ef70cf92c3aeeeaeb301e9993eff80df7721999b Mon Sep 17 00:00:00 2001 From: David Grove Date: Tue, 28 Jan 2025 08:47:16 -0500 Subject: [PATCH 333/369] fix golang image repo/tag --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 959b0146c..eb0b7ce82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Build the manager binary # BEGIN -- workaround lack of go-toolset for golang 1.23 -ARG GOLANG_IMAGE=golang:1.21 +ARG GOLANG_IMAGE=docker.io/library/golang:1.23 FROM ${GOLANG_IMAGE} AS golang FROM registry.access.redhat.com/ubi8/ubi@sha256:fd3bf22d0593e2ed26a1c74ce161c52295711a67de677b5938c87704237e49b0 AS builder From 1895a05f57a2187a1651afab7b732bc38f5b3cba Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 6 Feb 2025 16:15:26 +0100 Subject: [PATCH 334/369] Create oauth objects when RayCluster spec suspend is false --- pkg/controllers/raycluster_controller.go | 8 ++++++-- pkg/controllers/raycluster_controller_test.go | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 55bd34657..02c1a307d 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -193,7 +193,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - if cluster.Status.State != "suspended" && isRayDashboardOAuthEnabled(r.Config) && r.IsOpenShift { + if !isRayClusterSuspended(cluster) && isRayDashboardOAuthEnabled(r.Config) && r.IsOpenShift { logger.Info("Creating OAuth Objects") _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { @@ -239,7 +239,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{RequeueAfter: requeueTime}, err } - } else if cluster.Status.State != "suspended" && !isRayDashboardOAuthEnabled(r.Config) && !r.IsOpenShift { + } else if !isRayClusterSuspended(cluster) && !isRayDashboardOAuthEnabled(r.Config) && !r.IsOpenShift { logger.Info("We detected being on Vanilla Kubernetes!") logger.Info("Creating Dashboard Ingress") dashboardName := dashboardNameFromCluster(cluster) @@ -312,6 +312,10 @@ func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool { return cfg == nil || ptr.Deref(cfg.MTLSEnabled, true) } +func isRayClusterSuspended(cluster *rayv1.RayCluster) bool { + return cluster.Spec.Suspend != nil && ptr.Deref(cluster.Spec.Suspend, false) +} + func crbNameFromCluster(cluster *rayv1.RayCluster) string { if shouldUseOldName(cluster) { return cluster.Name + "-" + cluster.Namespace + "-auth" diff --git a/pkg/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go index db49adf98..c3769c5a9 100644 --- a/pkg/controllers/raycluster_controller_test.go +++ b/pkg/controllers/raycluster_controller_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/project-codeflare/codeflare-common/support" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" @@ -65,6 +66,7 @@ var _ = Describe("RayCluster controller", func() { }, RayStartParams: map[string]string{}, }, + Suspend: support.Ptr(false), }, } _, err = rayClient.RayV1().RayClusters(namespace.Name).Create(ctx, raycluster, metav1.CreateOptions{}) @@ -209,6 +211,7 @@ var _ = Describe("RayCluster controller", func() { }, RayStartParams: map[string]string{}, }, + Suspend: support.Ptr(false), }, } _, err := rayClient.RayV1().RayClusters(namespaceName).Create(ctx, rayclusterWithPullSecret, metav1.CreateOptions{}) From f59ec01d866389bfb15dc19d2ef4e041a6a85672 Mon Sep 17 00:00:00 2001 From: Pat O'Connor Date: Thu, 6 Feb 2025 12:04:27 +0000 Subject: [PATCH 335/369] fix: CVE-2024-45339 - update golang/glog Signed-off-by: Pat O'Connor --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6243acde8..e5dc66021 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/glog v1.1.2 // indirect + github.com/golang/glog v1.2.4 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect diff --git a/go.sum b/go.sum index 83697cd32..8948dacc0 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= From b7e3f343bbfeac77be930580be00a48ba31bf091 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 10 Feb 2025 12:28:26 +0000 Subject: [PATCH 336/369] Adjust test timeout on AppWrappers to decrease e2e tests duration --- test/e2e/deployment_appwrapper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/deployment_appwrapper_test.go b/test/e2e/deployment_appwrapper_test.go index 39d953c7b..5304212af 100644 --- a/test/e2e/deployment_appwrapper_test.go +++ b/test/e2e/deployment_appwrapper_test.go @@ -153,7 +153,7 @@ func TestDeploymentAppWrapper(t *testing.T) { // A deployment will not complete; so simply make sure it keeps running for reasonable interval test.T().Logf("Ensuring the AppWrapper %s/%s continues to run", aw.Namespace, aw.Name) - test.Consistently(AppWrappers(test, namespace), TestTimeoutMedium).Should( + test.Consistently(AppWrappers(test, namespace), TestTimeoutShort).Should( ContainElement(WithTransform(AppWrapperPhase, Equal(mcadv1beta2.AppWrapperRunning)))) test.T().Logf("Deleting AppWrapper %s/%s", aw.Namespace, aw.Name) From e99e560120f3332b8b8ab88c1caa160ddf8782f9 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 12 Feb 2025 09:14:22 -0500 Subject: [PATCH 337/369] Upgrade AppWrapper from v1.0.0 to v1.0.4 Features + Propagate SchedulingGates in RunWithPodSetsInfo + extend PodSets() to compute TopologyRequest + Support PreferNoSchedule labels for AutoPilot + Implement PodSet inference for JobSet Bugs and Regressions + move appwrapper finalizer injection from defaulting webhook to controller + use ptr.Equal to validate managedBy is immutable + fix handling of labels/annotations in utils.GetPodTemplateSpec --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 14 ++++++++++++++ config/rbac/role.yaml | 12 ++++++++++++ go.mod | 2 +- go.sum | 4 ++-- pkg/controllers/appwrapper_controller.go | 1 + 7 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9ab3481e8..68d152c24 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v1.0.0 +APPWRAPPER_VERSION ?= v1.0.4 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 798f5fbbc..c139832a9 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v1.0.0 +- github.com/project-codeflare/appwrapper/config/crd?ref=v1.0.4 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index 744b87949..6a3020756 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -88,6 +88,20 @@ spec: type: string description: NodeSelectors to be added to the PodSpecTemplate type: object + schedulingGates: + description: SchedulingGates to be added to the PodSpecTemplate + items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array tolerations: description: Tolerations to be added to the PodSpecTemplate items: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1f616d40d..b4be4fc2b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -164,6 +164,18 @@ rules: - get - list - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - kubeflow.org resources: diff --git a/go.mod b/go.mod index e5dc66021..bcc3e9da7 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 - github.com/project-codeflare/appwrapper v1.0.0 + github.com/project-codeflare/appwrapper v1.0.4 github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 8948dacc0..30448652e 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v1.0.0 h1:tKvBXxaIE5RwzUC8YZvxcnbuz4JUbiuLjo1IWO4Mciw= -github.com/project-codeflare/appwrapper v1.0.0/go.mod h1:pVCWURfk9DOa/4ig7z91PJWRMlsDchhQVmu7mB32L48= +github.com/project-codeflare/appwrapper v1.0.4 h1:364zQLX0tsi4LvBBYNKZL7PPbNWPbVU7vK6+/kVV/FQ= +github.com/project-codeflare/appwrapper v1.0.4/go.mod h1:A1b6bMFNMX5Btv3ckgeuAHVVZzp1G30pSBe6BE/xJWE= github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b h1:MOmv/aLx/kcHd7PBErx8XNSTW180s8Slf/uVM0uV4rw= github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index b160a7f97..ed65accd8 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -32,6 +32,7 @@ package controllers //+kubebuilder:rbac:groups=kubeflow.org,resources=pytorchjobs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=jobset.x-k8s.io,resources=jobsets,verbs=get;list;watch;create;update;patch;delete // permissions needed by Kueue's generic reconciller // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=list;get;watch From f6396217adeefa7667808bc1d7d5d772e462449a Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Fri, 14 Feb 2025 09:34:16 +0000 Subject: [PATCH 338/369] Update dependency versions for release v1.15.0 --- README.md | 8 ++++---- config/manager/params.env | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f4eb541e1..c3180e64d 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.14.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.14.0) | -| CodeFlare-SDK | [v0.26.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.26.0) | -| AppWrapper | [v0.30.0](https://github.com/project-codeflare/appwrapper/releases/tag/v0.30.0) | +| CodeFlare Operator | [v1.15.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.15.0) | +| CodeFlare-SDK | [v0.27.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.27.0) | +| AppWrapper | [v1.0.4](https://github.com/project-codeflare/appwrapper/releases/tag/v1.0.4) | | KubeRay | [v1.2.2](https://github.com/opendatahub-io/kuberay/releases/tag/v1.2.2) | -| Kueue | [v0.8.3](https://github.com/opendatahub-io/kueue/releases/tag/v0.8.3) | +| Kueue | [v0.10.1](https://github.com/opendatahub-io/kueue/releases/tag/v0.10.1) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index b28efad19..6d0c50b4e 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.14.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.15.0 namespace=opendatahub From bb927ef23356a0995bc4f7b8d6f0e85ca2f5094e Mon Sep 17 00:00:00 2001 From: oksanabaza Date: Fri, 14 Feb 2025 10:06:46 +0000 Subject: [PATCH 339/369] Fix workflow generate-component-metadata step --- .github/workflows/project-codeflare-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index e9d5f0b85..e2904697b 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -167,10 +167,12 @@ jobs: fi generate-component-metadata: - needs: [check-appwrapper-version, check-kuberay-version, check-kueue-version] runs-on: ubuntu-latest steps: + - name: Ensure config folder exists + run: mkdir -p config + - name: Generate component_metadata.yaml run: | cat < config/component_metadata.yaml From bcfc7ddb1f527da5b525174c6b77ac13f5ecc561 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Fri, 14 Feb 2025 14:02:09 +0000 Subject: [PATCH 340/369] Point links in compatibility matrix to upstream tags --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c3180e64d..6bb569e48 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ CodeFlare Stack Compatibility Matrix | CodeFlare Operator | [v1.15.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.15.0) | | CodeFlare-SDK | [v0.27.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.27.0) | | AppWrapper | [v1.0.4](https://github.com/project-codeflare/appwrapper/releases/tag/v1.0.4) | -| KubeRay | [v1.2.2](https://github.com/opendatahub-io/kuberay/releases/tag/v1.2.2) | -| Kueue | [v0.10.1](https://github.com/opendatahub-io/kueue/releases/tag/v0.10.1) | +| KubeRay | [v1.2.2](https://github.com/ray-project/kuberay/releases/tag/v1.2.2) | +| Kueue | [v0.10.1](https://github.com/kubernetes-sigs/kueue/releases/tag/v0.10.1) | ## Development From e2ba556a6e820252aacac7f884952a64ca817ec5 Mon Sep 17 00:00:00 2001 From: Krzysztof Malczuk Date: Thu, 27 Feb 2025 14:32:10 +0000 Subject: [PATCH 341/369] Upgrade cert generator image to use py3.11 --- config/e2e/patch_resources.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/e2e/patch_resources.yaml b/config/e2e/patch_resources.yaml index 138170c35..2def7d7a1 100644 --- a/config/e2e/patch_resources.yaml +++ b/config/e2e/patch_resources.yaml @@ -7,4 +7,4 @@ path: /spec/template/spec/containers/0/env/- value: name: CERT_GENERATOR_IMAGE - value: quay.io/modh/ray:2.35.0-py39-cu121 + value: quay.io/modh/ray:2.35.0-py311-cu121 From 643756ef3f7ae6643db721b5cb74314bce194228 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 6 Mar 2025 15:24:10 +0000 Subject: [PATCH 342/369] Add constant namespaces for RayDashboard tests --- test/e2e/mnist_rayjob_raycluster_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 3c24c55d2..9c4c80f9e 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -51,7 +51,7 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int test := With(t) // Create a namespace - namespace := test.NewTestNamespace() + namespace := test.NewTestNamespace(WithNamespaceName("test-ns-1")) // Create Kueue resources resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) @@ -122,7 +122,7 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number test := With(t) // Create a namespace - namespace := test.NewTestNamespace() + namespace := test.NewTestNamespace(WithNamespaceName("test-ns-2")) // Create Kueue resources resourceFlavor := CreateKueueResourceFlavor(test, v1beta1.ResourceFlavorSpec{}) From b525c598ef8f3005f1dbf85e6422be7465de15b0 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 6 Mar 2025 17:50:16 +0000 Subject: [PATCH 343/369] Update codeflare-common Go import --- go.mod | 2 +- go.sum | 4 ++-- test/e2e/mnist_rayjob_raycluster_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index bcc3e9da7..e0e124fd8 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 github.com/project-codeflare/appwrapper v1.0.4 - github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b + github.com/project-codeflare/codeflare-common v0.0.0-20250306164418-eb812487be82 github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 diff --git a/go.sum b/go.sum index 30448652e..943eeeb05 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v1.0.4 h1:364zQLX0tsi4LvBBYNKZL7PPbNWPbVU7vK6+/kVV/FQ= github.com/project-codeflare/appwrapper v1.0.4/go.mod h1:A1b6bMFNMX5Btv3ckgeuAHVVZzp1G30pSBe6BE/xJWE= -github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b h1:MOmv/aLx/kcHd7PBErx8XNSTW180s8Slf/uVM0uV4rw= -github.com/project-codeflare/codeflare-common v0.0.0-20250128135036-f501cd31fe8b/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= +github.com/project-codeflare/codeflare-common v0.0.0-20250306164418-eb812487be82 h1:cL1K2+r1lJVwBkhXiVFr2A9DphnylJmilYDIqg/W62M= +github.com/project-codeflare/codeflare-common v0.0.0-20250306164418-eb812487be82/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 9c4c80f9e..cac4b0955 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -50,7 +50,7 @@ func TestMnistRayJobRayClusterGpu(t *testing.T) { func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int) { test := With(t) - // Create a namespace + // Create a static namespace to ensure a consistent Ray Dashboard hostname entry in /etc/hosts before executing the test. namespace := test.NewTestNamespace(WithNamespaceName("test-ns-1")) // Create Kueue resources @@ -121,7 +121,7 @@ func TestMnistRayJobRayClusterAppWrapperGpu(t *testing.T) { func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, numberOfGpus int) { test := With(t) - // Create a namespace + // Create a static namespace to ensure a consistent Ray Dashboard hostname entry in /etc/hosts before executing the test. namespace := test.NewTestNamespace(WithNamespaceName("test-ns-2")) // Create Kueue resources From b612ce3f8b569c4b13836199edd26446a757a331 Mon Sep 17 00:00:00 2001 From: Krzysztof Malczuk Date: Fri, 7 Feb 2025 10:11:39 +0000 Subject: [PATCH 344/369] Update outdated README.md, update Makefile commands to support local kind test, upgrade kindest/node to 1.30.10 . --- Makefile | 2 +- README.md | 35 +++++++++++++++++++---------------- test/e2e/kind.sh | 2 +- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 68d152c24..ee7f880c3 100644 --- a/Makefile +++ b/Makefile @@ -393,7 +393,7 @@ test-component: envtest ginkgo ## Run component tests. .PHONY: test-e2e test-e2e: manifests fmt vet ## Run e2e tests. - go test -timeout 30m -v ./test/e2e + CODEFLARE_TEST_OUTPUT_DIR=/tmp/ CLUSTER_HOSTNAME=kind CODEFLARE_TEST_TIMEOUT_MEDIUM=5m CODEFLARE_TEST_TIMEOUT_LONG=40m go test -v -skip "^Test.*Gpu$$" ./test/e2e -timeout=60m .PHONY: kind-e2e kind-e2e: ## Set up e2e KinD cluster diff --git a/README.md b/README.md index 6bb569e48..ad60de570 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # codeflare-operator -Operator for installation and lifecycle management of CodeFlare distributed workload stack. +The CodeFlare-Operator has embedded two controllers, a [RayCluster controller](https://github.com/project-codeflare/codeflare-operator/blob/main/pkg/controllers/raycluster_controller.go) which creates resources including secrets, ingress, routes, service, serviceaccounts, clusterrolebinding resources; all needed for the RayClusters created to work as expected. + +There's an [AppWrapper Controller](https://github.com/project-codeflare/appwrapper/blob/main/internal/controller/appwrapper/appwrapper_controller.go), which is a flexible and workload-agnostic mechanism to enable Kueue to manage a group of Kubernetes resources as a single logical unit and to provide an additional level of automatic fault detection and recovery. + +For each controller, there are webhooks in place that can be found [here](https://github.com/project-codeflare/codeflare-operator/tree/main/pkg/controllers). @@ -24,6 +28,7 @@ Requirements: # brew install gnu-sed make install -e SED=/usr/local/bin/gsed ``` +- Kind - Kind is used in the kind-e2e command in the Makefile. Follow these instructions for the kind setup here ### Testing @@ -34,11 +39,9 @@ The e2e tests can be executed locally by running the following commands: ```bash # Create a KinD cluster make kind-e2e - # Install the CRDs - make install ``` - [!NOTE] +> [!NOTE] Some e2e tests cover the access to services via Ingresses, as end-users would do, which requires access to the Ingress controller load balancer by its IP. For it to work on macOS, this requires installing [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). @@ -47,16 +50,16 @@ The e2e tests can be executed locally by running the following commands: ```bash make setup-e2e ``` - - [!NOTE] + +> [!NOTE] Kueue will only activate its Ray integration if KubeRay is installed before Kueue (as done by this make target). - [!NOTE] +> [!NOTE] In OpenShift the KubeRay operator pod gets random user assigned. This user is then used to run Ray cluster. However the random user assigned by OpenShift doesn't have rights to store dataset downloaded as part of test execution, causing tests to fail. To prevent this failure on OpenShift user should enforce user 1000 for KubeRay and Ray cluster by creating this SCC in KubeRay operator namespace (replace the namespace placeholder): - ```yaml + ```yaml kind: SecurityContextConstraints apiVersion: security.openshift.io/v1 metadata: @@ -68,21 +71,21 @@ The e2e tests can be executed locally by running the following commands: uid: 1000 users: - 'system:serviceaccount:$(namespace):kuberay-operator' - ``` - -3. Start the operator locally: + ``` +3. In the /etc/hosts file add the following lines: ```bash - NAMESPACE=default make run + 127.0.0.1 ray-dashboard-raycluster-test-ns-1.kind + 127.0.0.1 ray-dashboard-raycluster-test-ns-2.kind ``` - Alternatively, You can run the operator from your IDE / debugger. - -4. In a separate terminal, set your output directory for test files, and run the e2e suite: +4. Build, push and deploy the codeflare-operator image: ```bash - export CODEFLARE_TEST_OUTPUT_DIR= + make image-push IMG=: + make deploy -e IMG=: -e ENV="e2e" ``` +5. To run the tests run the command ```bash make test-e2e ``` diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 29b19c347..49f777eba 100755 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -23,7 +23,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 + image: kindest/node:v1.30.10@sha256:4de75d0e82481ea846c0ed1de86328d821c1e6a6a91ac37bf804e5313670e507 extraPortMappings: - containerPort: 80 hostPort: 80 From 98fe2476fa83cd836e7f3271224822b727e640dd Mon Sep 17 00:00:00 2001 From: Srihari Date: Mon, 10 Mar 2025 14:53:04 +0530 Subject: [PATCH 345/369] Add Github Action workflow for updating DW Components Release matrix info to confluence --- .../workflows/project-codeflare-release.yml | 36 +++++- .../update-release-matrix-to-confluence.yml | 121 ++++++++++++++++++ 2 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/update-release-matrix-to-confluence.yml diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index e2904697b..7ce1fb9cf 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -30,14 +30,13 @@ on: description: 'GitHub organization/user containing repositories used for release' required: true default: 'project-codeflare' - quay-organization: - description: 'Quay organization used to push the built images to' - required: true - default: 'project-codeflare' community-operators-prod-organization: description: 'Owner of target community-operators-prod repository used to open a PR against' required: true default: 'redhat-openshift-ecosystem' + rhoai-release-version: + description: "RHOAI Release version for updating Component Release Matrix Versions Info " + required: true jobs: release-parameters: @@ -54,7 +53,6 @@ jobs: echo "Tested Kueue Version: ${{ github.event.inputs.kueue-version }}" echo "Is Stable: ${{ github.event.inputs.is-stable }}" echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" - echo "Quay Organization: ${{ github.event.inputs.quay-organization }}" echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" release-codeflare-sdk: @@ -74,7 +72,7 @@ jobs: run: | semver_version="${{ github.event.inputs.codeflare-sdk-version }}" plain_version="${semver_version:1}" - gh workflow run release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --ref ${{ github.ref }} --field release-version=${plain_version} --field is-stable=${{ github.event.inputs.is-stable }} --field quay-organization=${{ github.event.inputs.quay-organization }} + gh workflow run release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --ref ${{ github.ref }} --field release-version=${plain_version} --field is-stable=${{ github.event.inputs.is-stable }} --field quay-organization=project-codeflare env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash @@ -107,7 +105,7 @@ jobs: --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} \ --field kuberay-version=${{ github.event.inputs.kuberay-version }} \ --field kueue-version=${{ github.event.inputs.kueue-version }} \ - --field quay-organization=${{ github.event.inputs.quay-organization }} \ + --field quay-organization=project-codeflare \ --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} \ --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: @@ -184,3 +182,27 @@ jobs: - name: Verify generated file run: cat config/component_metadata.yaml + + Update_release_version_info_to_confluence: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Trigger and Update Component Release Matrix Versions Info to Confluence + run: | + gh workflow run update-release-matrix-to-confluence.yml --ref ${{ github.ref }} \ + --field rhoai-release-version=${{ github.event.inputs.rhoai-release-version }} \ + --field kueue-version=${{ github.event.inputs.kueue-version }} \ + --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} \ + --field codeflare-operator-version=${{ github.event.inputs.operator-version }} \ + --field kuberay-version=${{ github.event.inputs.kuberay-version }} \ + --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} + + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow update-release-matrix-to-confluence.yml --repo https://github.com/project-codeflare/codeflare-operator --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo https://github.com/project-codeflare/codeflare-operator --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash diff --git a/.github/workflows/update-release-matrix-to-confluence.yml b/.github/workflows/update-release-matrix-to-confluence.yml new file mode 100644 index 000000000..d129e6c7b --- /dev/null +++ b/.github/workflows/update-release-matrix-to-confluence.yml @@ -0,0 +1,121 @@ +name: Update Release Matrix to Confluence + +on: + workflow_dispatch: + inputs: + rhoai-release-version: + description: 'RHOAI Release Version' + required: true + kueue-version: + description: 'Kueue Version' + required: true + codeflare-sdk-version: + description: 'CodeFlare SDK Version' + required: true + codeflare-operator-version: + description: 'CodeFlare operator Version' + required: true + kuberay-version: + description: 'Tested KubeRay version' + required: true + appwrapper-version: + description: 'Tested appwrapper version' + required: true + +jobs: + update-confluence: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Release info Parameters + run: | + echo "RHOAI_RELEASE_VERSION=${{ github.event.inputs.rhoai-release-version }}" >> $GITHUB_ENV + echo "KUEUE_VERSION=${{ github.event.inputs.kueue-version }}" >> $GITHUB_ENV + echo "CODEFLARE_SDK_VERSION=${{ github.event.inputs.codeflare-sdk-version }}" >> $GITHUB_ENV + echo "CODEFLARE_OPERATOR_VERSION=${{ github.event.inputs.codeflare-operator-version }}" >> $GITHUB_ENV + echo "KUBERAY_VERSION=${{ github.event.inputs.kuberay-version }}" >> $GITHUB_ENV + echo "APPWRAPPER_VERSION=${{ github.event.inputs.appwrapper-version }}" >> $GITHUB_ENV + + - name: Fetch and Update Existing Release Matrix Page Content + run: | + echo "Fetching Release Matrix Confluence Page..." + response=$(curl -H "Authorization: Bearer ${{ secrets.CONFLUENCE_API_TOKEN }}" \ + "${{ secrets.CONFLUENCE_BASE_URL }}/rest/api/content?title=${{ secrets.PAGE_TITLE }}&spaceKey=${{ secrets.SPACE_KEY }}&expand=body.storage,version") + + echo "$response" | jq '.' > page_data.json + echo "Raw API Response: $response" + + PAGE_VERSION=$(jq '.results[0].version.number' page_data.json) + + if [[ -z "$PAGE_VERSION" || "$PAGE_VERSION" == "null" ]]; then + echo "Error: Could not retrieve current page version." + exit 1 + fi + echo "PAGE_VERSION=$PAGE_VERSION" >> $GITHUB_ENV + + EXISTING_CONTENT=$(jq -r '.results[0].body.storage.value' page_data.json) + + echo "Existing Release Matrix Page Content: $EXISTING_CONTENT" + + if [[ -z "$EXISTING_CONTENT" || "$EXISTING_CONTENT" == "null" ]]; then + echo "Error: Could not retrieve existing page content." + exit 1 + fi + + # Convert newlines to a placeholder to handle multi-line processing + PLACEHOLDER="__NL__" + MODIFIED_CONTENT=$(echo "$EXISTING_CONTENT" | tr '\n' "$PLACEHOLDER") + + # Update the page content with release info also check and update if the release version already exists in the table + if echo "$MODIFIED_CONTENT" | grep -q "]*>$RHOAI_RELEASE_VERSION"; then + UPDATED_PAGE_CONTENT=$(echo "$MODIFIED_CONTENT" | sed -E "s|(]*>$RHOAI_RELEASE_VERSION)[^<]+()[^<]+()[^<]+()[^<]+()[^<]+()|\1$KUEUE_VERSION\2$CODEFLARE_SDK_VERSION\3$CODEFLARE_OPERATOR_VERSION\4$KUBERAY_VERSION\5$APPWRAPPER_VERSION\6|") + else + UPDATED_ROW="$RHOAI_RELEASE_VERSION$KUEUE_VERSION$CODEFLARE_SDK_VERSION$CODEFLARE_OPERATOR_VERSION$KUBERAY_VERSION$APPWRAPPER_VERSION" + UPDATED_PAGE_CONTENT=$(echo "$MODIFIED_CONTENT" | sed "s||$UPDATED_ROW|") + fi + + # Correct JSON encoding without double escaping + UPDATED_PAGE_CONTENT=$(echo "$UPDATED_PAGE_CONTENT" | sed 's/_$//') # Remove trailing underscores + UPDATED_PAGE_CONTENT=$(jq -n --arg content "$UPDATED_PAGE_CONTENT" '$content' | tr -d '\r') + # Store as output + echo "UPDATED_PAGE_CONTENT=$UPDATED_PAGE_CONTENT" >> "$GITHUB_ENV" + + - name: Publish updated page content to confluence + run: | + + NEW_VERSION=$(( PAGE_VERSION + 1 )) + + if [[ -n "$UPDATED_PAGE_CONTENT" && "$UPDATED_PAGE_CONTENT" != "null" ]]; then + echo "Updating Confluence Page using PUT request..." + HTTP_RESPONSE=$(curl -s -o response.json -w "%{http_code}" -X PUT "${{ secrets.CONFLUENCE_BASE_URL }}/rest/api/content/${{ secrets.CONFLUENCE_PAGE_ID }}" \ + -H "Authorization: Bearer ${{ secrets.CONFLUENCE_API_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d "{ + \"id\": \"${{ secrets.CONFLUENCE_PAGE_ID }}\", + \"type\": \"page\", + \"title\": \"Distributed Workloads Release Details\", + \"space\": { \"key\": \"${{ secrets.SPACE_KEY }}\" }, + \"body\": { + \"storage\": { + \"value\": $UPDATED_PAGE_CONTENT, + \"representation\": \"storage\" + } + }, + \"version\": { + \"number\": $NEW_VERSION + } + }") + if [[ "$HTTP_RESPONSE" == "200" || "$HTTP_RESPONSE" == "201" ]]; then + echo "Successfully updated Confluence Page with release version details !" + echo "Response from Confluence:" + cat response.json + else + echo "Error: Failed to update Confluence page. HTTP Response Code: $HTTP_RESPONSE" + exit 1 + fi + else + echo "Error: UPDATED_PAGE_CONTENT is null or empty." + exit 1 + fi From a96625a346a2d0b0f65217380f223c16b6f2f5d7 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Fri, 28 Feb 2025 17:17:15 +0530 Subject: [PATCH 346/369] Add AMD GPU test for ray clusters --- .github/workflows/e2e_tests.yaml | 3 +- go.mod | 2 +- go.sum | 4 +- test/e2e/deployment_appwrapper_test.go | 2 +- test/e2e/job_appwrapper_test.go | 2 +- test/e2e/mnist_pytorch_appwrapper_test.go | 10 +- test/e2e/mnist_rayjob_raycluster_test.go | 152 ++++++++++++++-------- 7 files changed, 113 insertions(+), 62 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 634f1484b..2f754904e 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -89,7 +89,8 @@ jobs: export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} set -euo pipefail - go test -timeout 120m -v -skip "^Test.*Cpu$" ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 120m -v -skip "^Test.*Cpu$|^Test.*ROCmGpu$" ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' diff --git a/go.mod b/go.mod index e0e124fd8..3c9db62da 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 github.com/project-codeflare/appwrapper v1.0.4 - github.com/project-codeflare/codeflare-common v0.0.0-20250306164418-eb812487be82 + github.com/project-codeflare/codeflare-common v0.0.0-20250317102908-1c124db97844 github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 diff --git a/go.sum b/go.sum index 943eeeb05..15f22815d 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v1.0.4 h1:364zQLX0tsi4LvBBYNKZL7PPbNWPbVU7vK6+/kVV/FQ= github.com/project-codeflare/appwrapper v1.0.4/go.mod h1:A1b6bMFNMX5Btv3ckgeuAHVVZzp1G30pSBe6BE/xJWE= -github.com/project-codeflare/codeflare-common v0.0.0-20250306164418-eb812487be82 h1:cL1K2+r1lJVwBkhXiVFr2A9DphnylJmilYDIqg/W62M= -github.com/project-codeflare/codeflare-common v0.0.0-20250306164418-eb812487be82/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= +github.com/project-codeflare/codeflare-common v0.0.0-20250317102908-1c124db97844 h1:hEjZ2pV4Fp81wytijJZ7uHWovKIqirVBA/t1F5hIrbA= +github.com/project-codeflare/codeflare-common v0.0.0-20250317102908-1c124db97844/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/test/e2e/deployment_appwrapper_test.go b/test/e2e/deployment_appwrapper_test.go index 5304212af..6bf4cc793 100644 --- a/test/e2e/deployment_appwrapper_test.go +++ b/test/e2e/deployment_appwrapper_test.go @@ -45,7 +45,7 @@ func TestDeploymentAppWrapper(t *testing.T) { defer func() { _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) }() - clusterQueue := createClusterQueue(test, resourceFlavor, 0) + clusterQueue := createClusterQueue(test, resourceFlavor, CPU) defer func() { _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) }() diff --git a/test/e2e/job_appwrapper_test.go b/test/e2e/job_appwrapper_test.go index 9805db202..ee435f309 100644 --- a/test/e2e/job_appwrapper_test.go +++ b/test/e2e/job_appwrapper_test.go @@ -43,7 +43,7 @@ func TestBatchJobAppWrapper(t *testing.T) { defer func() { _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) }() - clusterQueue := createClusterQueue(test, resourceFlavor, 0) + clusterQueue := createClusterQueue(test, resourceFlavor, CPU) defer func() { _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) }() diff --git a/test/e2e/mnist_pytorch_appwrapper_test.go b/test/e2e/mnist_pytorch_appwrapper_test.go index d2e1b5741..d67ae2561 100644 --- a/test/e2e/mnist_pytorch_appwrapper_test.go +++ b/test/e2e/mnist_pytorch_appwrapper_test.go @@ -32,15 +32,15 @@ import ( ) func TestMnistPyTorchAppWrapperCpu(t *testing.T) { - runMnistPyTorchAppWrapper(t, "cpu", 0) + runMnistPyTorchAppWrapper(t, CPU) } func TestMnistPyTorchAppWrapperGpu(t *testing.T) { - runMnistPyTorchAppWrapper(t, "gpu", 1) + runMnistPyTorchAppWrapper(t, NVIDIA) } // Trains the MNIST dataset as a batch Job in an AppWrapper, and asserts successful completion of the training job. -func runMnistPyTorchAppWrapper(t *testing.T, accelerator string, numberOfGpus int) { +func runMnistPyTorchAppWrapper(t *testing.T, accelerator Accelerator) { test := With(t) // Create a namespace @@ -51,7 +51,7 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string, numberOfGpus in defer func() { _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) }() - clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) + clusterQueue := createClusterQueue(test, resourceFlavor, accelerator) defer func() { _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) }() @@ -109,7 +109,7 @@ func runMnistPyTorchAppWrapper(t *testing.T, accelerator string, numberOfGpus in {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, {Name: "PIP_INDEX_URL", Value: GetPipIndexURL()}, {Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()}, - {Name: "ACCELERATOR", Value: accelerator}, + {Name: "ACCELERATOR", Value: accelerator.Type}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index cac4b0955..01fcbc4ce 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "net/url" + "strings" "testing" . "github.com/onsi/gomega" @@ -40,14 +41,18 @@ import ( // directly managed by Kueue, and asserts successful completion of the training job. func TestMnistRayJobRayClusterCpu(t *testing.T) { - runMnistRayJobRayCluster(t, "cpu", 0) + runMnistRayJobRayCluster(t, CPU, GetRayImage()) } -func TestMnistRayJobRayClusterGpu(t *testing.T) { - runMnistRayJobRayCluster(t, "gpu", 1) +func TestMnistRayJobRayClusterCudaGpu(t *testing.T) { + runMnistRayJobRayCluster(t, NVIDIA, GetRayImage()) } -func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int) { +func TestMnistRayJobRayClusterROCmGpu(t *testing.T) { + runMnistRayJobRayCluster(t, AMD, GetRayROCmImage()) +} + +func runMnistRayJobRayCluster(t *testing.T, accelerator Accelerator, rayImage string) { test := With(t) // Create a static namespace to ensure a consistent Ray Dashboard hostname entry in /etc/hosts before executing the test. @@ -58,11 +63,12 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int defer func() { _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) }() - clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) + + clusterQueue := createClusterQueue(test, resourceFlavor, accelerator) defer func() { _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) }() - CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) + localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Create MNIST training script mnist := constructMNISTConfigMap(test, namespace) @@ -71,7 +77,7 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // Create RayCluster and assign it to the localqueue - rayCluster := constructRayCluster(test, namespace, mnist, numberOfGpus) + rayCluster := constructRayCluster(test, namespace, localQueue.Name, mnist, accelerator, rayImage, false) rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name) @@ -81,7 +87,7 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) // Create RayJob - rayJob := constructRayJob(test, namespace, rayCluster, accelerator, numberOfGpus) + rayJob := constructRayJob(test, namespace, rayCluster, accelerator, rayImage) rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) @@ -110,15 +116,19 @@ func runMnistRayJobRayCluster(t *testing.T, accelerator string, numberOfGpus int } func TestMnistRayJobRayClusterAppWrapperCpu(t *testing.T) { - runMnistRayJobRayClusterAppWrapper(t, "cpu", 0) + runMnistRayJobRayClusterAppWrapper(t, CPU, GetRayImage()) } -func TestMnistRayJobRayClusterAppWrapperGpu(t *testing.T) { - runMnistRayJobRayClusterAppWrapper(t, "gpu", 1) +func TestMnistRayJobRayClusterAppWrapperCudaGpu(t *testing.T) { + runMnistRayJobRayClusterAppWrapper(t, NVIDIA, GetRayImage()) +} + +func TestMnistRayJobRayClusterAppWrapperROCmGpu(t *testing.T) { + runMnistRayJobRayClusterAppWrapper(t, AMD, GetRayROCmImage()) } // Same as TestMNISTRayJobRayCluster, except the RayCluster is wrapped in an AppWrapper -func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, numberOfGpus int) { +func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator Accelerator, rayImage string) { test := With(t) // Create a static namespace to ensure a consistent Ray Dashboard hostname entry in /etc/hosts before executing the test. @@ -129,7 +139,7 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number defer func() { _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) }() - clusterQueue := createClusterQueue(test, resourceFlavor, numberOfGpus) + clusterQueue := createClusterQueue(test, resourceFlavor, accelerator) defer func() { _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) }() @@ -142,7 +152,7 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // Create RayCluster, wrap in AppWrapper and assign to localqueue - rayCluster := constructRayCluster(test, namespace, mnist, numberOfGpus) + rayCluster := constructRayCluster(test, namespace, localQueue.Name, mnist, accelerator, rayImage, true) raw := Raw(test, rayCluster) raw = RemoveCreationTimestamp(test, raw) @@ -183,7 +193,7 @@ func runMnistRayJobRayClusterAppWrapper(t *testing.T, accelerator string, number Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) // Create RayJob - rayJob := constructRayJob(test, namespace, rayCluster, accelerator, numberOfGpus) + rayJob := constructRayJob(test, namespace, rayCluster, accelerator, rayImage) rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) @@ -223,11 +233,11 @@ func TestRayClusterImagePullSecret(t *testing.T) { defer func() { _ = test.Client().Kueue().KueueV1beta1().ResourceFlavors().Delete(test.Ctx(), resourceFlavor.Name, metav1.DeleteOptions{}) }() - clusterQueue := createClusterQueue(test, resourceFlavor, 0) + clusterQueue := createClusterQueue(test, resourceFlavor, CPU) defer func() { _ = test.Client().Kueue().KueueV1beta1().ClusterQueues().Delete(test.Ctx(), clusterQueue.Name, metav1.DeleteOptions{}) }() - CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) + localQueue := CreateKueueLocalQueue(test, namespace.Name, clusterQueue.Name, AsDefaultQueue) // Create MNIST training script mnist := constructMNISTConfigMap(test, namespace) @@ -236,7 +246,7 @@ func TestRayClusterImagePullSecret(t *testing.T) { test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // Create RayCluster with imagePullSecret and assign it to the localqueue - rayCluster := constructRayCluster(test, namespace, mnist, 0) + rayCluster := constructRayCluster(test, namespace, localQueue.Name, mnist, CPU, GetRayImage(), false) rayCluster.Spec.HeadGroupSpec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "custom-pull-secret"}} rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) @@ -266,8 +276,8 @@ func constructMNISTConfigMap(test Test, namespace *corev1.Namespace) *corev1.Con } } -func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.ConfigMap, numberOfGpus int) *rayv1.RayCluster { - return &rayv1.RayCluster{ +func constructRayCluster(_ Test, namespace *corev1.Namespace, localQueueName string, mnist *corev1.ConfigMap, accelerator Accelerator, rayImage string, isWrappedInAppWrapper bool) *rayv1.RayCluster { + raycluster := rayv1.RayCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1.GroupVersion.String(), Kind: "RayCluster", @@ -287,7 +297,7 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf Containers: []corev1.Container{ { Name: "ray-head", - Image: GetRayImage(), + Image: rayImage, Ports: []corev1.ContainerPort{ { ContainerPort: 6379, @@ -333,16 +343,10 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - Tolerations: []corev1.Toleration{ - { - Key: "nvidia.com/gpu", - Operator: corev1.TolerationOpExists, - }, - }, Containers: []corev1.Container{ { Name: "ray-worker", - Image: GetRayImage(), + Image: rayImage, Lifecycle: &corev1.Lifecycle{ PreStop: &corev1.LifecycleHandler{ Exec: &corev1.ExecAction{ @@ -354,12 +358,10 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("250m"), corev1.ResourceMemory: resource.MustParse("1G"), - "nvidia.com/gpu": resource.MustParse(fmt.Sprint(numberOfGpus)), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4G"), - "nvidia.com/gpu": resource.MustParse(fmt.Sprint(numberOfGpus)), }, }, VolumeMounts: []corev1.VolumeMount{ @@ -388,10 +390,46 @@ func constructRayCluster(_ Test, namespace *corev1.Namespace, mnist *corev1.Conf }, }, } + + // Add label if raycluster is not wrapped in the app wrapper + if !isWrappedInAppWrapper { + if raycluster.ObjectMeta.Labels == nil { + raycluster.ObjectMeta.Labels = make(map[string]string) + } + raycluster.ObjectMeta.Labels["kueue.x-k8s.io/queue-name"] = localQueueName + } + + if accelerator.IsGpu() { + // Add toleration for the GPU + raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Tolerations = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Tolerations, corev1.Toleration{ + Key: accelerator.ResourceLabel, + Operator: corev1.TolerationOpExists, + }) + // Add GPU resource quota + raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Resources.Requests[corev1.ResourceName(accelerator.ResourceLabel)] = resource.MustParse("1") + raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Resources.Limits[corev1.ResourceName(accelerator.ResourceLabel)] = resource.MustParse("1") + } + + return &raycluster } -func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayCluster, accelerator string, numberOfGpus int) *rayv1.RayJob { - return &rayv1.RayJob{ +func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayCluster, accelerator Accelerator, rayImage string) *rayv1.RayJob { + pipPackages := []string{ + "pytorch_lightning==2.4.0", + "torchmetrics==1.6.0", + "torchvision==0.19.1", + } + + // Append AMD-specific packages + if accelerator == AMD { + pipPackages = append(pipPackages, + "--extra-index-url https://download.pytorch.org/whl/rocm6.1", + "torch==2.4.1+rocm6.1", + ) + } + + // Construct RayJob with the final pip list + rayJob := rayv1.RayJob{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1.GroupVersion.String(), Kind: "RayJob", @@ -402,17 +440,15 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC }, Spec: rayv1.RayJobSpec{ Entrypoint: "python /home/ray/jobs/mnist.py", - RuntimeEnvYAML: ` - pip: - - pytorch_lightning==2.4.0 - - torchmetrics==1.6.0 - - torchvision==0.20.1 - env_vars: - MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `" - PIP_INDEX_URL: "` + GetPipIndexURL() + `" - PIP_TRUSTED_HOST: "` + GetPipTrustedHost() + `" - ACCELERATOR: "` + accelerator + `" -`, + RuntimeEnvYAML: fmt.Sprintf(` +pip: + - %s +env_vars: + MNIST_DATASET_URL: "%s" + PIP_INDEX_URL: "%s" + PIP_TRUSTED_HOST: "%s" + ACCELERATOR: "%s" +`, strings.Join(pipPackages, "\n - "), GetMnistDatasetURL(), GetPipIndexURL(), GetPipTrustedHost(), accelerator.Type), ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, }, @@ -422,7 +458,7 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{ { - Image: GetRayImage(), + Image: rayImage, Name: "rayjob-submitter-pod", }, }, @@ -430,9 +466,15 @@ func constructRayJob(_ Test, namespace *corev1.Namespace, rayCluster *rayv1.RayC }, EntrypointNumCpus: 2, // Using EntrypointNumGpus doesn't seem to work properly on KinD cluster with GPU, EntrypointNumCpus seems reliable - EntrypointNumGpus: float32(numberOfGpus), + EntrypointNumGpus: 0, }, } + + if accelerator.IsGpu() { + rayJob.Spec.EntrypointNumGpus = 1 + } + + return &rayJob } func getRayDashboardURL(test Test, namespace, rayClusterName string) string { @@ -477,12 +519,12 @@ func getRayDashboardURL(test Test, namespace, rayClusterName string) string { } // Create ClusterQueue -func createClusterQueue(test Test, resourceFlavor *v1beta1.ResourceFlavor, numberOfGpus int) *v1beta1.ClusterQueue { +func createClusterQueue(test Test, resourceFlavor *v1beta1.ResourceFlavor, accelerator Accelerator) *v1beta1.ClusterQueue { cqSpec := v1beta1.ClusterQueueSpec{ NamespaceSelector: &metav1.LabelSelector{}, ResourceGroups: []v1beta1.ResourceGroup{ { - CoveredResources: []corev1.ResourceName{corev1.ResourceName("cpu"), corev1.ResourceName("memory"), corev1.ResourceName("nvidia.com/gpu")}, + CoveredResources: []corev1.ResourceName{corev1.ResourceName("cpu"), corev1.ResourceName("memory")}, Flavors: []v1beta1.FlavorQuotas{ { Name: v1beta1.ResourceFlavorReference(resourceFlavor.Name), @@ -495,15 +537,23 @@ func createClusterQueue(test Test, resourceFlavor *v1beta1.ResourceFlavor, numbe Name: corev1.ResourceMemory, NominalQuota: resource.MustParse("12Gi"), }, - { - Name: corev1.ResourceName("nvidia.com/gpu"), - NominalQuota: resource.MustParse(fmt.Sprint(numberOfGpus)), - }, }, }, }, }, }, } + + if accelerator.IsGpu() { + // Add ResourceLabel to CoveredResources + cqSpec.ResourceGroups[0].CoveredResources = append(cqSpec.ResourceGroups[0].CoveredResources, corev1.ResourceName(accelerator.ResourceLabel)) + + // Add GPU resource quota + cqSpec.ResourceGroups[0].Flavors[0].Resources = append(cqSpec.ResourceGroups[0].Flavors[0].Resources, v1beta1.ResourceQuota{ + Name: corev1.ResourceName(accelerator.ResourceLabel), + NominalQuota: resource.MustParse("1"), + }) + } + return CreateKueueClusterQueue(test, cqSpec) } From c03f1b503a10e51c0c8a4d7c73157096a26062b3 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 21 Mar 2025 10:17:23 -0400 Subject: [PATCH 347/369] update codeflare-common import --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3c9db62da..3ad251351 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 github.com/project-codeflare/appwrapper v1.0.4 - github.com/project-codeflare/codeflare-common v0.0.0-20250317102908-1c124db97844 + github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 diff --git a/go.sum b/go.sum index 15f22815d..432eb578b 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/appwrapper v1.0.4 h1:364zQLX0tsi4LvBBYNKZL7PPbNWPbVU7vK6+/kVV/FQ= github.com/project-codeflare/appwrapper v1.0.4/go.mod h1:A1b6bMFNMX5Btv3ckgeuAHVVZzp1G30pSBe6BE/xJWE= -github.com/project-codeflare/codeflare-common v0.0.0-20250317102908-1c124db97844 h1:hEjZ2pV4Fp81wytijJZ7uHWovKIqirVBA/t1F5hIrbA= -github.com/project-codeflare/codeflare-common v0.0.0-20250317102908-1c124db97844/go.mod h1:DPSv5khRiRDFUD43SF8da+MrVQTWmxNhuKJmwSLOyO0= +github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df h1:na2yAFNrALqd1+EtfejXtAMOXNTHJvKb6+8TLKnfopc= +github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df/go.mod h1:tu2Wk/RgUlTaSPk7TKadn4DEIjz0qhBCF/qSq2ZqVp0= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From 3cd5eba39be2616eb1679e5ab347b496ba23bf13 Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Thu, 27 Mar 2025 23:45:08 +0000 Subject: [PATCH 348/369] only use the applicationsNamespace in openshift the codeflare operator checks for kuberay namespaces to apply network policies so it uses opendatahub DSCInitialization CRD which isn't available on vanilla k8s clusters so added a condition to check for DSCInitialization only when running on openshift This fixes the gatejobs running the codeflare-sdk repo as the e2e test jobs are failing because DSCInitialization isn't available on kind cluster that is being used for testing. --- pkg/controllers/raycluster_controller.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 02c1a307d..842f6694f 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -269,18 +269,21 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Locate the KubeRay operator deployment: // - First try to get the ODH / RHOAI application namespace from the DSCInitialization // - Or fallback to the well-known defaults + // add check if running on openshift or vanilla kubernetes var kubeRayNamespaces []string - dsci := &dsciv1.DSCInitialization{} - err := r.Client.Get(ctx, client.ObjectKey{Name: "default-dsci"}, dsci) - if errors.IsNotFound(err) { - kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} - } else if err != nil { - return ctrl.Result{}, err - } else { - kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} + kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} + + if r.IsOpenShift { + dsci := &dsciv1.DSCInitialization{} + err := r.Client.Get(ctx, client.ObjectKey{Name: "default-dsci"}, dsci) + if err != nil { + return ctrl.Result{}, err + } else { + kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} + } } - _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredHeadNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err := r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredHeadNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update NetworkPolicy") } From 133cc4ce8adab8502aa1ed555ad030c21579245a Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Fri, 28 Mar 2025 12:30:16 +0000 Subject: [PATCH 349/369] use fallback namespaces if dsci cluster isn't found Signed-off-by: Saad Zaher --- pkg/controllers/raycluster_controller.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 842f6694f..28acc0ef4 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -271,16 +271,19 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // - Or fallback to the well-known defaults // add check if running on openshift or vanilla kubernetes var kubeRayNamespaces []string - kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} + kubeRayNamespaces = []string{cluster.Namespace} if r.IsOpenShift { dsci := &dsciv1.DSCInitialization{} + err := r.Client.Get(ctx, client.ObjectKey{Name: "default-dsci"}, dsci) - if err != nil { + if errors.IsNotFound(err) { + kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} + } else if err != nil { return ctrl.Result{}, err - } else { - kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} } + kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} + } _, err := r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredHeadNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) From ef278c014e3613232ef384caf4bce837cc024c0f Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Fri, 28 Mar 2025 15:33:25 +0000 Subject: [PATCH 350/369] only use default-dsci namespace when available Signed-off-by: Saad Zaher --- pkg/controllers/raycluster_controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 28acc0ef4..00ecf21a7 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -281,8 +281,9 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} } else if err != nil { return ctrl.Result{}, err + } else { + kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} } - kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} } From cf1d822f03e202b8e79e0ebfa30cfe398643a939 Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Tue, 1 Apr 2025 13:53:19 +0100 Subject: [PATCH 351/369] use ray-system as default ray namespace Signed-off-by: Saad Zaher --- pkg/controllers/constants.go | 9 +++++++++ pkg/controllers/raycluster_controller.go | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 pkg/controllers/constants.go diff --git a/pkg/controllers/constants.go b/pkg/controllers/constants.go new file mode 100644 index 000000000..afb031437 --- /dev/null +++ b/pkg/controllers/constants.go @@ -0,0 +1,9 @@ +package controllers + +const ( + kubeRayDefaultNamespace = "ray-system" + kubeRayOperatorNamespace = "kuberay-operator" + defaultDSCINamespace = "default-dsci" + odhNamespace = "opendatahub" + rhdsAppsNamespace = "redhat-ods-applications" +) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 00ecf21a7..f22d24d43 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -271,14 +271,14 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // - Or fallback to the well-known defaults // add check if running on openshift or vanilla kubernetes var kubeRayNamespaces []string - kubeRayNamespaces = []string{cluster.Namespace} + kubeRayNamespaces = []string{kubeRayDefaultNamespace} if r.IsOpenShift { dsci := &dsciv1.DSCInitialization{} - err := r.Client.Get(ctx, client.ObjectKey{Name: "default-dsci"}, dsci) + err := r.Client.Get(ctx, client.ObjectKey{Name: defaultDSCINamespace}, dsci) if errors.IsNotFound(err) { - kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} + kubeRayNamespaces = []string{odhNamespace, rhdsAppsNamespace} } else if err != nil { return ctrl.Result{}, err } else { @@ -557,7 +557,7 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf networkingv1ac.NetworkPolicyIngressRule(). WithFrom( networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector(). - WithMatchLabels(map[string]string{"app.kubernetes.io/component": "kuberay-operator"})). + WithMatchLabels(map[string]string{"app.kubernetes.io/component": kubeRayOperatorNamespace})). WithNamespaceSelector(metav1ac.LabelSelector(). WithMatchExpressions(metav1ac.LabelSelectorRequirement(). WithKey(corev1.LabelMetadataName). From 9d1836d2bc9627398a59dd60a9955558c003bef7 Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Tue, 1 Apr 2025 13:54:37 +0100 Subject: [PATCH 352/369] update copyrights for constants.go Signed-off-by: Saad Zaher --- pkg/controllers/constants.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/controllers/constants.go b/pkg/controllers/constants.go index afb031437..c7089c48c 100644 --- a/pkg/controllers/constants.go +++ b/pkg/controllers/constants.go @@ -1,3 +1,19 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package controllers const ( From 6ef824b47c3dea1c076692eb24cc85f94af821fd Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 7 Apr 2025 17:42:42 -0400 Subject: [PATCH 353/369] Update AppWrapper from 1.0.4 to 1.0.7 Significant Changes: + node monitoring: correctly recognize NotReady Nodes as unscheduable + remove stray logging message + enhanced TAS support + Update Kueue from 0.10.1 to 0.10.2 --- Makefile | 4 ++-- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 14 ++++++++++++++ go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index ee7f880c3..d21f8746e 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,12 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v1.0.4 +APPWRAPPER_VERSION ?= v1.0.7 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the default version of Kueue (used for testing) -KUEUE_VERSION ?= v0.10.1 +KUEUE_VERSION ?= v0.10.2 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index c139832a9..2cb2676ef 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v1.0.4 +- github.com/project-codeflare/appwrapper/config/crd?ref=v1.0.7 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index 6a3020756..9f4a9731a 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -148,6 +148,13 @@ spec: items: description: AppWrapperPodSet describes an homogeneous set of pods properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map that may be used to store and retrieve + arbitrary metadata about the PodSet to customize its treatment by the AppWrapper controller. + type: object path: description: Path is the path Component.Template to the PodTemplateSpec for this PodSet type: string @@ -263,6 +270,13 @@ spec: items: description: AppWrapperPodSet describes an homogeneous set of pods properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map that may be used to store and retrieve + arbitrary metadata about the PodSet to customize its treatment by the AppWrapper controller. + type: object path: description: Path is the path Component.Template to the PodTemplateSpec for this PodSet type: string diff --git a/go.mod b/go.mod index 3ad251351..1b5d2e44b 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,11 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 - github.com/project-codeflare/appwrapper v1.0.4 + github.com/project-codeflare/appwrapper v1.0.7 github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 k8s.io/api v0.31.4 k8s.io/apiextensions-apiserver v0.31.2 k8s.io/apimachinery v0.31.4 @@ -23,7 +23,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.19.3 - sigs.k8s.io/kueue v0.10.1 + sigs.k8s.io/kueue v0.10.2 sigs.k8s.io/yaml v1.4.0 ) @@ -99,7 +99,7 @@ require ( golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.26.0 // indirect + golang.org/x/tools v0.28.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/go.sum b/go.sum index 432eb578b..80ee5fb6e 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v1.0.4 h1:364zQLX0tsi4LvBBYNKZL7PPbNWPbVU7vK6+/kVV/FQ= -github.com/project-codeflare/appwrapper v1.0.4/go.mod h1:A1b6bMFNMX5Btv3ckgeuAHVVZzp1G30pSBe6BE/xJWE= +github.com/project-codeflare/appwrapper v1.0.7 h1:sbSXpyqEnlZJNZrYC8bk5FH88BXd4WQemiVAu0wT8iQ= +github.com/project-codeflare/appwrapper v1.0.7/go.mod h1:m5GmMVTZfMI7kOmvtUZeGXQbaBZOwWKzRY4tSZO6kn8= github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df h1:na2yAFNrALqd1+EtfejXtAMOXNTHJvKb6+8TLKnfopc= github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df/go.mod h1:tu2Wk/RgUlTaSPk7TKadn4DEIjz0qhBCF/qSq2ZqVp0= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= @@ -276,8 +276,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -370,8 +370,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -469,8 +469,8 @@ sigs.k8s.io/jobset v0.7.1/go.mod h1:cbBuQ6QrTU88x5PrDqcC5AI28YyTSQCIcRTLNTJoHDE= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kueue v0.10.1 h1:td+Nae1z9L4IrkqJKOIMsKp2TmwUAvk/hWKhSaFupwk= -sigs.k8s.io/kueue v0.10.1/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= +sigs.k8s.io/kueue v0.10.2 h1:p0C2zZiSCb6Dt77jCU/DMx2LFujN4etIGjjj7q+Ylao= +sigs.k8s.io/kueue v0.10.2/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.3 h1:sCP7Vv3xx/CWIuTPVN38lUPx0uw0lcLfzaiDa8Ja01A= From bebce01d85de2ac098d74506b4a3d2318ed0b096 Mon Sep 17 00:00:00 2001 From: Pat O'Connor Date: Mon, 14 Apr 2025 17:16:40 +0100 Subject: [PATCH 354/369] task(RHOAIENG-22446): Updated OWNERS with Ray Team Signed-off-by: Pat O'Connor --- OWNERS | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/OWNERS b/OWNERS index 14ed5926f..ab7c4e938 100644 --- a/OWNERS +++ b/OWNERS @@ -1,21 +1,31 @@ approvers: - astefanutti + - chipspeak - ChristianZaccaria - jbusche - kpostoffice + - kryanbeane + - laurafitzgerald + - pawelpaszki - sutaakar + - szaher - tedhtchang - varshaprasad96 reviewers: - astefanutti - Bobbins228 + - chipspeak - ChristianZaccaria - dimakis - Fiona-Waters - jbusche - kpostoffice + - kryanbeane + - laurafitzgerald + - pawelpaszki - sutaakar + - szaher - tedhtchang - varshaprasad96 From 35e76cb5f5c132754fae5a0307fc32b98268323f Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Mon, 14 Apr 2025 14:20:34 +0100 Subject: [PATCH 355/369] Auto-Detect kuberay-operator namespace Enable auto-detection of the namespace where kuberay-operator is deployed so we can enable network traffic between raycluster namespace and kuberay-operator namespace. Signed-off-by: Saad Zaher --- pkg/controllers/raycluster_controller.go | 48 ++++++++++++++++++++---- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index f22d24d43..92f8cf0cb 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -270,8 +270,16 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // - First try to get the ODH / RHOAI application namespace from the DSCInitialization // - Or fallback to the well-known defaults // add check if running on openshift or vanilla kubernetes + kubeRayNamespace, err := r.getKubeRayOperatorNamespace(ctx) + if err != nil { + logger.Error(err, "Failed to get KubeRay operator namespace") + + return ctrl.Result{RequeueAfter: requeueTime}, err + } + logger.Info("Detected KubeRay operator namespace", "namespace", kubeRayNamespace) + var kubeRayNamespaces []string - kubeRayNamespaces = []string{kubeRayDefaultNamespace} + kubeRayNamespaces = []string{kubeRayNamespace} if r.IsOpenShift { dsci := &dsciv1.DSCInitialization{} @@ -287,7 +295,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } - _, err := r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredHeadNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredHeadNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update NetworkPolicy") } @@ -319,6 +327,32 @@ func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool { return cfg == nil || ptr.Deref(cfg.MTLSEnabled, true) } +// getKubeRayOperatorNamespace tries to get the namespace of the KubeRay operator +func (r *RayClusterReconciler) getKubeRayOperatorNamespace(ctx context.Context) (string, error) { + logger := ctrl.LoggerFrom(ctx) + + pods, err := r.kubeClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=kuberay-operator", + }) + if err != nil { + logger.Error(err, "failed to get kuberay-operator namespace") + + return kubeRayDefaultNamespace, err + } + + if len(pods.Items) == 0 { + logger.Info( + "No kuberay-operator pods found, using default kuberay-operator namespace", + "namespace", + kubeRayDefaultNamespace, + ) + + return kubeRayDefaultNamespace, nil + } + + return pods.Items[0].Namespace, nil +} + func isRayClusterSuspended(cluster *rayv1.RayCluster) bool { return cluster.Spec.Suspend != nil && ptr.Deref(cluster.Spec.Suspend, false) } @@ -549,8 +583,8 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf ), networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt32(10001)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt32(8265)), ).WithFrom( networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector()), ), @@ -564,12 +598,12 @@ func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConf WithOperator(metav1.LabelSelectorOpIn). WithValues(kubeRayNamespaces...)))). WithPorts( - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt32(8265)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt32(10001)), ), networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt32(8080)), ). WithFrom( networkingv1ac.NetworkPolicyPeer().WithNamespaceSelector(metav1ac.LabelSelector(). From d2d22ede2a25821008ca74a6af1fe213a7613237 Mon Sep 17 00:00:00 2001 From: Laura Fitzgerald Date: Wed, 14 May 2025 14:50:49 +0100 Subject: [PATCH 356/369] update route link as previous file was removed --- pkg/controllers/suite_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index 6828bdada..52ef093e4 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -61,7 +61,7 @@ func TestAPIs(t *testing.T) { const ( RayClusterCRDFileDownload = "https://raw.githubusercontent.com/ray-project/kuberay/master/ray-operator/config/crd/bases/ray.io_rayclusters.yaml" - RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/zz_generated.crd-manifests/routes-Default.crd.yaml" + RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/zz_generated.crd-manifests/routes.crd.yaml" ) var _ = BeforeSuite(func() { @@ -79,6 +79,8 @@ var _ = BeforeSuite(func() { defer fRoute.Close() resp, err := http.Get(RouteCRDFileDownload) Expect(err).ToNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK), "Failed to download Route CRD: %s", RouteCRDFileDownload) + defer resp.Body.Close() _, err = io.Copy(fRoute, resp.Body) Expect(err).ToNot(HaveOccurred()) fRaycluster, err = os.Create("./test-crds/raycluster.yaml") @@ -86,6 +88,8 @@ var _ = BeforeSuite(func() { defer fRaycluster.Close() resp, err = http.Get(RayClusterCRDFileDownload) Expect(err).ToNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK), "Failed to download RayCluster CRD: %s", RayClusterCRDFileDownload) + defer resp.Body.Close() _, err = io.Copy(fRaycluster, resp.Body) Expect(err).ToNot(HaveOccurred()) From caa859d60ce10f72dea88a0bffc375185826ffcf Mon Sep 17 00:00:00 2001 From: Pat O'Connor Date: Wed, 14 May 2025 10:41:13 +0100 Subject: [PATCH 357/369] task(RHOAIENG-25519): Update CodeFlare Compat Matrix Signed-off-by: Pat O'Connor --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ad60de570..2b2b9f4f0 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.15.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.15.0) | -| CodeFlare-SDK | [v0.27.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.27.0) | -| AppWrapper | [v1.0.4](https://github.com/project-codeflare/appwrapper/releases/tag/v1.0.4) | -| KubeRay | [v1.2.2](https://github.com/ray-project/kuberay/releases/tag/v1.2.2) | -| Kueue | [v0.10.1](https://github.com/kubernetes-sigs/kueue/releases/tag/v0.10.1) | +| CodeFlare-SDK | [v0.28.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.28.1) | +| AppWrapper | [v1.1.2](https://github.com/project-codeflare/appwrapper/releases/tag/v1.1.2) | +| KubeRay | [v1.3.2](https://github.com/ray-project/kuberay/releases/tag/v1.3.2) | +| Kueue | [v0.11.4](https://github.com/kubernetes-sigs/kueue/releases/tag/v0.11.4) | ## Development From 751cfa96a1957dbc174ba18da24f4d7a201245a5 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 20 Mar 2025 16:43:08 -0400 Subject: [PATCH 358/369] Bump AppWrapper version to 1.1.1 --- Makefile | 4 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/rbac/role.yaml | 68 ----- go.mod | 97 +++--- go.sum | 369 ++++++++++++++++++----- main.go | 37 +-- pkg/controllers/appwrapper_controller.go | 18 +- 7 files changed, 356 insertions(+), 239 deletions(-) diff --git a/Makefile b/Makefile index d21f8746e..a34d355c3 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,12 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v1.0.7 +APPWRAPPER_VERSION ?= v1.1.1 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the default version of Kueue (used for testing) -KUEUE_VERSION ?= v0.10.2 +KUEUE_VERSION ?= v0.11.0 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 2cb2676ef..606c465d8 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v1.0.7 +- github.com/project-codeflare/appwrapper/config/crd?ref=v1.1.1 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b4be4fc2b..ad958dae6 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -14,14 +14,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch - apiGroups: - "" resources: @@ -188,58 +180,6 @@ rules: - patch - update - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloadpriorityclasses - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/finalizers - verbs: - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get - - patch - - update - apiGroups: - networking.k8s.io resources: @@ -327,14 +267,6 @@ rules: - patch - update - watch -- apiGroups: - - scheduling.k8s.io - resources: - - priorityclasses - verbs: - - get - - list - - watch - apiGroups: - scheduling.sigs.k8s.io resources: diff --git a/go.mod b/go.mod index 1b5d2e44b..7b749e12b 100644 --- a/go.mod +++ b/go.mod @@ -4,31 +4,41 @@ go 1.23.0 require ( github.com/go-logr/logr v1.4.2 - github.com/onsi/ginkgo/v2 v2.22.0 - github.com/onsi/gomega v1.36.1 + github.com/onsi/ginkgo/v2 v2.23.0 + github.com/onsi/gomega v1.36.2 github.com/open-policy-agent/cert-controller v0.12.0 github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 - github.com/project-codeflare/appwrapper v1.0.7 + github.com/project-codeflare/appwrapper v1.1.1 github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df - github.com/ray-project/kuberay/ray-operator v1.2.2 + github.com/ray-project/kuberay/ray-operator v1.3.1 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - k8s.io/api v0.31.4 - k8s.io/apiextensions-apiserver v0.31.2 - k8s.io/apimachinery v0.31.4 + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 + k8s.io/api v0.32.3 + k8s.io/apiextensions-apiserver v0.32.3 + k8s.io/apimachinery v0.32.3 k8s.io/client-go v11.0.0+incompatible - k8s.io/component-base v0.31.4 + k8s.io/component-base v0.32.3 k8s.io/klog/v2 v2.130.1 - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 - sigs.k8s.io/controller-runtime v0.19.3 - sigs.k8s.io/kueue v0.10.2 + k8s.io/utils v0.0.0-20241210054802-24370beab758 + sigs.k8s.io/controller-runtime v0.20.3 + sigs.k8s.io/kueue v0.11.0 sigs.k8s.io/yaml v1.4.0 ) replace k8s.io/client-go => k8s.io/client-go v0.31.4 +replace k8s.io/api => k8s.io/api v0.31.4 + +replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.31.2 + +replace k8s.io/apimachinery v0.32.3 => k8s.io/apimachinery v0.31.4 + +replace sigs.k8s.io/controller-runtime v0.20.3 => sigs.k8s.io/controller-runtime v0.19.3 + +replace sigs.k8s.io/kueue v0.11.0 => sigs.k8s.io/kueue v0.10.1 + replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 @@ -36,33 +46,32 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // Workaround deprecation of cluster.Status.State in ray-operator v1.2.2 -replace github.com/ray-project/kuberay/ray-operator v1.2.2 => github.com/ray-project/kuberay/ray-operator v1.2.1 +replace github.com/ray-project/kuberay/ray-operator v1.3.1 => github.com/ray-project/kuberay/ray-operator v1.2.1 require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonpointer v0.21.1 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/glog v1.2.4 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -70,10 +79,9 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/kubeflow/mpi-operator v0.6.0 // indirect - github.com/kubeflow/training-operator v1.8.1 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/kubeflow/training-operator v1.9.0 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -83,33 +91,32 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_golang v1.21.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.57.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/x448/float16 v0.8.4 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.28.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect + golang.org/x/net v0.37.0 // indirect + golang.org/x/oauth2 v0.28.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.11.0 // indirect + golang.org/x/tools v0.31.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.31.4 // indirect - k8s.io/component-helpers v0.31.4 // indirect - k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect - sigs.k8s.io/jobset v0.7.1 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.3 // indirect + k8s.io/kube-openapi v0.0.0-20250304201544-e5f78fe3ede9 // indirect + sigs.k8s.io/jobset v0.8.0 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect ) diff --git a/go.sum b/go.sum index 80ee5fb6e..c3320ef79 100644 --- a/go.sum +++ b/go.sum @@ -3,43 +3,49 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= -github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= -github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -50,27 +56,39 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= +github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -92,25 +110,31 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -122,6 +146,7 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= @@ -147,31 +172,33 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeflow/mpi-operator v0.6.0 h1:RzzG03oyQIiHe/SH2k19N+Hp4QkxEsG2l9G1T7US+ys= -github.com/kubeflow/mpi-operator v0.6.0/go.mod h1:lu3cj3ESq3SdS29nbHC26TuXiTYYQYTHJ6fcy5Xv20c= -github.com/kubeflow/training-operator v1.8.1 h1:mVvoBSCInbMBX0gGSn4+Ihj3ycwoLO1iEAw+0qPhQfc= -github.com/kubeflow/training-operator v1.8.1/go.mod h1:T6I15h1S09ncH5C6St/QEC7Dy6dpHZA5sPFo+VoJAvE= +github.com/kubeflow/training-operator v1.9.0 h1:L+ep5YQT1Pq62O3VjW6G+IRQw+NpFlPRJJj8TgydBhQ= +github.com/kubeflow/training-operator v1.9.0/go.mod h1:tkXcAngbhpdskDE75smgfdqOW17tmWJ+2389+FzMNvo= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -193,15 +220,50 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= -github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.23.0 h1:FA1xjp8ieYDzlgS5ABTpdUDB7wtngggONc8a7ku2NqQ= +github.com/onsi/ginkgo/v2 v2.23.0/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= -github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= +github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/open-policy-agent/cert-controller v0.12.0 h1:RKXlBafMcCh+++I1geJetXo77tAjyb4542DQc/+aZIw= github.com/open-policy-agent/cert-controller v0.12.0/go.mod h1:N5bCFXdAXMYx0PdS6ZQ9lrDQQMz+F6deoChym6VleXw= github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a h1:gQtOJ50XFyL2Xh3lDD9zP4KQ2PY4mZKQ9hDcWc81Sp8= @@ -218,26 +280,29 @@ github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 h1:mLJlN8PdZgI github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202/go.mod h1:D3yfotGqRY1LWaHcoSCeadhdyvfrQWYJXui1M+JqKvY= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v1.0.7 h1:sbSXpyqEnlZJNZrYC8bk5FH88BXd4WQemiVAu0wT8iQ= -github.com/project-codeflare/appwrapper v1.0.7/go.mod h1:m5GmMVTZfMI7kOmvtUZeGXQbaBZOwWKzRY4tSZO6kn8= +github.com/project-codeflare/appwrapper v1.1.1 h1:ymFtH4bWpFyf4MgLvzONEgJ5A9utjAU5kSMLpSyeaBU= +github.com/project-codeflare/appwrapper v1.1.1/go.mod h1:+Pfxe92YSxrFqO1ovRZTQ1PRrQqkdq5fLCxn1Ok6ZlQ= github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df h1:na2yAFNrALqd1+EtfejXtAMOXNTHJvKb6+8TLKnfopc= github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df/go.mod h1:tu2Wk/RgUlTaSPk7TKadn4DEIjz0qhBCF/qSq2ZqVp0= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= +github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= -github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= +github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= github.com/ray-project/kuberay/ray-operator v1.2.1 h1:H7ofodGclghsU2TxbDHs+gvqvsOp5DJ/vAPGySL1DIE= github.com/ray-project/kuberay/ray-operator v1.2.1/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -245,16 +310,25 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -262,6 +336,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -273,11 +348,24 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -285,6 +373,17 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -296,6 +395,7 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -304,19 +404,48 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= +golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -340,23 +469,80 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -370,14 +556,31 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= -gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -399,8 +602,11 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -427,7 +633,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.31.4 h1:I2QNzitPVsPeLQvexMEsj945QumYraqv9m74isPDKhM= k8s.io/api v0.31.4/go.mod h1:d+7vgXLvmcdT1BCo79VEgJxHHryww3V5np2OYTr6jdw= k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0= @@ -435,46 +640,54 @@ k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKW k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM= k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apiserver v0.31.4 h1:JbtnTaXVYEAYIHJil6Wd74Wif9sd8jVcBw84kwEmp7o= -k8s.io/apiserver v0.31.4/go.mod h1:JJjoTjZ9PTMLdIFq7mmcJy2B9xLN3HeAUebW6xZyIP0= k8s.io/client-go v0.31.4 h1:t4QEXt4jgHIkKKlx06+W3+1JOwAFU/2OPiOo7H92eRQ= k8s.io/client-go v0.31.4/go.mod h1:kvuMro4sFYIa8sulL5Gi5GFqUPvfH2O/dXuKstbaaeg= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.31.4 h1:wCquJh4ul9O8nNBSB8N/o8+gbfu3BVQkVw9jAUY/Qtw= -k8s.io/component-base v0.31.4/go.mod h1:G4dgtf5BccwiDT9DdejK0qM6zTK0jwDGEKnCmb9+u/s= -k8s.io/component-helpers v0.31.4 h1:pqokuXozyWVrVBMmx0AMcKqNWqXhR00OZvpAE5hG5NM= -k8s.io/component-helpers v0.31.4/go.mod h1:Ddq5GYRK/1uNoPNgJh9N5osPutvBweQEcIG6b8kcvgQ= +k8s.io/component-base v0.32.3 h1:98WJvvMs3QZ2LYHBzvltFSeJjEx7t5+8s71P7M74u8k= +k8s.io/component-base v0.32.3/go.mod h1:LWi9cR+yPAv7cu2X9rZanTiFKB2kHA+JjmhkKjCZRpI= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-aggregator v0.31.2 h1:Uw1zUP2D/4wiSjKWVVzSOcCGLuW/+IdRwjjC0FJooYU= k8s.io/kube-aggregator v0.31.2/go.mod h1:41/VIXH+/Qcg9ERNAY6bRF/WQR6xL1wFgYagdHac1X4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f h1:bnWtxXWdAl5bVOCEPoNdvMkyj6cTW3zxHuwKIakuV9w= -k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f/go.mod h1:G0W3eI9gG219NHRq3h5uQaRBl4pj4ZpwzRP5ti8y770= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20250304201544-e5f78fe3ede9 h1:t0huyHnz6HsokckRxAF1bY0cqPFwzINKCL7yltEjZQc= +k8s.io/kube-openapi v0.0.0-20250304201544-e5f78fe3ede9/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= -sigs.k8s.io/jobset v0.7.1 h1:m28IaaKrQyZ8qa0Q7jK3U5/6TEW+27QPALjlalLP/0A= -sigs.k8s.io/jobset v0.7.1/go.mod h1:cbBuQ6QrTU88x5PrDqcC5AI28YyTSQCIcRTLNTJoHDE= +sigs.k8s.io/jobset v0.8.0 h1:80cJcPld+IMdKFOqzEW4et3Y6lGAPcP8YmBZ+aiKGYA= +sigs.k8s.io/jobset v0.8.0/go.mod h1:yitjuGOExl2p964nhyevQGIkfiPSRHcdC3zNBneKCT8= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kueue v0.10.2 h1:p0C2zZiSCb6Dt77jCU/DMx2LFujN4etIGjjj7q+Ylao= -sigs.k8s.io/kueue v0.10.2/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kueue v0.10.1 h1:td+Nae1z9L4IrkqJKOIMsKp2TmwUAvk/hWKhSaFupwk= +sigs.k8s.io/kueue v0.10.1/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.3 h1:sCP7Vv3xx/CWIuTPVN38lUPx0uw0lcLfzaiDa8Ja01A= -sigs.k8s.io/structured-merge-diff/v4 v4.4.3/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/main.go b/main.go index 044716fee..b7cf9ff52 100644 --- a/main.go +++ b/main.go @@ -63,7 +63,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/yaml" routev1 "github.com/openshift/api/route/v1" @@ -83,7 +82,6 @@ var ( ) const ( - workloadAPI = "workloads.kueue.x-k8s.io" rayclusterAPI = "rayclusters.ray.io" ) @@ -97,8 +95,6 @@ func init() { utilruntime.Must(dsciv1.AddToScheme(scheme)) // AppWrapper utilruntime.Must(awv1beta2.AddToScheme(scheme)) - // Kueue - utilruntime.Must(kueue.AddToScheme(scheme)) } // +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get @@ -277,38 +273,17 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m return nil } - // AppWrapper webhook doesn't depend on WorkloadAPI availablity but does need certsReady - go setupAppWrapperWebhooks(mgr, cfg, certsReady) - - if isAPIAvailable(ctx, mgr, workloadAPI) { - setupLog.Info("Workload API available, enabling AppWrappers") - go setupAppWrapperController(mgr, cfg, certsReady) - return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config) - } else { - // If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly - // restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator. - // It is not possible to add an indexer once the mgr has started so, a restart if the only avenue. - setupLog.Info("Workload API not available, setting up waiter for Workload API availability") - go waitForAPI(ctx, mgr, workloadAPI, func() { - setupLog.Info("Workload API now available, triggering controller restart") - cancel() - }) - return nil - } + // AppWrapper webhook and controller need certsReady + go setupAppWrapperControllerAndWebhooks(mgr, cfg, certsReady) + return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config) } -func setupAppWrapperWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { +func setupAppWrapperControllerAndWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { setupLog.Info("Waiting for certificate generation to complete") <-certsReady - setupLog.Info("Setting up AppWrapper webhooks") - exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") -} - -func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) { - setupLog.Info("Waiting for certificate generation to complete") - <-certsReady - setupLog.Info("Setting up AppWrapper controller") + setupLog.Info("Setting up AppWrapper controller and webhooks") exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper controller") + exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks") } // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update diff --git a/pkg/controllers/appwrapper_controller.go b/pkg/controllers/appwrapper_controller.go index ed65accd8..172099f33 100644 --- a/pkg/controllers/appwrapper_controller.go +++ b/pkg/controllers/appwrapper_controller.go @@ -23,6 +23,9 @@ package controllers //+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/status,verbs=get;update;patch //+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/finalizers,verbs=update +// permission for events +//+kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update;patch + // permission to edit wrapped resources: pods, services, jobs, podgroups, pytorchjobs, rayclusters //+kubebuilder:rbac:groups="",resources=pods;services,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete @@ -34,18 +37,5 @@ package controllers //+kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=jobset.x-k8s.io,resources=jobsets,verbs=get;list;watch;create;update;patch;delete -// permissions needed by Kueue's generic reconciller -// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=list;get;watch -// +kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update;patch -// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/finalizers,verbs=update -// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch -// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch - -// permissions needed by Kueue's base webhook -// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch - -// permission to watch nodes and edit clusterqueues for Autopilot integration +// permission to watch nodes for the Autopilot integration //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch -//+kubebuilder:rbac:groups=kueue.x-k8s.io,resources=clusterqueues,verbs=get;list;watch;update;patch From 167c65f0a6a69a25e404b1417f8a632a1661da0c Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 24 Mar 2025 11:41:31 -0400 Subject: [PATCH 359/369] Looser coupling of Kueue versions Decouple the Kueue version deployed on the cluster for CI testing from the golang dependency on Kueue APIs used in the e2e go test code. --- Makefile | 3 +-- go.mod | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index a34d355c3..a8167395e 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ APPWRAPPER_VERSION ?= v1.1.1 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} -# KUEUE_VERSION defines the default version of Kueue (used for testing) +# KUEUE_VERSION defines the version of Kueue deployed for testing KUEUE_VERSION ?= v0.11.0 USE_RHOAI ?= true @@ -160,7 +160,6 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. go get github.com/ray-project/kuberay/ray-operator@$(KUBERAY_VERSION) - go get sigs.k8s.io/kueue@$(KUEUE_VERSION) go get github.com/project-codeflare/appwrapper@$(APPWRAPPER_VERSION) go mod tidy diff --git a/go.mod b/go.mod index 7b749e12b..0628408ee 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 github.com/project-codeflare/appwrapper v1.1.1 github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df - github.com/ray-project/kuberay/ray-operator v1.3.1 + github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 k8s.io/api v0.32.3 @@ -23,7 +23,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20241210054802-24370beab758 sigs.k8s.io/controller-runtime v0.20.3 - sigs.k8s.io/kueue v0.11.0 + sigs.k8s.io/kueue v0.10.1 sigs.k8s.io/yaml v1.4.0 ) @@ -37,8 +37,6 @@ replace k8s.io/apimachinery v0.32.3 => k8s.io/apimachinery v0.31.4 replace sigs.k8s.io/controller-runtime v0.20.3 => sigs.k8s.io/controller-runtime v0.19.3 -replace sigs.k8s.io/kueue v0.11.0 => sigs.k8s.io/kueue v0.10.1 - replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 @@ -46,7 +44,7 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // Workaround deprecation of cluster.Status.State in ray-operator v1.2.2 -replace github.com/ray-project/kuberay/ray-operator v1.3.1 => github.com/ray-project/kuberay/ray-operator v1.2.1 +replace github.com/ray-project/kuberay/ray-operator v1.2.2 => github.com/ray-project/kuberay/ray-operator v1.2.1 require ( github.com/aymerick/douceur v0.2.0 // indirect From a3e2846bcbd2e119eb91d5de33fb127735e2896f Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 28 May 2025 11:44:29 -0400 Subject: [PATCH 360/369] Update to AppWrapper v1.1.2 --- Makefile | 2 +- config/crd/appwrapper/kustomization.yaml | 2 +- config/crd/crd-appwrapper.yml | 12 ++++++------ go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index a8167395e..b1283af68 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # APPWRAPPER_VERSION defines the default version of the AppWrapper controller -APPWRAPPER_VERSION ?= v1.1.1 +APPWRAPPER_VERSION ?= v1.1.2 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} diff --git a/config/crd/appwrapper/kustomization.yaml b/config/crd/appwrapper/kustomization.yaml index 606c465d8..d86cb5d82 100644 --- a/config/crd/appwrapper/kustomization.yaml +++ b/config/crd/appwrapper/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/appwrapper/config/crd?ref=v1.1.1 +- github.com/project-codeflare/appwrapper/config/crd?ref=v1.1.2 diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml index 9f4a9731a..2fb8f5bda 100644 --- a/config/crd/crd-appwrapper.yml +++ b/config/crd/crd-appwrapper.yml @@ -144,9 +144,9 @@ spec: type: object type: array podSets: - description: DeclaredPodSets for the Component (optional for known PodCreating GVKs) + description: DeclaredPodSets for the Component (optional for known GVKs whose PodSets can be automatically inferred) items: - description: AppWrapperPodSet describes an homogeneous set of pods + description: AppWrapperPodSet describes a homogeneous set of pods properties: annotations: additionalProperties: @@ -156,7 +156,7 @@ spec: arbitrary metadata about the PodSet to customize its treatment by the AppWrapper controller. type: object path: - description: Path is the path Component.Template to the PodTemplateSpec for this PodSet + description: Path is the path within Component.Template to the PodTemplateSpec for this PodSet type: string replicas: description: Replicas is the number of pods in this PodSet @@ -185,7 +185,7 @@ spec: - components type: object status: - description: AppWrapperStatus defines the observed state of the appwrapper + description: AppWrapperStatus defines the observed state of the AppWrapper properties: componentStatus: description: ComponentStatus parallels the Components array in the Spec and tracks the actually deployed resources @@ -268,7 +268,7 @@ spec: podSets: description: PodSets is the validated PodSets for the Component (either from AppWrapperComponent.DeclaredPodSets or inferred by the controller) items: - description: AppWrapperPodSet describes an homogeneous set of pods + description: AppWrapperPodSet describes a homogeneous set of pods properties: annotations: additionalProperties: @@ -278,7 +278,7 @@ spec: arbitrary metadata about the PodSet to customize its treatment by the AppWrapper controller. type: object path: - description: Path is the path Component.Template to the PodTemplateSpec for this PodSet + description: Path is the path within Component.Template to the PodTemplateSpec for this PodSet type: string replicas: description: Replicas is the number of pods in this PodSet diff --git a/go.mod b/go.mod index 0628408ee..acb6d78ea 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 github.com/openshift/api v0.0.0-20240904015708-69df64132c91 github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 - github.com/project-codeflare/appwrapper v1.1.1 + github.com/project-codeflare/appwrapper v1.1.2 github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df github.com/ray-project/kuberay/ray-operator v1.2.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index c3320ef79..47c2e43a1 100644 --- a/go.sum +++ b/go.sum @@ -287,8 +287,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/appwrapper v1.1.1 h1:ymFtH4bWpFyf4MgLvzONEgJ5A9utjAU5kSMLpSyeaBU= -github.com/project-codeflare/appwrapper v1.1.1/go.mod h1:+Pfxe92YSxrFqO1ovRZTQ1PRrQqkdq5fLCxn1Ok6ZlQ= +github.com/project-codeflare/appwrapper v1.1.2 h1:iEEyYGSDXOxMXc/AZEvTDDXy1DNi+cWUQ93uM6XAXJc= +github.com/project-codeflare/appwrapper v1.1.2/go.mod h1:+Pfxe92YSxrFqO1ovRZTQ1PRrQqkdq5fLCxn1Ok6ZlQ= github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df h1:na2yAFNrALqd1+EtfejXtAMOXNTHJvKb6+8TLKnfopc= github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df/go.mod h1:tu2Wk/RgUlTaSPk7TKadn4DEIjz0qhBCF/qSq2ZqVp0= github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= From 91eb1700a8fe1599cef6c3460d7ebd14cbff0f08 Mon Sep 17 00:00:00 2001 From: David Grove Date: Wed, 28 May 2025 12:40:14 -0400 Subject: [PATCH 361/369] deploy Kueue 0.11.5 in CI tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b1283af68..494a57eae 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the version of Kueue deployed for testing -KUEUE_VERSION ?= v0.11.0 +KUEUE_VERSION ?= v0.11.5 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) From 266891b9b83f5285bcf5abe6c058df2ef521d648 Mon Sep 17 00:00:00 2001 From: David Grove Date: Mon, 9 Jun 2025 09:14:56 -0400 Subject: [PATCH 362/369] bump kueue from 0.11.5 to 0.11.6 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 494a57eae..c1f418041 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION} # KUEUE_VERSION defines the version of Kueue deployed for testing -KUEUE_VERSION ?= v0.11.5 +KUEUE_VERSION ?= v0.11.6 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) From cf4026db3f5e48109499c4ac5d39b0b79a9e1f3f Mon Sep 17 00:00:00 2001 From: Pawel Paszki Date: Mon, 16 Jun 2025 09:45:51 +0100 Subject: [PATCH 363/369] conf: update dependencies to match versions used in the latest planned release --- Makefile | 2 +- go.mod | 12 +++++++----- go.sum | 8 ++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index c1f418041..31ce98cd6 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ KUEUE_VERSION ?= v0.11.6 USE_RHOAI ?= true # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.2.2 +KUBERAY_VERSION ?= v1.3.2 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/go.mod b/go.mod index acb6d78ea..3aaabd1f7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/project-codeflare/codeflare-operator -go 1.23.0 +go 1.24.0 + +toolchain go1.24.4 require ( github.com/go-logr/logr v1.4.2 @@ -12,7 +14,7 @@ require ( github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202 github.com/project-codeflare/appwrapper v1.1.2 github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df - github.com/ray-project/kuberay/ray-operator v1.2.2 + github.com/ray-project/kuberay/ray-operator v1.3.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 k8s.io/api v0.32.3 @@ -23,7 +25,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20241210054802-24370beab758 sigs.k8s.io/controller-runtime v0.20.3 - sigs.k8s.io/kueue v0.10.1 + sigs.k8s.io/kueue v0.11.6 sigs.k8s.io/yaml v1.4.0 ) @@ -43,8 +45,8 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 -// Workaround deprecation of cluster.Status.State in ray-operator v1.2.2 -replace github.com/ray-project/kuberay/ray-operator v1.2.2 => github.com/ray-project/kuberay/ray-operator v1.2.1 +// Workaround deprecation of cluster.Status.State in ray-operator v1.3.2 +replace github.com/ray-project/kuberay/ray-operator v1.3.2 => github.com/ray-project/kuberay/ray-operator v1.2.2 require ( github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 47c2e43a1..58d493837 100644 --- a/go.sum +++ b/go.sum @@ -300,8 +300,8 @@ github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= -github.com/ray-project/kuberay/ray-operator v1.2.1 h1:H7ofodGclghsU2TxbDHs+gvqvsOp5DJ/vAPGySL1DIE= -github.com/ray-project/kuberay/ray-operator v1.2.1/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= +github.com/ray-project/kuberay/ray-operator v1.2.2 h1:wj4qe9SmJfD1ubgEaVPuAsnU/WFDvremzR8j3JslBdk= +github.com/ray-project/kuberay/ray-operator v1.2.2/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= @@ -677,8 +677,8 @@ sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNza sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/kueue v0.10.1 h1:td+Nae1z9L4IrkqJKOIMsKp2TmwUAvk/hWKhSaFupwk= -sigs.k8s.io/kueue v0.10.1/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= +sigs.k8s.io/kueue v0.11.6 h1:ZuI5GY3aMrARO2iP1/7H9CkEiqYupqnt/bprchHzE0I= +sigs.k8s.io/kueue v0.11.6/go.mod h1:I/5sQDCTcUSpHYzHSv0jeJVLMOp2L92g7XDbJzqfsnE= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= From 7e10e68b6e0089e56cd82c44546cfdb6e9e82f9c Mon Sep 17 00:00:00 2001 From: Pawel Paszki Date: Mon, 16 Jun 2025 10:00:30 +0100 Subject: [PATCH 364/369] conf: revert kueue update --- go.mod | 8 +++----- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 3aaabd1f7..3e12e7959 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/project-codeflare/codeflare-operator -go 1.24.0 - -toolchain go1.24.4 +go 1.23.0 require ( github.com/go-logr/logr v1.4.2 @@ -25,7 +23,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20241210054802-24370beab758 sigs.k8s.io/controller-runtime v0.20.3 - sigs.k8s.io/kueue v0.11.6 + sigs.k8s.io/kueue v0.10.1 sigs.k8s.io/yaml v1.4.0 ) @@ -46,7 +44,7 @@ replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.open replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 // Workaround deprecation of cluster.Status.State in ray-operator v1.3.2 -replace github.com/ray-project/kuberay/ray-operator v1.3.2 => github.com/ray-project/kuberay/ray-operator v1.2.2 +replace github.com/ray-project/kuberay/ray-operator v1.3.2 => github.com/ray-project/kuberay/ray-operator v1.2.1 require ( github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 58d493837..47c2e43a1 100644 --- a/go.sum +++ b/go.sum @@ -300,8 +300,8 @@ github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= -github.com/ray-project/kuberay/ray-operator v1.2.2 h1:wj4qe9SmJfD1ubgEaVPuAsnU/WFDvremzR8j3JslBdk= -github.com/ray-project/kuberay/ray-operator v1.2.2/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= +github.com/ray-project/kuberay/ray-operator v1.2.1 h1:H7ofodGclghsU2TxbDHs+gvqvsOp5DJ/vAPGySL1DIE= +github.com/ray-project/kuberay/ray-operator v1.2.1/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= @@ -677,8 +677,8 @@ sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNza sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/kueue v0.11.6 h1:ZuI5GY3aMrARO2iP1/7H9CkEiqYupqnt/bprchHzE0I= -sigs.k8s.io/kueue v0.11.6/go.mod h1:I/5sQDCTcUSpHYzHSv0jeJVLMOp2L92g7XDbJzqfsnE= +sigs.k8s.io/kueue v0.10.1 h1:td+Nae1z9L4IrkqJKOIMsKp2TmwUAvk/hWKhSaFupwk= +sigs.k8s.io/kueue v0.10.1/go.mod h1:3yzOvGI0sPOC3VL1ihVIrzc8mkSyCVTL+SrouewwRWw= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= From 9cd648c15a87928a38a46f3374d631a62f9df1cc Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Mon, 16 Jun 2025 10:20:05 +0000 Subject: [PATCH 365/369] Update dependency versions for release v1.16.0 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2b2b9f4f0..50228b780 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.15.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.15.0) | -| CodeFlare-SDK | [v0.28.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.28.1) | +| CodeFlare Operator | [v1.16.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.16.0) | +| CodeFlare-SDK | [v0.29.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.29.0) | | AppWrapper | [v1.1.2](https://github.com/project-codeflare/appwrapper/releases/tag/v1.1.2) | | KubeRay | [v1.3.2](https://github.com/ray-project/kuberay/releases/tag/v1.3.2) | -| Kueue | [v0.11.4](https://github.com/kubernetes-sigs/kueue/releases/tag/v0.11.4) | +| Kueue | [v0.11.6](https://github.com/kubernetes-sigs/kueue/releases/tag/v0.11.6) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 6d0c50b4e..05ddd6940 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.15.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.16.0 namespace=opendatahub From ec0e56296d4edec048c78c47d0a5745c1c2bca17 Mon Sep 17 00:00:00 2001 From: Ajay Jaganathan <36824134+AjayJagan@users.noreply.github.com> Date: Wed, 2 Jul 2025 16:23:21 +0530 Subject: [PATCH 366/369] Update dockerfile to start supporting multi-arch builds --- Dockerfile | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index eb0b7ce82..c9f0a99ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,7 @@ # Build the manager binary +FROM registry.access.redhat.com/ubi9/go-toolset:1.23 AS builder -# BEGIN -- workaround lack of go-toolset for golang 1.23 -ARG GOLANG_IMAGE=docker.io/library/golang:1.23 -FROM ${GOLANG_IMAGE} AS golang - -FROM registry.access.redhat.com/ubi8/ubi@sha256:fd3bf22d0593e2ed26a1c74ce161c52295711a67de677b5938c87704237e49b0 AS builder -ARG GOLANG_VERSION=1.23.0 - -# Install system dependencies -RUN dnf upgrade -y && dnf install -y \ - gcc \ - make \ - openssl-devel \ - git \ - && dnf clean all && rm -rf /var/cache/yum - -# Install Go -ENV PATH=/usr/local/go/bin:$PATH - -COPY --from=golang /usr/local/go /usr/local/go -# End of Go versioning workaround +ARG TARGETOS TARGETARCH WORKDIR /workspace # Copy the Go Modules manifests @@ -32,11 +14,10 @@ RUN go mod download COPY main.go main.go COPY pkg/ pkg/ -# Build USER root -RUN CGO_ENABLED=1 GOOS=linux GOARCH=${GOARCH} make go-build-for-image +RUN CGO_ENABLED=1 GOOS=linux GOARCH=${TARGETARCH:-amd64} make go-build-for-image -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest WORKDIR / COPY --from=builder /workspace/manager . From b696c863ab5b5e5b84aa963974ee3e7a1559a9eb Mon Sep 17 00:00:00 2001 From: Ajay Jaganathan <36824134+AjayJagan@users.noreply.github.com> Date: Fri, 11 Jul 2025 00:53:45 +0530 Subject: [PATCH 367/369] Separate args Co-authored-by: Bryan Keane --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c9f0a99ad..0e3abde40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ # Build the manager binary FROM registry.access.redhat.com/ubi9/go-toolset:1.23 AS builder -ARG TARGETOS TARGETARCH +ARG TARGETOS +ARG TARGETARCH WORKDIR /workspace # Copy the Go Modules manifests From 7fc158cd532c31a61d7e7c07e8235ecb2abebf8b Mon Sep 17 00:00:00 2001 From: Pat O'Connor Date: Fri, 11 Jul 2025 11:33:08 +0100 Subject: [PATCH 368/369] task(RHOAIENG-27838): Update CodeFlare Compat matrix Signed-off-by: Pat O'Connor --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50228b780..baa6bfe2b 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.16.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.16.0) | -| CodeFlare-SDK | [v0.29.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.29.0) | +| CodeFlare-SDK | [v0.30.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.30.0) | | AppWrapper | [v1.1.2](https://github.com/project-codeflare/appwrapper/releases/tag/v1.1.2) | -| KubeRay | [v1.3.2](https://github.com/ray-project/kuberay/releases/tag/v1.3.2) | +| KubeRay | [v1.4.0](https://github.com/ray-project/kuberay/releases/tag/v1.4.0) | | Kueue | [v0.11.6](https://github.com/kubernetes-sigs/kueue/releases/tag/v0.11.6) | From dbf405ba69ee99a910dab2d499f1de4c3c207f39 Mon Sep 17 00:00:00 2001 From: lilylinh Date: Thu, 11 Sep 2025 14:41:38 +0100 Subject: [PATCH 369/369] task(RHOAIENG-33036):Update Codeflare compatibility matrix Signed-off-by: lilylinh --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index baa6bfe2b..0e9845fdf 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.16.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.16.0) | -| CodeFlare-SDK | [v0.30.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.30.0) | +| CodeFlare-SDK | [v0.31.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.31.0) | | AppWrapper | [v1.1.2](https://github.com/project-codeflare/appwrapper/releases/tag/v1.1.2) | | KubeRay | [v1.4.0](https://github.com/ray-project/kuberay/releases/tag/v1.4.0) | | Kueue | [v0.11.6](https://github.com/kubernetes-sigs/kueue/releases/tag/v0.11.6) |