rancher · HarrisonWAffel · Feb 21, 2025 · Jan 14, 2025
@@ -115,6 +115,26 @@ If `field.cattle.io/no-creator-rbac` annotation is set, `field.cattle.io/creator
  - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field exists in the new cluster object with a value different from the old one, the webhook will permit the update with a warning indicating that these changes will not take effect until version management is enabled for the cluster.
  - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field is missing, the webhook will permit the request to allow users to remove the unused fields via API or Terraform.
 
+
+##### Feature: Cluster Agent Scheduling Customization
+
+The `SchedulingCustomization` subfield of the `DeploymentCustomization` field defines the properties of a Pod Disruption Budget and Priority Class which will be automatically deployed by Rancher for the cattle-cluster-agent.
+
+The `schedulingCustomization.PriorityClass` field contains two attributes
+
++ `value`: This must be an integer value equal to or between negative 1 billion and 1 billion.
++ `preemption`: This must be a string value which indicates the desired preemption behavior, its value can be either `PreemptLowerPriority` or `Never`. Any other value must be rejected.
+
+The `schedulingCustomization.PodDisruptionBudget` field contains two attributes
+
++ `minAvailable`: This is a string value that indicates the minimum number of agent replicas that must be running at a given time.
++ `maxUnavailable`: This is a string value that indicates the maximum number of agent replicas that can be unavailable at a given time.
+
+Both `minAvailable` and `maxUnavailable` must be a string which represents a non-negative whole number, or a whole number percentage greater than or equal to `0%` and less than or equal to `100%`. Only one of the two fields can have a non-zero or empty value at a given time. These fields use the following regex when assessing if a given percentage value is valid:
+```regex
+^([0-9]|[1-9][0-9]|100)%$
+```
+
 ## ClusterProxyConfig
 
 ### Validation Checks
@@ -407,6 +427,12 @@ When settings are updated, the following additional checks take place:
   have a status condition `AgentTlsStrictCheck` set to `True`, unless the new setting has an overriding
   annotation `cattle.io/force=true`.
 
+
+- `cluster-agent-default-priority-class` must contain a valid JSON object which matches the format of a `v1.PriorityClassSpec` object. The Value field must be greater than or equal to negative 1 billion and less than or equal to 1 billion. The Preemption field must be a string value set to either `PreemptLowerPriority` or `Never`.
+
+
+- `cluster-agent-default-pod-disruption-budget` must contain a valid JSON object which matches the format of a `v1.PodDisruptionBudgetSpec` object. The `minAvailable` and `maxUnavailable` fields must have a string value that is either a non-negative whole number, or a non-negative whole number percentage value less than or equal to `100%`.
+
 ## Token
 
 ### Validation Checks
@@ -499,6 +525,25 @@ A `Toleration` is matched to a regex which is provided by upstream [apimachinery
 
 For the `Affinity` based rules, the `podAffinity`/`podAntiAffinity` are validated via label selectors via [this apimachinery function](https://github.com/kubernetes/apimachinery/blob/02a41040d88da08de6765573ae2b1a51f424e1ca/pkg/apis/meta/v1/validation/validation.go#L56) whereas the `nodeAffinity` `nodeSelectorTerms` are validated via the same `Toleration` function.
 
+#### cluster.spec.clusterAgentDeploymentCustomization.schedulingCustomization
+
+The `SchedulingCustomization` subfield of the `DeploymentCustomization` field defines the properties of a Pod Disruption Budget and Priority Class which will be automatically deployed by Rancher for the cattle-cluster-agent.
+
+The `schedulingCustomization.PriorityClass` field contains two attributes
+
++ `value`: This must be an integer value equal to or between negative 1 billion and 1 billion.
++ `preemption`: This must be a string value which indicates the desired preemption behavior, its value can be either `PreemptLowerPriority` or `Never`. Any other value must be rejected.
+
+The `schedulingCustomization.PodDisruptionBudget` field contains two attributes
+
++ `minAvailable`: This is a string value that indicates the minimum number of agent replicas that must be running at a given time.
++ `maxUnavailable`: This is a string value that indicates the maximum number of agent replicas that can be unavailable at a given time.
+
+Both `minAvailable` and `maxUnavailable` must be a string which represents a non-negative whole number, or a whole number percentage greater than or equal to `0%` and less than or equal to `100%`. Only one of the two fields can have a non-zero or empty value at a given time. These fields use the following regex when assessing if a given percentage value is valid:
+```regex
+^([0-9]|[1-9][0-9]|100)%$
+```
+
 ### Mutation Checks
 
 #### On Create

@@ -43,7 +43,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/rancher/dynamiclistener v0.6.1
 	github.com/rancher/lasso v0.2.1
-	github.com/rancher/rancher/pkg/apis v0.0.0-20250213173112-3d729db8a848
+	github.com/rancher/rancher/pkg/apis v0.0.0-20250220153925-3abb578f42fe
 	github.com/rancher/rke v1.8.0-rc.1
 	github.com/rancher/wrangler/v3 v3.2.0-rc.3
 	github.com/robfig/cron v1.2.0

@@ -157,8 +157,8 @@ github.com/rancher/lasso v0.2.1 h1:SZTqMVQn8cAOqvwGBd1/EYOIJ/MGN+UfJrOWvHd4jHU=
 github.com/rancher/lasso v0.2.1/go.mod h1:KSV3jBXfdXqdCuMm2uC8kKB9q/wuDYb3h0eHZoRjShM=
 github.com/rancher/norman v0.5.1 h1:jbp49IcX2Hn+N2QA3MHdIXeUG0VgCSIjJs4xnqG+j90=
 github.com/rancher/norman v0.5.1/go.mod h1:qX/OG/4wY27xSAcSdRilUBxBumV6Ey2CWpAeaKnBQDs=
-github.com/rancher/rancher/pkg/apis v0.0.0-20250213173112-3d729db8a848 h1:0mNj9JwUmMtn5lGfPoE1AiCXMRuCRwMbhnmFVqktswM=
-github.com/rancher/rancher/pkg/apis v0.0.0-20250213173112-3d729db8a848/go.mod h1:FfFL3Pw7ds9aaaA0JvZ3m8kJXTg6DNknxLBC0vODpuI=
+github.com/rancher/rancher/pkg/apis v0.0.0-20250220153925-3abb578f42fe h1:DNGD4RCs1k5PxAHUc1zA9FiEfowcejQQcGAItwUIDh4=
+github.com/rancher/rancher/pkg/apis v0.0.0-20250220153925-3abb578f42fe/go.mod h1:0JtLfvgj4YiwddyHEvhF3yEK9k5c22CWs55DppqdP5o=
 github.com/rancher/rke v1.7.2 h1:+2fcl0gCjRHzf1ev9C9ptQ1pjYbDngC1Qv8V/0ki/dk=
 github.com/rancher/rke v1.7.2/go.mod h1:+x++Mvl0A3jIzNLiu8nkraqZXiHg6VPWv0Xl4iQCg+A=
 github.com/rancher/wrangler/v3 v3.2.0-rc.3 h1:MySHWLxLLrGrM2sq5YYp7Ol1kQqYt9lvIzjGR50UZ+c=

@@ -1,6 +1,8 @@
 package common
 
 import (
+	"regexp"
+
 	"github.com/rancher/webhook/pkg/admission"
 	"github.com/rancher/webhook/pkg/auth"
 	"github.com/sirupsen/logrus"
@@ -19,8 +21,15 @@ const (
 	CreatorPrincipalNameAnn = "field.cattle.io/creator-principal-name"
 	// NoCreatorRBACAnn is an annotation key to indicate that a cluster doesn't need
 	NoCreatorRBACAnn = "field.cattle.io/no-creator-rbac"
+	// SchedulingCustomizationFeatureName is the feature name for enabling customization of PDBs and PCs for the
+	// cattle-cluster-agent
+	SchedulingCustomizationFeatureName = "cluster-agent-scheduling-customization"
 )
 
+// PdbPercentageRegex ensures that a given string is a properly formatted percentage value
+// between 0% and 100% so that it can be used in a Pod Disruption Budget
+var PdbPercentageRegex = regexp.MustCompile("^([0-9]|[1-9][0-9]|100)%$")
+
 // ConvertAuthnExtras converts authnv1 type extras to authzv1 extras. Technically these are both
 // type alias to string, so the conversion is straightforward
 func ConvertAuthnExtras(extra map[string]authnv1.ExtraValue) map[string]authzv1.ExtraValue {

@@ -23,3 +23,23 @@ If `field.cattle.io/no-creator-rbac` annotation is set, `field.cattle.io/creator
  - If the cluster represents other types of clusters and the annotation is present, the webhook will permit the request with a warning that the annotation is intended for imported RKE2/k3s clusters and will not take effect on this cluster.
  - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field exists in the new cluster object with a value different from the old one, the webhook will permit the update with a warning indicating that these changes will not take effect until version management is enabled for the cluster.
  - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field is missing, the webhook will permit the request to allow users to remove the unused fields via API or Terraform.
+
+
+#### Feature: Cluster Agent Scheduling Customization
+
+The `SchedulingCustomization` subfield of the `DeploymentCustomization` field defines the properties of a Pod Disruption Budget and Priority Class which will be automatically deployed by Rancher for the cattle-cluster-agent.
+
+The `schedulingCustomization.PriorityClass` field contains two attributes
+
++ `value`: This must be an integer value equal to or between negative 1 billion and 1 billion.
++ `preemption`: This must be a string value which indicates the desired preemption behavior, its value can be either `PreemptLowerPriority` or `Never`. Any other value must be rejected.
+
+The `schedulingCustomization.PodDisruptionBudget` field contains two attributes
+
++ `minAvailable`: This is a string value that indicates the minimum number of agent replicas that must be running at a given time.
++ `maxUnavailable`: This is a string value that indicates the maximum number of agent replicas that can be unavailable at a given time.
+
+Both `minAvailable` and `maxUnavailable` must be a string which represents a non-negative whole number, or a whole number percentage greater than or equal to `0%` and less than or equal to `100%`. Only one of the two fields can have a non-zero or empty value at a given time. These fields use the following regex when assessing if a given percentage value is valid:
+```regex
+^([0-9]|[1-9][0-9]|100)%$
+```
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"strconv"
 
 	"github.com/blang/semver"
 	apisv3 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
@@ -17,6 +18,7 @@ import (
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	v1 "k8s.io/api/authorization/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,13 +38,15 @@ func NewValidator(
 	sar authorizationv1.SubjectAccessReviewInterface,
 	cache v3.PodSecurityAdmissionConfigurationTemplateCache,
 	userCache v3.UserCache,
+	featureCache v3.FeatureCache,
 	settingCache v3.SettingCache,
 ) *Validator {
 	return &Validator{
 		admitter: admitter{
 			sar:          sar,
 			psact:        cache,
-			userCache:    userCache,    // userCache is nil for downstream clusters.
+			userCache:    userCache, // userCache is nil for downstream clusters.
+			featureCache: featureCache,
 			settingCache: settingCache, // settingCache is nil for downstream clusters
 		},
 	}
@@ -79,6 +83,7 @@ type admitter struct {
 	sar          authorizationv1.SubjectAccessReviewInterface
 	psact        v3.PodSecurityAdmissionConfigurationTemplateCache
 	userCache    v3.UserCache
+	featureCache v3.FeatureCache
 	settingCache v3.SettingCache
 }
 
@@ -117,6 +122,14 @@ func (a *admitter) Admit(request *admission.Request) (*admissionv1.AdmissionResp
 		}
 	}
 
+	if response, err = a.validatePodDisruptionBudget(oldCluster, newCluster, request.Operation); err != nil || !response.Allowed {
+		return response, err
+	}
+
+	if response, err = a.validatePriorityClass(oldCluster, newCluster, request.Operation); err != nil || !response.Allowed {
+		return response, err
+	}
+
 	response, err = a.validatePSACT(oldCluster, newCluster, request.Operation)
 	if err != nil {
 		return nil, fmt.Errorf("failed to validate PodSecurityAdmissionConfigurationTemplate(PSACT): %w", err)
@@ -265,6 +278,155 @@ func (a *admitter) validatePSACT(oldCluster, newCluster *apisv3.Cluster, op admi
 	return admission.ResponseAllowed(), nil
 }
 
+// validatePriorityClass validates that the Priority Class defined in the cluster SchedulingCustomization field is properly
+// configured. The cluster-agent-scheduling-customization feature must be enabled to configure a Priority Class, however an existing
+// Priority Class may be deleted even if the feature is disabled.
+func (a *admitter) validatePriorityClass(oldCluster, newCluster *apisv3.Cluster, op admissionv1.Operation) (*admissionv1.AdmissionResponse, error) {
+	if op != admissionv1.Create && op != admissionv1.Update {
+		return admission.ResponseAllowed(), nil
+	}
+
+	newClusterScheduling := getSchedulingCustomization(newCluster)
+	oldClusterScheduling := getSchedulingCustomization(oldCluster)
+
+	var newPC, oldPC *apisv3.PriorityClassSpec
+	if newClusterScheduling != nil {
+		newPC = newClusterScheduling.PriorityClass
+	}
+
+	if oldClusterScheduling != nil {
+		oldPC = oldClusterScheduling.PriorityClass
+	}
+
+	if newPC == nil {
+		return admission.ResponseAllowed(), nil
+	}
+
+	featuredEnabled, err := a.featureCache.Get(common.SchedulingCustomizationFeatureName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine status of '%s' feature", common.SchedulingCustomizationFeatureName)
+	}
+
+	enabled := featuredEnabled.Status.Default
+	if featuredEnabled.Spec.Value != nil {
+		enabled = *featuredEnabled.Spec.Value
+	}
+
+	// if the feature is disabled then we should not permit any changes between the old and new clusters other than deletion
+	if !enabled && oldPC != nil {
+		if reflect.DeepEqual(*oldPC, *newPC) {
+			return admission.ResponseAllowed(), nil
+		}
+
+		return admission.ResponseBadRequest(fmt.Sprintf("'%s' feature is disabled, will only permit removal of Scheduling Customization fields until reenabled", common.SchedulingCustomizationFeatureName)), nil
+	}
+
+	if !enabled && oldPC == nil {
+		return admission.ResponseBadRequest(fmt.Sprintf("the '%s' feature must be enabled in order to configure a Priority Class or Pod Disruption Budget", common.SchedulingCustomizationFeatureName)), nil
+	}
+
+	if newPC.Preemption != nil && *newPC.Preemption != corev1.PreemptNever && *newPC.Preemption != corev1.PreemptLowerPriority && *newPC.Preemption != "" {
+		return admission.ResponseBadRequest("Priority Class Preemption value must be 'Never', 'PreemptLowerPriority', or empty"), nil
+	}
+
+	if newPC.Value > 1000000000 {
+		return admission.ResponseBadRequest("Priority Class value cannot be greater than 1 billion"), nil
+	}
+
+	if newPC.Value < -1000000000 {
+		return admission.ResponseBadRequest("Priority Class value cannot be less than negative 1 billion"), nil
+	}
+
+	return admission.ResponseAllowed(), nil
+}
+
+// validatePodDisruptionBudget validates that the Pod Disruption Budget defined in the cluster SchedulingCustomization field is properly
+// configured. The cluster-agent-scheduling-customization feature must be enabled to configure a Pod Disruption Budget, however an existing
+// Pod Disruption Budget may be deleted even if the feature is disabled.
+func (a *admitter) validatePodDisruptionBudget(oldCluster, newCluster *apisv3.Cluster, op admissionv1.Operation) (*admissionv1.AdmissionResponse, error) {
+	if op != admissionv1.Create && op != admissionv1.Update {
+		return admission.ResponseAllowed(), nil
+	}
+	newClusterScheduling := getSchedulingCustomization(newCluster)
+	oldClusterScheduling := getSchedulingCustomization(oldCluster)
+
+	var newPDB, oldPDB *apisv3.PodDisruptionBudgetSpec
+	if newClusterScheduling != nil {
+		newPDB = newClusterScheduling.PodDisruptionBudget
+	}
+
+	if oldClusterScheduling != nil {
+		oldPDB = oldClusterScheduling.PodDisruptionBudget
+	}
+
+	if newPDB == nil {
+		return admission.ResponseAllowed(), nil
+	}
+
+	featuredEnabled, err := a.featureCache.Get(common.SchedulingCustomizationFeatureName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine status of '%s' feature", common.SchedulingCustomizationFeatureName)
+	}
+
+	enabled := featuredEnabled.Status.Default
+	if featuredEnabled.Spec.Value != nil {
+		enabled = *featuredEnabled.Spec.Value
+	}
+
+	// if the feature is disabled then we should not permit any changes between the old and new clusters other than deletion
+	if !enabled && oldPDB != nil {
+		if reflect.DeepEqual(*oldPDB, *newPDB) {
+			return admission.ResponseAllowed(), nil
+		}
+
+		return admission.ResponseBadRequest(fmt.Sprintf("'%s' feature is disabled, will only permit removal of Scheduling Customization fields until reenabled", common.SchedulingCustomizationFeatureName)), nil
+	}
+
+	if !enabled && oldPDB == nil {
+		return admission.ResponseBadRequest(fmt.Sprintf("the '%s' feature must be enabled in order to configure a Priority Class or Pod Disruption Budget", common.SchedulingCustomizationFeatureName)), nil
+	}
+
+	minAvailStr := newPDB.MinAvailable
+	maxUnavailStr := newPDB.MaxUnavailable
+
+	if (minAvailStr == "" && maxUnavailStr == "") ||
+		(minAvailStr == "0" && maxUnavailStr == "0") ||
+		(minAvailStr != "" && minAvailStr != "0") && (maxUnavailStr != "" && maxUnavailStr != "0") {
+		return admission.ResponseBadRequest("both minAvailable and maxUnavailable cannot be set to a non zero value, at least one must be omitted or set to zero"), nil
+	}
+
+	minAvailIsString := false
+	maxUnavailIsString := false
+
+	minAvailInt, err := strconv.Atoi(minAvailStr)
+	if err != nil {
+		minAvailIsString = minAvailStr != ""
+	}
+
+	maxUnavailInt, err := strconv.Atoi(maxUnavailStr)
+	if err != nil {
+		maxUnavailIsString = maxUnavailStr != ""
+	}
+
+	if !minAvailIsString && minAvailInt < 0 {
+		return admission.ResponseBadRequest("minAvailable cannot be set to a negative integer"), nil
+	}
+
+	if !maxUnavailIsString && maxUnavailInt < 0 {
+		return admission.ResponseBadRequest("maxUnavailable cannot be set to a negative integer"), nil
+	}
+
+	if minAvailIsString && !common.PdbPercentageRegex.Match([]byte(minAvailStr)) {
+		return admission.ResponseBadRequest(fmt.Sprintf("minAvailable must be a non-negative whole integer or a percentage value between 0 and 100, regex used is '%s'", common.PdbPercentageRegex.String())), nil
+	}
+
+	if maxUnavailIsString && maxUnavailStr != "" && !common.PdbPercentageRegex.Match([]byte(maxUnavailStr)) {
+		return admission.ResponseBadRequest(fmt.Sprintf("minAvailable must be a non-negative whole integer or a percentage value between 0 and 100, regex used is '%s'", common.PdbPercentageRegex.String())), nil
+	}
+
+	return admission.ResponseAllowed(), nil
+}
+
 // checkPSAConfigOnCluster validates the cluster spec when DefaultPodSecurityAdmissionConfigurationTemplateName is set.
 func (a *admitter) checkPSAConfigOnCluster(cluster *apisv3.Cluster) (*admissionv1.AdmissionResponse, error) {
 	// validate that extra_args.admission-control-config-file is not set at the same time
@@ -310,6 +472,22 @@ func (a *admitter) checkPSAConfigOnCluster(cluster *apisv3.Cluster) (*admissionv
 	return admission.ResponseAllowed(), nil
 }
 
+func getSchedulingCustomization(cluster *apisv3.Cluster) *apisv3.AgentSchedulingCustomization {
+	if cluster == nil {
+		return nil
+	}
+
+	if cluster.Spec.ClusterAgentDeploymentCustomization == nil {
+		return nil
+	}
+
+	if cluster.Spec.ClusterAgentDeploymentCustomization.SchedulingCustomization == nil {
+		return nil
+	}
+
+	return cluster.Spec.ClusterAgentDeploymentCustomization.SchedulingCustomization
+}
+
 // validateVersionManagementFeature validates the annotation for the version management feature is set with valid value on the imported RKE2/K3s cluster,
 // additionally, it permits but include a warning to the response if either of the following is true:
 //   - the annotation is found on a cluster rather than imported RKE2/K3s cluster;