USHIFT-6130 Support prefer-align-cpus-by-uncorecache featureGate

docs/user/howto_config.md

@@ -14,6 +14,11 @@ apiServer:
         maxFileSize: 0
         maxFiles: 0
         profile: ""
+    featureGates:
+        customNoUpgrade:
+            disabled: []
+            enabled: []
+        featureSet: ""
     namedCertificates:
         - certPath: ""
           keyPath: ""
@@ -155,6 +160,11 @@ apiServer:
         maxFileSize: 200
         maxFiles: 10
         profile: Default
+    featureGates:
+        customNoUpgrade:
+            disabled: []
+            enabled: []
+        featureSet: ""
     namedCertificates:
         - certPath: ""
           keyPath: ""

packaging/microshift/config.yaml

@@ -30,6 +30,11 @@ apiServer:
         # to serve from the API server. Allowed values: VersionTLS12, VersionTLS13.
         # Defaults to VersionTLS12.
         minVersion: VersionTLS12
+    featureGates:
+        featureSet: ""
+        customNoUpgrade:
+            enabled: []
+            disabled: []
 debugging:
     # Valid values are: "Normal", "Debug", "Trace", "TraceAll".
     # Defaults to "Normal".

pkg/config/apiserver.go

@@ -2,9 +2,12 @@ package config
 
 import (
 	"fmt"
+	"reflect"
 	"slices"
+	"strings"
 
 	configv1 "github.com/openshift/api/config/v1"
+	featuresUtils "github.com/openshift/api/features"
 	"github.com/openshift/library-go/pkg/crypto"
 )
 
@@ -27,6 +30,8 @@ type ApiServer struct {
 
 	TLS TLSConfig `json:"tls"`
 
+	FeatureGates FeatureGates `json:"featureGates"`
+
 	// The URL and Port of the API server cannot be changed by the user.
 	URL  string `json:"-"`
 	Port int    `json:"-"`
@@ -127,3 +132,76 @@ func (t *TLSConfig) Validate() error {
 func getIANACipherSuites(suites []string) []string {
 	return crypto.OpenSSLToIANACipherSuites(suites)
 }
+
+const (
+	FeatureSetCustomNoUpgrade      = "CustomNoUpgrade"
+	FeatureSetTechPreviewNoUpgrade = "TechPreviewNoUpgrade"
+	FeatureSetDevPreviewNoUpgrade  = "DevPreviewNoUpgrade"
+)
+
+type CustomNoUpgrade struct {
+	Enabled  []string `json:"enabled"`
+	Disabled []string `json:"disabled"`
+}
+
+type FeatureGates struct {
+	FeatureSet      string          `json:"featureSet"`
+	CustomNoUpgrade CustomNoUpgrade `json:"customNoUpgrade"`
+}
+
+func (fg FeatureGates) ConvertToCLIFlags() ([]string, error) {
+	ret := []string{}
+
+	switch fg.FeatureSet {
+	case FeatureSetCustomNoUpgrade:
+		for _, feature := range fg.CustomNoUpgrade.Enabled {
+			ret = append(ret, fmt.Sprintf("%s=true", feature))
+		}
+		for _, feature := range fg.CustomNoUpgrade.Disabled {
+			ret = append(ret, fmt.Sprintf("%s=false", feature))
+		}
+	case FeatureSetDevPreviewNoUpgrade, FeatureSetTechPreviewNoUpgrade:
+		fgEnabledDisabled, err := featuresUtils.FeatureSets(featuresUtils.SelfManaged, configv1.FeatureSet(fg.FeatureSet))
+		if err != nil {
+			return nil, fmt.Errorf("failed to get feature set gates: %w", err)
+		}
+		for _, f := range fgEnabledDisabled.Enabled {
+			ret = append(ret, fmt.Sprintf("%s=true", f.FeatureGateAttributes.Name))
+		}
+		for _, f := range fgEnabledDisabled.Disabled {
+			ret = append(ret, fmt.Sprintf("%s=false", f.FeatureGateAttributes.Name))
+		}
+	}
+	return ret, nil
+}
+
+// Implement the GoStringer interface for better %#v printing
+func (fg FeatureGates) GoString() string {
+	return fmt.Sprintf("FeatureGates{FeatureSet: %q, CustomNoUpgrade: %#v}", fg.FeatureSet, fg.CustomNoUpgrade)
+}
+
+func (fg *FeatureGates) validateFeatureGates() error {
+	// FG is unset
+	if fg == nil || reflect.DeepEqual(*fg, FeatureGates{}) {
+		return nil
+	}
+	// Must use a recognized feature set, or else empty
+	if fg.FeatureSet != "" && fg.FeatureSet != FeatureSetCustomNoUpgrade && fg.FeatureSet != FeatureSetTechPreviewNoUpgrade && fg.FeatureSet != FeatureSetDevPreviewNoUpgrade {
+		return fmt.Errorf("invalid feature set: %s", fg.FeatureSet)
+	}
+	// Must set FeatureSet to CustomNoUpgrade to use custom feature gates
+	if fg.FeatureSet != FeatureSetCustomNoUpgrade && (len(fg.CustomNoUpgrade.Enabled) > 0 || len(fg.CustomNoUpgrade.Disabled) > 0) {
+		return fmt.Errorf("CustomNoUpgrade must be empty when FeatureSet is empty")
+	}
+	// Must not have any feature gates that are enabled and disabled at the same time
+	var illegalFeatures []string
+	for _, enabledFeature := range fg.CustomNoUpgrade.Enabled {
+		if slices.Contains(fg.CustomNoUpgrade.Disabled, enabledFeature) {
+			illegalFeatures = append(illegalFeatures, enabledFeature)
+		}
+	}
+	if len(illegalFeatures) > 0 {
+		return fmt.Errorf("featuregates cannot be enabled and disabled at the same time: %s", strings.Join(illegalFeatures, ", "))
+	}
+	return nil
+}

pkg/config/config.go

@@ -56,7 +56,6 @@ type Config struct {
 	Ingress   IngressConfig `json:"ingress"`
 	Storage   Storage       `json:"storage"`
 	Telemetry Telemetry     `json:"telemetry"`
-
 	// Settings specified in this section are transferred as-is into the Kubelet config.
 	// +kubebuilder:validation:Schemaless
 	Kubelet map[string]any `json:"kubelet"`
@@ -418,6 +417,15 @@ func (c *Config) incorporateUserSettings(u *Config) {
 	if u.Ingress.AccessLogging.HttpCaptureCookies != nil {
 		c.Ingress.AccessLogging.HttpCaptureCookies = u.Ingress.AccessLogging.HttpCaptureCookies
 	}
+	if u.ApiServer.FeatureGates.FeatureSet != "" {
+		c.ApiServer.FeatureGates.FeatureSet = u.ApiServer.FeatureGates.FeatureSet
+	}
+	if len(u.ApiServer.FeatureGates.CustomNoUpgrade.Enabled) > 0 {
+		c.ApiServer.FeatureGates.CustomNoUpgrade.Enabled = u.ApiServer.FeatureGates.CustomNoUpgrade.Enabled
+	}
+	if len(u.ApiServer.FeatureGates.CustomNoUpgrade.Disabled) > 0 {
+		c.ApiServer.FeatureGates.CustomNoUpgrade.Disabled = u.ApiServer.FeatureGates.CustomNoUpgrade.Disabled
+	}
 }
 
 // updateComputedValues examins the existing settings and converts any
@@ -647,6 +655,10 @@ func (c *Config) validate() error {
 		return fmt.Errorf("error validating Generic Device Plugin configuration: %v", err)
 	}
 
+	if err := c.ApiServer.FeatureGates.validateFeatureGates(); err != nil {
+		return fmt.Errorf("error validating feature gates: %v", err)
+	}
+
 	return nil
 }
 

pkg/config/config_test.go

@@ -800,6 +800,56 @@ func TestValidate(t *testing.T) {
 			}(),
 			expectErr: true,
 		},
+		{
+			name: "feature-gates-unset",
+			config: func() *Config {
+				c := mkDefaultConfig()
+				c.ApiServer.FeatureGates = FeatureGates{}
+				return c
+			}(),
+			expectErr: false,
+		},
+		{
+			name: "feature-gates-invalid-feature-set",
+			config: func() *Config {
+				c := mkDefaultConfig()
+				c.ApiServer.FeatureGates.FeatureSet = "invalid"
+				return c
+			}(),
+			expectErr: true,
+		},
+		{
+			name: "feature-gates-custom-no-upgrade-with-feature-set",
+			config: func() *Config {
+				c := mkDefaultConfig()
+				c.ApiServer.FeatureGates.FeatureSet = "CustomNoUpgrade"
+				c.ApiServer.FeatureGates.CustomNoUpgrade.Enabled = []string{"feature1"}
+				c.ApiServer.FeatureGates.CustomNoUpgrade.Disabled = []string{"feature2"}
+				return c
+			}(),
+		},
+		{
+			name: "feature-gates-custom-no-upgrade-with-feature-set-empty",
+			config: func() *Config {
+				c := mkDefaultConfig()
+				c.ApiServer.FeatureGates.FeatureSet = ""
+				c.ApiServer.FeatureGates.CustomNoUpgrade.Enabled = []string{"feature1"}
+				c.ApiServer.FeatureGates.CustomNoUpgrade.Disabled = []string{"feature2"}
+				return c
+			}(),
+			expectErr: true,
+		},
+		{
+			name: "feature-gates-custom-no-upgrade-enabled-and-disabled-have-same-feature-gate",
+			config: func() *Config {
+				c := mkDefaultConfig()
+				c.ApiServer.FeatureGates.FeatureSet = "CustomNoUpgrade"
+				c.ApiServer.FeatureGates.CustomNoUpgrade.Enabled = []string{"feature1"}
+				c.ApiServer.FeatureGates.CustomNoUpgrade.Disabled = []string{"feature1"}
+				return c
+			}(),
+			expectErr: true,
+		},
 	}
 	for _, tt := range ttests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/controllers/kube-apiserver.go

@@ -63,7 +63,6 @@ var (
 		embedded.MustAsset("controllers/kube-apiserver/config-overrides.yaml"),
 	}
 )
-
 var fixedTLSProfile *configv1.TLSProfileSpec
 
 func init() {
@@ -169,6 +168,15 @@ func (s *KubeAPIServer) configure(ctx context.Context, cfg *config.Config) error
 		return fmt.Errorf("failed to discover etcd servers: %w", err)
 	}
 
+	featureGateArgs, err := cfg.ApiServer.FeatureGates.ConvertToCLIFlags()
+	if err != nil {
+		return fmt.Errorf("failed to convert feature gates to CLI flags: %w", err)
+	}
+
+	// Build the final feature gates list
+	hardcodedFeatureGates := []string{"UserNamespacesSupport=true", "UserNamespacesPodSecurityStandards=true"}
+	featureGateArgs = append(featureGateArgs, hardcodedFeatureGates...)
+
 	overrides := &kubecontrolplanev1.KubeAPIServerConfig{
 		APIServerArguments: map[string]kubecontrolplanev1.Arguments{
 			"advertise-address":             {s.advertiseAddress},
@@ -221,7 +229,7 @@ func (s *KubeAPIServer) configure(ctx context.Context, cfg *config.Config) error
 			"enable-admission-plugins":              {},
 			"send-retry-after-while-not-ready-once": {"true"},
 			"shutdown-delay-duration":               {"5s"},
-			"feature-gates":                         {"UserNamespacesSupport=true", "UserNamespacesPodSecurityStandards=true"},
+			"feature-gates":                         featureGateArgs,
 		},
 		GenericAPIServerConfig: configv1.GenericAPIServerConfig{
 			AdmissionConfig: configv1.AdmissionConfig{

sequenceDiagram.mmd

@@ -0,0 +1,73 @@
+sequenceDiagram
+    participant ConfigFile as config.yaml
+    participant ActiveConfig as config.ActiveConfig()
+    participant FillDefaults as config.fillDefaults()
+    participant IncorporateSettings as config.incorporateUserSettings()
+    participant UpdateComputed as config.updateComputedValues()
+    participant KASRun as KubeAPIServer.Run()
+    participant KASConfigure as KubeAPIServer.configure()
+    participant Merge as resourcemerge.MergePrunedProcessConfig()
+    participant TempFile as Create Temp File
+    participant NewCmd as kubeapiserver.NewAPIServerCommand()
+    participant Execute as cmd.ExecuteContext()
+    participant GetOpenshiftConfig as enablement.GetOpenshiftConfig()
+    participant ConfigToFlags as openshiftkubeapiserver.ConfigToFlags()
+    participant ParseFlags as cmd.ParseFlags()
+    participant KASMain as Run() [kube-apiserver]
+
+    Note over ConfigFile: User configuration in<br/>/etc/microshift/config.yaml<br/><br/>apiServer:<br/>  advertiseAddress: "10.43.0.2"<br/>  port: 6443<br/>  auditLog:<br/>    profile: "Default"<br/>  tls:<br/>    minVersion: "VersionTLS12"<br/>    cipherSuites: [...]<br/>  featureGates:<br/>    featureSet: "CustomNoUpgrade"<br/>    customNoUpgrade:<br/>      enabled: ["UserNamespacesSupport"]
+
+    ConfigFile->>+ActiveConfig: Read config.yaml<br/>[files.go:120]
+
+    ActiveConfig->>+FillDefaults: Set default values<br/>[config.go:92]
+    Note right of FillDefaults: Sets default ApiServer values:<br/>- Port: 6443<br/>- URL: "https://localhost:6443"<br/>- AuditLog defaults<br/>- SubjectAltNames
+    FillDefaults-->>-ActiveConfig: Config with defaults
+
+    ActiveConfig->>+IncorporateSettings: Merge user settings<br/>[config.go:195]
+    Note right of IncorporateSettings: Merges user-provided values:<br/>- ApiServer.AdvertiseAddress<br/>- ApiServer.AuditLog.*<br/>- ApiServer.TLS.*<br/>- ApiServer.NamedCertificates<br/>- ApiServer.FeatureGates.*
+    IncorporateSettings-->>-ActiveConfig: Config with user settings
+
+    ActiveConfig->>+UpdateComputed: Compute derived values<br/>[config.go:434]
+    Note right of UpdateComputed: Computes:<br/>- AdvertiseAddresses (from ServiceNetwork)<br/>- TLS.UpdateValues() - cipher suites<br/>- SkipInterface flag
+    UpdateComputed-->>-ActiveConfig: Final Config
+
+    ActiveConfig-->>-ConfigFile: *config.Config
+
+    Note over KASRun: MicroShift creates and starts<br/>kube-apiserver controller
+
+    KASRun->>+KASConfigure: configure(ctx, cfg)<br/>[kube-apiserver.go:97]
+
+    Note right of KASConfigure: Uses cfg.ApiServer fields to build<br/>KubeAPIServerConfig overrides:<br/><br/>- advertise-address<br/>- audit-policy-file<br/>- audit-log-*<br/>- tls-cert-file<br/>- tls-private-key-file<br/>- tls-min-version<br/>- tls-cipher-suites<br/>- feature-gates<br/>- service-cluster-ip-range<br/>- disable-admission-plugins<br/>- enable-admission-plugins
+
+    KASConfigure->>+Merge: Merge config layers<br/>[kube-apiserver.go:267]
+    Note right of Merge: Merges 3 layers:<br/>1. defaultconfig.yaml (embedded)<br/>2. config-overrides.yaml (embedded)<br/>3. Runtime overrides from cfg.ApiServer<br/><br/>Creates KubeAPIServerConfig struct<br/>with all settings merged
+    Merge-->>-KASConfigure: kasConfigBytes (marshaled YAML)
+
+    KASConfigure-->>-KASRun: Configuration complete
+
+    KASRun->>TempFile: Create temp config file<br/>[kube-apiserver.go:363]
+    Note right of TempFile: Write kasConfigBytes to:<br/>/tmp/kube-apiserver-config-*.yaml
+
+    KASRun->>+NewCmd: NewAPIServerCommand()<br/>[kube-apiserver.go:391]
+    Note right of NewCmd: Initialize kube-apiserver command<br/>from kubernetes/cmd/kube-apiserver
+    NewCmd-->>-KASRun: cmd *cobra.Command
+
+    KASRun->>+Execute: cmd.ExecuteContext(ctx)<br/>[kube-apiserver.go:404]
+    Note right of Execute: Args:<br/>--openshift-config /tmp/kube-apiserver-config-*.yaml<br/>-v [verbosity]
+
+    Execute->>+GetOpenshiftConfig: Read config file<br/>[intialization.go:27]
+    Note right of GetOpenshiftConfig: Reads temp config file and<br/>deserializes into<br/>KubeAPIServerConfig struct<br/><br/>Resolves file paths<br/>Sets recommended defaults
+    GetOpenshiftConfig-->>-Execute: *KubeAPIServerConfig
+
+    Execute->>+ConfigToFlags: Convert to CLI flags<br/>[flags.go:18]
+    Note right of ConfigToFlags: Converts KubeAPIServerConfig.APIServerArguments<br/>to command-line flags:<br/><br/>Map[string][]string -> []string<br/><br/>Examples:<br/>"advertise-address": ["10.43.0.2"]<br/>  -> --advertise-address=10.43.0.2<br/><br/>"feature-gates": ["UserNamespacesSupport=true"]<br/>  -> --feature-gates=UserNamespacesSupport=true<br/><br/>Also converts:<br/>- ServingInfo -> tls-* flags<br/>- AuditConfig -> audit-* flags<br/>- AdmissionConfig -> admission-* flags
+    ConfigToFlags-->>-Execute: []string (CLI flags)
+
+    Execute->>+ParseFlags: Parse merged flags<br/>[server.go:122]
+    Note right of ParseFlags: Re-parses command flags with<br/>values from OpenShift config<br/><br/>This updates the ServerRunOptions<br/>with all the ApiServer settings
+    ParseFlags-->>-Execute: Updated options
+
+    Execute->>+KASMain: Run(ctx, completedOptions)<br/>[server.go:153]
+    Note right of KASMain: Kube-apiserver starts with<br/>all configuration applied:<br/><br/>- TLS settings<br/>- Audit logging<br/>- Feature gates<br/>- Admission plugins<br/>- Serving configuration
+
+    Note over KASMain: Kube-apiserver running with<br/>all config.ApiServer values<br/>applied via command-line flags