OCPBUGS-73945: Allow custom feature gates to be exempted from upgrade blocking

cmd/generate-config/config/config-openapi-spec.json

@@ -64,10 +64,12 @@
           "type": "object",
           "required": [
             "customNoUpgrade",
-            "featureSet"
+            "featureSet",
+            "specialHandlingSupportExceptionRequired"
           ],
           "properties": {
             "customNoUpgrade": {
+              "description": "CustomNoUpgrade is used to enable/disable feature gates that block x- and y-stream upgrades.",
               "type": "object",
               "required": [
                 "disabled",
@@ -90,6 +92,28 @@
             },
             "featureSet": {
               "type": "string"
+            },
+            "specialHandlingSupportExceptionRequired": {
+              "description": "SpecialHandlingSupportExceptionRequired is used to enable/disable feature gates that are exempt from blocking x- and y-stream upgrades.\nFeatures listed here do not need to be duplicated in CustomNoUpgrade.",
+              "type": "object",
+              "required": [
+                "disabled",
+                "enabled"
+              ],
+              "properties": {
+                "disabled": {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "enabled": {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              }
             }
           }
         },

docs/user/howto_config.md

@@ -19,6 +19,9 @@ apiServer:
             disabled: []
             enabled: []
         featureSet: ""
+        specialHandlingSupportExceptionRequired:
+            disabled: []
+            enabled: []
     namedCertificates:
         - certPath: ""
           keyPath: ""
@@ -168,6 +171,9 @@ apiServer:
             disabled: []
             enabled: []
         featureSet: ""
+        specialHandlingSupportExceptionRequired:
+            disabled: []
+            enabled: []
     namedCertificates:
         - certPath: ""
           keyPath: ""

packaging/microshift/config.yaml

@@ -15,10 +15,16 @@ apiServer:
         # profile is the OpenShift profile specifying a specific logging policy
         profile: Default
     featureGates:
+        # CustomNoUpgrade is used to enable/disable feature gates that block x- and y-stream upgrades.
         customNoUpgrade:
             disabled: []
             enabled: []
         featureSet: ""
+        # SpecialHandlingSupportExceptionRequired is used to enable/disable feature gates that are exempt from blocking x- and y-stream upgrades.
+        # Features listed here do not need to be duplicated in CustomNoUpgrade.
+        specialHandlingSupportExceptionRequired:
+            disabled: []
+            enabled: []
     # List of custom certificates used to secure requests to specific host names
     namedCertificates:
         - certPath: ""

pkg/admin/prerun/featuregate_lock.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/openshift/microshift/pkg/config"
 	"github.com/openshift/microshift/pkg/util"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/yaml"
 )
@@ -22,24 +23,26 @@ var (
 // featureGateLockFile represents the structure of the lock file
 // that tracks custom feature gate configuration and prevents changes/upgrades
 type featureGateLockFile struct {
-	FeatureSet      string                 `json:"featureSet"`
-	CustomNoUpgrade config.CustomNoUpgrade `json:"customNoUpgrade"`
-	Version         versionMetadata        `json:"version"`
+	FeatureSet      string                       `json:"featureSet"`
+	CustomNoUpgrade config.EnableDisableFeatures `json:"customNoUpgrade"`
+	Version         versionMetadata              `json:"version"`
 }
 
 // FeatureGateLockManagement manages the feature gate lock file
 // that prevents upgrades and config changes when custom feature gates are configured
 func FeatureGateLockManagement(cfg *config.Config) error {
 	klog.InfoS("START feature gate lock management")
-	if err := featureGateLockManagement(cfg); err != nil {
+
+	fgCfg := &cfg.ApiServer.FeatureGates
+	if err := featureGateLockManagement(fgCfg); err != nil {
 		klog.ErrorS(err, "FAIL feature gate lock management")
 		return err
 	}
 	klog.InfoS("END feature gate lock management")
 	return nil
 }
 
-func featureGateLockManagement(cfg *config.Config) error {
+func featureGateLockManagement(fgCfg *config.FeatureGates) error {
 	// If a lock file exists, it must be validated regardless of current config
 	// This prevents users from removing feature gates from config in order to block upgrades and configuration changes
 	lockExists, err := util.PathExists(featureGateLockFilePath)
@@ -48,18 +51,18 @@ func featureGateLockManagement(cfg *config.Config) error {
 	}
 	// Lock file exists - validate configuration
 	if lockExists {
-		return runValidationsChecks(cfg)
+		return runValidationsChecks(fgCfg)
 	}
 	// No lock file exists yet and custom feature gates are configured, so this is the first time configuring custom feature gates
-	if cfg.ApiServer.FeatureGates.FeatureSet != "" {
-		return createFeatureGateLockFile(cfg)
+	if fgCfg.FeatureSet != "" {
+		return createFeatureGateLockFile(fgCfg)
 	}
 	// No lock file and no custom feature gates - normal operation
 	return nil
 }
 
 // createFeatureGateLockFile creates the lock file with current configuration
-func createFeatureGateLockFile(cfg *config.Config) error {
+func createFeatureGateLockFile(fgCfg *config.FeatureGates) error {
 	klog.InfoS("Creating feature gate lock file - this cluster can no longer be upgraded",
 		"path", featureGateLockFilePath)
 
@@ -70,8 +73,8 @@ func createFeatureGateLockFile(cfg *config.Config) error {
 	}
 
 	lockFile := featureGateLockFile{
-		FeatureSet:      cfg.ApiServer.FeatureGates.FeatureSet,
-		CustomNoUpgrade: cfg.ApiServer.FeatureGates.CustomNoUpgrade,
+		FeatureSet:      fgCfg.FeatureSet,
+		CustomNoUpgrade: fgCfg.CustomNoUpgrade,
 		Version:         currentVersion,
 	}
 
@@ -88,7 +91,7 @@ func createFeatureGateLockFile(cfg *config.Config) error {
 
 // runValidationsChecks validates the feature gate lock file and the current configuration
 // It returns an error if the configuration is invalid or if an x or y stream version upgrade has occurred.
-func runValidationsChecks(cfg *config.Config) error {
+func runValidationsChecks(fgCfg *config.FeatureGates) error {
 	klog.InfoS("Validating feature gate lock file", "path", featureGateLockFilePath)
 
 	lockFile, err := readFeatureGateLockFile(featureGateLockFilePath)
@@ -97,45 +100,70 @@ func runValidationsChecks(cfg *config.Config) error {
 	}
 
 	// Check if feature gate configuration has changed
-	if err := configValidationChecksPass(lockFile, cfg.ApiServer.FeatureGates); err != nil {
+	if err := configValidationChecksPass(lockFile, fgCfg); err != nil {
 		return fmt.Errorf("detected invalid changes in feature gate configuration: %w\n\n"+
 			"To restore MicroShift to a supported state, you must:\n"+
 			"1. Run: sudo microshift-cleanup-data --all\n"+
 			"2. Remove custom feature gates from /etc/microshift/config.yaml\n"+
 			"3. Restart MicroShift: sudo systemctl restart microshift", err)
 	}
 
-	// Check if version has changed (upgrade attempted)
-	currentExecutableVersion, err := getExecutableVersion()
-	if err != nil {
-		return fmt.Errorf("failed to get current executable version: %w", err)
-	}
-
-	if lockFile.Version.Major != currentExecutableVersion.Major || lockFile.Version.Minor != currentExecutableVersion.Minor {
-		return fmt.Errorf("version upgrade detected with custom feature gates: locked version %s, current version %s\n\n"+
-			"Upgrades are not supported when custom feature gates are configured.\n"+
-			"Custom feature gates (%s) were configured in version %s.\n"+
-			"To restore MicroShift to a supported state, you must:\n"+
-			"1. Roll back to version %s, OR\n"+
-			"2. Run: sudo microshift-cleanup-data --all\n"+
-			"3. Remove custom feature gates from /etc/microshift/config.yaml\n"+
-			"4. Restart MicroShift: sudo systemctl restart microshift",
-			lockFile.Version.String(), currentExecutableVersion.String(),
-			lockFile.FeatureSet, lockFile.Version.String(), lockFile.Version.String())
+	if err := upgradeChecksPass(lockFile, fgCfg); err != nil {
+		return err
 	}
 
 	klog.InfoS("Feature gate lock file validation successful")
 	return nil
 }
 
-func configValidationChecksPass(prev featureGateLockFile, current config.FeatureGates) error {
-	if prev.FeatureSet != "" && current.FeatureSet == "" {
+func configValidationChecksPass(prev featureGateLockFile, fgCfg *config.FeatureGates) error {
+	if prev.FeatureSet != "" && fgCfg.FeatureSet == "" {
 		// Disallow changing from feature set to no feature set
 		return fmt.Errorf("cannot unset feature set. Previous config had feature set %q, current config has no feature set configured", prev.FeatureSet)
 	}
-	if prev.FeatureSet == config.FeatureSetCustomNoUpgrade && current.FeatureSet != config.FeatureSetCustomNoUpgrade {
+	if prev.FeatureSet == config.FeatureSetCustomNoUpgrade && fgCfg.FeatureSet != config.FeatureSetCustomNoUpgrade {
 		// Disallow changing from custom feature gates to any other feature set
-		return fmt.Errorf("cannot change CustomNoUpgrade feature set. Previous feature set was %q, current feature set is %q", prev.FeatureSet, current.FeatureSet)
+		return fmt.Errorf("cannot change CustomNoUpgrade feature set. Previous feature set was %q, current feature set is %q", prev.FeatureSet, fgCfg.FeatureSet)
+	}
+	return nil
+}
+
+func upgradeChecksPass(lockFile featureGateLockFile, fgCfg *config.FeatureGates) error {
+	currentExecutableVersion, err := getExecutableVersion()
+	lockedVersion := lockFile.Version
+	if err != nil {
+		return fmt.Errorf("failed to get current executable version: %w", err)
+	}
+
+	if lockedVersion.Major != currentExecutableVersion.Major || lockedVersion.Minor != currentExecutableVersion.Minor {
+		extractFeatureGatesWithoutExemptions := func(lhs []string, rhs []string) []string {
+			lhsSet := sets.New(lhs...)
+			rhsSet := sets.New(rhs...)
+			return lhsSet.Difference(rhsSet).UnsortedList()
+		}
+
+		// Parse out featureGates from the last known cluster state and filter them by the latest config. This allows users to update
+		// the feature gate config and upgrade the cluster simultaneously if they want. In the typical case where the config does not change
+		// between upgrades, non-exempt featureGates are still validated and will still block the upgrade.
+		customNoUpgradeEnabled := extractFeatureGatesWithoutExemptions(lockFile.CustomNoUpgrade.Enabled, fgCfg.SpecialHandlingSupportExceptionRequired.Enabled)
+		customNoUpgradeDisabled := extractFeatureGatesWithoutExemptions(lockFile.CustomNoUpgrade.Disabled, fgCfg.SpecialHandlingSupportExceptionRequired.Disabled)
+
+		// If there are any gates that lack a special handling support exception, return an error.
+		if len(customNoUpgradeEnabled) > 0 || len(customNoUpgradeDisabled) > 0 {
+			return fmt.Errorf("version upgrade detected with custom feature gates: locked version %s, current version %s\n\n"+
+				"Upgrades are not supported when custom feature gates are configured.\n"+
+				"Custom feature gates were configured in version %s.\n"+
+				"Gates Enabled: %s\n"+
+				"Gates Disabled: %s\n"+
+				"To restore MicroShift to a supported state, you must:\n"+
+				"1. Roll back to version %s, OR\n"+
+				"2. Run: sudo microshift-cleanup-data --all\n"+
+				"3. Remove custom feature gates from /etc/microshift/config.yaml\n"+
+				"4. Restart MicroShift: sudo systemctl restart microshift",
+				lockedVersion.String(), currentExecutableVersion.String(),
+				lockedVersion.String(), customNoUpgradeEnabled,
+				customNoUpgradeDisabled, lockedVersion.String())
+		}
 	}
 	return nil
 }