[release-4.13] ETCD-403: Revert user configuration for etcd and change default max quota

docs/howto_config.md

@@ -21,11 +21,7 @@ apiServer:
 debugging:
   logLevel: ""
 etcd:
-  quotaBackendSize: ""
-  defragCheckFreq: ""
-  doStartupDefrag: false
-  minDefragSize: ""
-  maxFragmentedPercentage: 0
+  memoryLimitMB: 0
 ```
 
 ## Default Settings
@@ -49,11 +45,7 @@ apiServer:
 debugging:
   logLevel: "Normal"
 etcd:
-  quotaBackendSize: "2Gi"
-  defragCheckFreq: "5m"
-  doStartupDefrag: true
-  minDefragSize: "100Mi"
-  maxFragmentedPercentage: 45
+  memoryLimitMB: 0
 ```
 
 ## Service NodePort range
@@ -90,6 +82,14 @@ List of ports that you must avoid:
 | 10259/tcp     | kube scheduler
 |---------------|-----------------------------------------------------------------|
 
+## Etcd Memory Limit
+
+By default, etcd will be allowed to use as much memory as it needs to handle the load on the system; however, in memory constrained systems, it may be preferred or necessary to limit the amount of memory etcd is allowed to use at a given time.
+
+Setting the `memoryLimitMB` to a value greater than 0 will result in a soft memory limit being applied to etcd; etcd will be allowed to go over this value during operation, but memory will be more aggresively reclaimed from it if it does. A value of `128` megabytes is the recommended starting place for this limit; however, the configuration floor is `50` megabytes - attempting to set the limit below 50 megabytes will result in the configuration being 50 megabytes.
+
+Please note that values between 50 and 128 megabytes will heavily trade off memory footprint for etcd performance: the lower the memory limit, the more time etcd will spend on paging memory to disk and will take longer to respond to queries or even timing requests out if the limit is low and the etcd usage is high.
+
 # Auto-applying Manifests
 
 MicroShift leverages `kustomize` for Kubernetes-native templating and declarative management of resource objects. Upon start-up, it searches `/etc/microshift/manifests` and `/usr/lib/microshift/manifests` directories for a `kustomization.yaml` file. If it finds one, it automatically runs `kubectl apply -k` command to apply that manifest.

etcd/cmd/microshift-etcd/run.go

@@ -190,6 +190,9 @@ func setURL(hostnames []string, port string) []url.URL {
 	return urls
 }
 
+// The following 'fragemented' logic is copied from the Openshift Cluster Etcd Operator.
+//
+//	https://github.com/openshift/cluster-etcd-operator/blob/0584b0d1c8868535baf889d8c199f605aef4a3ae/pkg/operator/defragcontroller/defragcontroller.go#L282
 func isBackendFragmented(b backend.Backend, maxFragmentedPercentage float64, minDefragBytes int64) bool {
 	fragmentedPercentage := checkFragmentationPercentage(b.Size(), b.SizeInUse())
 	if fragmentedPercentage > 0.00 {

packaging/microshift/config.yaml

@@ -31,18 +31,5 @@ debugging:
   #logLevel: 'Normal'
 
 etcd:
-  # The limit on the size of the etcd database; the etcd server will start failing writes if its size on disk reaches this value. (Default: 2GB)
-  #quotaBackendSize: '2Gi'
-
-  # How often to check the conditions for defragmenting the etcd database (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). (Default: 5 minutes)
-  #defragCheckFreq: '5m'
-
-  # Whether or not to defragment the etcd database when the etcd server finishes starting. (Default: true)
-  #doStartupDefrag: true
-
-  # Defragment conditions: if both of the following are true when the condition check (controlled by the DefragCheckFreq) runs, defragment the etcd database.
-  # The minimum size of the etcd database, if the database is smaller than this value, defragmenting will not occur. (Default: 100MB)
-  #minDefragSize: '100Mi'
-
-  # The maximum allowed fragmented percentage, if the database is fragmented less than this value, defragmenting will not occur. (Default: 45)
-  #maxFragmentedPercentage: 45
+  # Memory limit for etcd, in Megabytes: 0 is no limit.
+  #memoryLimitMB: 0

pkg/config/config.go

@@ -17,7 +17,6 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/pflag"
 
-	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/component-base/logs"
 	"k8s.io/klog/v2"
@@ -58,7 +57,9 @@ type IngressConfig struct {
 	ServingKey         []byte
 }
 
-type InternalEtcdConfig struct {
+type EtcdConfig struct {
+	// Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit.
+	MemoryLimit uint64
 	// The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value
 	QuotaBackendBytes int64
 	// If the backend is fragmented more than `maxFragmentedPercentage`
@@ -71,19 +72,6 @@ type InternalEtcdConfig struct {
 	DoStartupDefrag bool
 }
 
-type EtcdConfig struct {
-	// The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value
-	QuotaBackendSize string
-	// If the backend is fragmented more than `maxFragmentedPercentage`
-	//		and the database size is greater than `minDefragSize`, do a defrag.
-	MinDefragSize           string
-	MaxFragmentedPercentage float64
-	// How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set).
-	DefragCheckFreq string
-	// Whether or not to do a defrag when the server finishes starting
-	DoStartupDefrag bool
-}
-
 type MicroshiftConfig struct {
 	LogVLevel int `json:"logVLevel"`
 
@@ -102,18 +90,28 @@ type MicroshiftConfig struct {
 	BaseDomain       string        `json:"baseDomain"`
 	Cluster          ClusterConfig `json:"cluster"`
 
-	Ingress IngressConfig      `json:"-"`
-	Etcd    InternalEtcdConfig `json:"etcd"`
+	Ingress IngressConfig `json:"-"`
+	Etcd    EtcdConfig    `json:"etcd"`
 }
 
 // Top level config file
 type Config struct {
-	DNS       DNS        `json:"dns"`
-	Network   Network    `json:"network"`
-	Node      Node       `json:"node"`
-	ApiServer ApiServer  `json:"apiServer"`
-	Debugging Debugging  `json:"debugging"`
-	Etcd      EtcdConfig `json:"etcd"`
+	DNS       DNS       `json:"dns"`
+	Network   Network   `json:"network"`
+	Node      Node      `json:"node"`
+	ApiServer ApiServer `json:"apiServer"`
+	Debugging Debugging `json:"debugging"`
+	Etcd      Etcd      `json:"etcd"`
+}
+
+const (
+	// Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum.
+	EtcdMinimumMemoryLimit = 50
+)
+
+type Etcd struct {
+	// Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit.
+	MemoryLimitMB uint64 `json:"memoryLimitMB"`
 }
 
 type Network struct {
@@ -252,12 +250,13 @@ func NewMicroshiftConfig() *MicroshiftConfig {
 			ServiceCIDR:          "10.43.0.0/16",
 			ServiceNodePortRange: "30000-32767",
 		},
-		Etcd: InternalEtcdConfig{
+		Etcd: EtcdConfig{
+			MemoryLimit:             0,                 // No limit
 			MinDefragBytes:          100 * 1024 * 1024, // 100MB
 			MaxFragmentedPercentage: 45,                // percent
 			DefragCheckFreq:         5 * time.Minute,
 			DoStartupDefrag:         true,
-			QuotaBackendBytes:       2 * 1024 * 1024 * 1024, // 2GB
+			QuotaBackendBytes:       8 * 1024 * 1024 * 1024, // 8GB
 		},
 	}
 }
@@ -416,35 +415,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error {
 		c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress
 	}
 
-	if config.Etcd.DefragCheckFreq != "" {
-		d, err := time.ParseDuration(config.Etcd.DefragCheckFreq)
-		if err != nil {
-			return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err)
-		}
-		c.Etcd.DefragCheckFreq = d
-	}
-	if config.Etcd.MinDefragSize != "" {
-		q, err := resource.ParseQuantity(config.Etcd.MinDefragSize)
-		if err != nil {
-			return fmt.Errorf("failed to parse etcd minDefragSize: %v", err)
-		}
-		if !q.IsZero() {
-			c.Etcd.MinDefragBytes = q.Value()
-		}
-	}
-	if config.Etcd.MaxFragmentedPercentage > 0 {
-		c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage
-	}
-	if config.Etcd.QuotaBackendSize != "" {
-		q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize)
-		if err != nil {
-			return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err)
-		}
-		if !q.IsZero() {
-			c.Etcd.QuotaBackendBytes = q.Value()
+	if config.Etcd.MemoryLimitMB > 0 {
+		// If the memory limit is than the minimum, set it to the minimum and continue.
+		if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit {
+			c.Etcd.MemoryLimit = EtcdMinimumMemoryLimit
+		} else {
+			c.Etcd.MemoryLimit = config.Etcd.MemoryLimitMB
 		}
 	}
-	c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag
 
 	return nil
 }

pkg/config/config_test.go

@@ -57,13 +57,6 @@ func TestConfigFile(t *testing.T) {
 				Debugging: Debugging{
 					LogLevel: "Debug",
 				},
-				Etcd: EtcdConfig{
-					QuotaBackendSize:        "2Gi",
-					MinDefragSize:           "100Mi",
-					MaxFragmentedPercentage: 45,
-					DefragCheckFreq:         "5m",
-					DoStartupDefrag:         true,
-				},
 			},
 			expected: MicroshiftConfig{
 				LogVLevel:           4,
@@ -78,8 +71,9 @@ func TestConfigFile(t *testing.T) {
 					ServiceCIDR:          "40.30.20.10/16",
 					ServiceNodePortRange: "1024-32767",
 				},
-				Etcd: InternalEtcdConfig{
-					QuotaBackendBytes:       2 * 1024 * 1024 * 1024,
+				Etcd: EtcdConfig{
+					MemoryLimit:             0,
+					QuotaBackendBytes:       8 * 1024 * 1024 * 1024,
 					MinDefragBytes:          100 * 1024 * 1024,
 					MaxFragmentedPercentage: 45,
 					DefragCheckFreq:         5 * time.Minute,
@@ -157,13 +151,6 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) {
 				Debugging: Debugging{
 					LogLevel: "Debug",
 				},
-				Etcd: EtcdConfig{
-					QuotaBackendSize:        "2Gi",
-					MinDefragSize:           "100Mi",
-					MaxFragmentedPercentage: 45,
-					DefragCheckFreq:         "5m",
-					DoStartupDefrag:         true,
-				},
 			},
 			expected: MicroshiftConfig{
 				LogVLevel:           4,
@@ -180,8 +167,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) {
 					ServiceNodePortRange: "1024-32767",
 					DNS:                  "40.30.0.10",
 				},
-				Etcd: InternalEtcdConfig{
-					QuotaBackendBytes:       2 * 1024 * 1024 * 1024,
+				Etcd: EtcdConfig{
+					MemoryLimit:             0,
+					QuotaBackendBytes:       8 * 1024 * 1024 * 1024,
 					MinDefragBytes:          100 * 1024 * 1024,
 					MaxFragmentedPercentage: 45,
 					DefragCheckFreq:         5 * time.Minute,