From 87bb638c9b6f937b665659ebe6b8234560954249 Mon Sep 17 00:00:00 2001 From: Allen Ray Date: Wed, 15 Mar 2023 11:40:01 -0400 Subject: [PATCH] ETCD-403: Revert user configuration for etcd and change default max quota --- docs/howto_config.md | 20 ++--- etcd/cmd/microshift-etcd/run.go | 3 + .../openshift/microshift/pkg/config/config.go | 82 +++++++------------ packaging/microshift/config.yaml | 17 +--- pkg/config/config.go | 82 +++++++------------ pkg/config/config_test.go | 24 ++---- pkg/controllers/etcd.go | 25 ++++-- 7 files changed, 99 insertions(+), 154 deletions(-) diff --git a/docs/howto_config.md b/docs/howto_config.md index 5fb4f11a2b..6103dee49e 100644 --- a/docs/howto_config.md +++ b/docs/howto_config.md @@ -21,11 +21,7 @@ apiServer: debugging: logLevel: "" etcd: - quotaBackendSize: "" - defragCheckFreq: "" - doStartupDefrag: false - minDefragSize: "" - maxFragmentedPercentage: 0 + memoryLimitMB: 0 ``` ## Default Settings @@ -49,11 +45,7 @@ apiServer: debugging: logLevel: "Normal" etcd: - quotaBackendSize: "2Gi" - defragCheckFreq: "5m" - doStartupDefrag: true - minDefragSize: "100Mi" - maxFragmentedPercentage: 45 + memoryLimitMB: 0 ``` ## Service NodePort range @@ -90,6 +82,14 @@ List of ports that you must avoid: | 10259/tcp | kube scheduler |---------------|-----------------------------------------------------------------| +## Etcd Memory Limit + +By default, etcd will be allowed to use as much memory as it needs to handle the load on the system; however, in memory constrained systems, it may be preferred or necessary to limit the amount of memory etcd is allowed to use at a given time. + +Setting the `memoryLimitMB` to a value greater than 0 will result in a soft memory limit being applied to etcd; etcd will be allowed to go over this value during operation, but memory will be more aggresively reclaimed from it if it does. A value of `128` megabytes is the recommended starting place for this limit; however, the configuration floor is `50` megabytes - attempting to set the limit below 50 megabytes will result in the configuration being 50 megabytes. + +Please note that values between 50 and 128 megabytes will heavily trade off memory footprint for etcd performance: the lower the memory limit, the more time etcd will spend on paging memory to disk and will take longer to respond to queries or even timing requests out if the limit is low and the etcd usage is high. + # Auto-applying Manifests MicroShift leverages `kustomize` for Kubernetes-native templating and declarative management of resource objects. Upon start-up, it searches `/etc/microshift/manifests` and `/usr/lib/microshift/manifests` directories for a `kustomization.yaml` file. If it finds one, it automatically runs `kubectl apply -k` command to apply that manifest. diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 0026092976..0ea60ef6ec 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -190,6 +190,9 @@ func setURL(hostnames []string, port string) []url.URL { return urls } +// The following 'fragemented' logic is copied from the Openshift Cluster Etcd Operator. +// +// https://github.com/openshift/cluster-etcd-operator/blob/0584b0d1c8868535baf889d8c199f605aef4a3ae/pkg/operator/defragcontroller/defragcontroller.go#L282 func isBackendFragmented(b backend.Backend, maxFragmentedPercentage float64, minDefragBytes int64) bool { fragmentedPercentage := checkFragmentationPercentage(b.Size(), b.SizeInUse()) if fragmentedPercentage > 0.00 { diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 11359242ea..fc541ae654 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -17,7 +17,6 @@ import ( "github.com/mitchellh/go-homedir" "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/component-base/logs" "k8s.io/klog/v2" @@ -58,7 +57,9 @@ type IngressConfig struct { ServingKey []byte } -type InternalEtcdConfig struct { +type EtcdConfig struct { + // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. + MemoryLimit uint64 // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value QuotaBackendBytes int64 // If the backend is fragmented more than `maxFragmentedPercentage` @@ -71,19 +72,6 @@ type InternalEtcdConfig struct { DoStartupDefrag bool } -type EtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` @@ -102,18 +90,28 @@ type MicroshiftConfig struct { BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd InternalEtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd EtcdConfig `json:"etcd"` } // Top level config file type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` - Etcd EtcdConfig `json:"etcd"` + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Debugging Debugging `json:"debugging"` + Etcd Etcd `json:"etcd"` +} + +const ( + // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. + EtcdMinimumMemoryLimit = 50 +) + +type Etcd struct { + // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. + MemoryLimitMB uint64 `json:"memoryLimitMB"` } type Network struct { @@ -252,12 +250,13 @@ func NewMicroshiftConfig() *MicroshiftConfig { ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: InternalEtcdConfig{ + Etcd: EtcdConfig{ + MemoryLimit: 0, // No limit MinDefragBytes: 100 * 1024 * 1024, // 100MB MaxFragmentedPercentage: 45, // percent DefragCheckFreq: 5 * time.Minute, DoStartupDefrag: true, - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GB + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, // 8GB }, } } @@ -416,35 +415,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } - if config.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) - if err != nil { - return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) - } - c.Etcd.DefragCheckFreq = d - } - if config.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) - if err != nil { - return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) - } - if !q.IsZero() { - c.Etcd.MinDefragBytes = q.Value() - } - } - if config.Etcd.MaxFragmentedPercentage > 0 { - c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage - } - if config.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) - if err != nil { - return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) - } - if !q.IsZero() { - c.Etcd.QuotaBackendBytes = q.Value() + if config.Etcd.MemoryLimitMB > 0 { + // If the memory limit is than the minimum, set it to the minimum and continue. + if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + c.Etcd.MemoryLimit = EtcdMinimumMemoryLimit + } else { + c.Etcd.MemoryLimit = config.Etcd.MemoryLimitMB } } - c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag return nil } diff --git a/packaging/microshift/config.yaml b/packaging/microshift/config.yaml index a74e6c120f..70cbbaae63 100644 --- a/packaging/microshift/config.yaml +++ b/packaging/microshift/config.yaml @@ -31,18 +31,5 @@ debugging: #logLevel: 'Normal' etcd: - # The limit on the size of the etcd database; the etcd server will start failing writes if its size on disk reaches this value. (Default: 2GB) - #quotaBackendSize: '2Gi' - - # How often to check the conditions for defragmenting the etcd database (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). (Default: 5 minutes) - #defragCheckFreq: '5m' - - # Whether or not to defragment the etcd database when the etcd server finishes starting. (Default: true) - #doStartupDefrag: true - - # Defragment conditions: if both of the following are true when the condition check (controlled by the DefragCheckFreq) runs, defragment the etcd database. - # The minimum size of the etcd database, if the database is smaller than this value, defragmenting will not occur. (Default: 100MB) - #minDefragSize: '100Mi' - - # The maximum allowed fragmented percentage, if the database is fragmented less than this value, defragmenting will not occur. (Default: 45) - #maxFragmentedPercentage: 45 + # Memory limit for etcd, in Megabytes: 0 is no limit. + #memoryLimitMB: 0 diff --git a/pkg/config/config.go b/pkg/config/config.go index 11359242ea..fc541ae654 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,7 +17,6 @@ import ( "github.com/mitchellh/go-homedir" "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/component-base/logs" "k8s.io/klog/v2" @@ -58,7 +57,9 @@ type IngressConfig struct { ServingKey []byte } -type InternalEtcdConfig struct { +type EtcdConfig struct { + // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. + MemoryLimit uint64 // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value QuotaBackendBytes int64 // If the backend is fragmented more than `maxFragmentedPercentage` @@ -71,19 +72,6 @@ type InternalEtcdConfig struct { DoStartupDefrag bool } -type EtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` @@ -102,18 +90,28 @@ type MicroshiftConfig struct { BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd InternalEtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd EtcdConfig `json:"etcd"` } // Top level config file type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` - Etcd EtcdConfig `json:"etcd"` + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Debugging Debugging `json:"debugging"` + Etcd Etcd `json:"etcd"` +} + +const ( + // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. + EtcdMinimumMemoryLimit = 50 +) + +type Etcd struct { + // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. + MemoryLimitMB uint64 `json:"memoryLimitMB"` } type Network struct { @@ -252,12 +250,13 @@ func NewMicroshiftConfig() *MicroshiftConfig { ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: InternalEtcdConfig{ + Etcd: EtcdConfig{ + MemoryLimit: 0, // No limit MinDefragBytes: 100 * 1024 * 1024, // 100MB MaxFragmentedPercentage: 45, // percent DefragCheckFreq: 5 * time.Minute, DoStartupDefrag: true, - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GB + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, // 8GB }, } } @@ -416,35 +415,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } - if config.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) - if err != nil { - return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) - } - c.Etcd.DefragCheckFreq = d - } - if config.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) - if err != nil { - return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) - } - if !q.IsZero() { - c.Etcd.MinDefragBytes = q.Value() - } - } - if config.Etcd.MaxFragmentedPercentage > 0 { - c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage - } - if config.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) - if err != nil { - return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) - } - if !q.IsZero() { - c.Etcd.QuotaBackendBytes = q.Value() + if config.Etcd.MemoryLimitMB > 0 { + // If the memory limit is than the minimum, set it to the minimum and continue. + if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + c.Etcd.MemoryLimit = EtcdMinimumMemoryLimit + } else { + c.Etcd.MemoryLimit = config.Etcd.MemoryLimitMB } } - c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag return nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1be50403c6..3616c79a28 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -57,13 +57,6 @@ func TestConfigFile(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - Etcd: EtcdConfig{ - QuotaBackendSize: "2Gi", - MinDefragSize: "100Mi", - MaxFragmentedPercentage: 45, - DefragCheckFreq: "5m", - DoStartupDefrag: true, - }, }, expected: MicroshiftConfig{ LogVLevel: 4, @@ -78,8 +71,9 @@ func TestConfigFile(t *testing.T) { ServiceCIDR: "40.30.20.10/16", ServiceNodePortRange: "1024-32767", }, - Etcd: InternalEtcdConfig{ - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, + Etcd: EtcdConfig{ + MemoryLimit: 0, + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, DefragCheckFreq: 5 * time.Minute, @@ -157,13 +151,6 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - Etcd: EtcdConfig{ - QuotaBackendSize: "2Gi", - MinDefragSize: "100Mi", - MaxFragmentedPercentage: 45, - DefragCheckFreq: "5m", - DoStartupDefrag: true, - }, }, expected: MicroshiftConfig{ LogVLevel: 4, @@ -180,8 +167,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { ServiceNodePortRange: "1024-32767", DNS: "40.30.0.10", }, - Etcd: InternalEtcdConfig{ - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, + Etcd: EtcdConfig{ + MemoryLimit: 0, + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, DefragCheckFreq: 5 * time.Minute, diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index 5596e8cb25..98e56458ae 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -36,10 +36,14 @@ var ( HealthCheckWait = time.Duration(3 * time.Second) ) -type EtcdService struct{} +type EtcdService struct { + memoryLimit uint64 +} func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { - return &EtcdService{} + return &EtcdService{ + memoryLimit: cfg.Etcd.MemoryLimit, + } } func (s *EtcdService) Name() string { return "etcd" } @@ -59,24 +63,31 @@ func (s *EtcdService) Run(ctx context.Context, ready chan<- struct{}, stopped ch etcdPath := filepath.Join(filepath.Dir(microshiftExecPath), "microshift-etcd") // Not running the etcd binary directly, the proper etcd arguments are handled // in etcd/cmd/microshift-etcd/run.go. - args := []string{"run"} + args := []string{} // If we're launching MicroShift as a service, we need to do the same // with etcd, so wrap it in a transient systemd-unit that's tied // to the MicroShift service lifetime. var exe string if runningAsSvc { - args = append([]string{ + args = append(args, "--uid=root", "--scope", - "--property", "BindsTo=microshift.service", "--unit", "microshift-etcd", - etcdPath, - }, args...) + "--property", "BindsTo=microshift.service", + ) + + if s.memoryLimit > 0 { + args = append(args, "--property", fmt.Sprintf("MemoryHigh=%vM", s.memoryLimit)) + } + + args = append(args, etcdPath) + exe = "systemd-run" } else { exe = etcdPath } + args = append(args, "run") // Not using context as canceling ctx sends SIGKILL to process cmd := exec.Command(exe, args...)