diff --git a/docs/howto_config.md b/docs/howto_config.md index ce635c6528..7edd953575 100644 --- a/docs/howto_config.md +++ b/docs/howto_config.md @@ -39,7 +39,7 @@ The configuration settings alongside with the supported command line arguments a | nodeIP | --node-ip | MICROSHIFT_NODEIP | The IP address of the node, defaults to IP of the default route | hostnameOverride | --hostname-override | MICROSHIFT_HOSTNAMEOVERRIDE | The name of the node, defaults to hostname | logLevel | --v | MICROSHIFT_LOGVLEVEL | Log verbosity (Normal, Debug, Trace, TraceAll) -| subjectAltNames | --subject-alt-names | MICROSHIFT_SUBJECTALTNAMES | Subject Alternative Names for apiserver certificates +| subjectAltNames | --subject-alt-names | MICROSHIFT_SUBJECTALTNAMES | Subject Alternative Names for apiserver certificates ## Default Settings diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 063ad4bd0d..ea151892b4 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -70,6 +70,21 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err return nil, err } + externalCertNames := []string{ + cfg.NodeName, + "api." + cfg.BaseDomain, + } + externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) + // When Kube apiserver advertise address matches the node IP we can not add + // it to the certificates or else the internal pod access to apiserver is + // broken. Because of client-go not using SNI and the way apiserver handles + // which certificate to serve which destination IP, internal pods start + // getting the external certificate, which is signed by a different CA and + // does not match the hostname. + if cfg.KASAdvertiseAddress != cfg.NodeIP { + externalCertNames = append(externalCertNames, cfg.NodeIP) + } + certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) certChains, err := certchains.NewCertificateChains( @@ -231,11 +246,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err Name: "kube-external-serving", ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, - Hostnames: append( - cfg.SubjectAltNames, - cfg.NodeName, - "api."+cfg.BaseDomain, - ), + Hostnames: externalCertNames, }, ), diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 2e1e41f54a..dd013f4c7b 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -93,6 +93,7 @@ func RunMicroshift(cfg *config.MicroshiftConfig, flags *pflag.FlagSet) error { } m := servicemanager.NewServiceManager() + util.Must(m.AddService(node.NewNetworkConfiguration(cfg))) util.Must(m.AddService(controllers.NewEtcd(cfg))) util.Must(m.AddService(sysconfwatch.NewSysConfWatchController(cfg))) util.Must(m.AddService(controllers.NewKubeAPIServer(cfg))) diff --git a/pkg/config/config.go b/pkg/config/config.go index dd94b4cc3b..04b5060d63 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -59,11 +59,20 @@ type IngressConfig struct { type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` - SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` - BaseDomain string `json:"baseDomain"` - Cluster ClusterConfig `json:"cluster"` + SubjectAltNames []string `json:"subjectAltNames"` + NodeName string `json:"nodeName"` + NodeIP string `json:"nodeIP"` + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. + KASAdvertiseAddress string `json:"kasAdvertiseAddress"` + // Determines if kube-apiserver controller should configure the + // KASAdvertiseAddress in the loopback interface. Automatically computed. + SkipKASInterface bool `json:"-"` + BaseDomain string `json:"baseDomain"` + Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` } @@ -118,6 +127,9 @@ type DNS struct { type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` + // AdvertiseAddress for endpoint slices in kubernetes service. Developer + // only parameter, wont show in show-config commands or docs. + AdvertiseAddress string `json:"advertiseAddress,omitempty"` } type Node struct { @@ -363,6 +375,9 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if len(config.ApiServer.SubjectAltNames) > 0 { c.SubjectAltNames = config.ApiServer.SubjectAltNames } + if len(config.ApiServer.AdvertiseAddress) > 0 { + c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress + } return nil } @@ -425,6 +440,21 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string, flags *pflag.FlagS } c.Cluster.DNS = clusterDNS + // If KAS advertise address is not configured then grab it from the service + // CIDR automatically. + if len(c.KASAdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) + if err != nil { + return fmt.Errorf("error getting apiserver IP: %v", err) + } + c.KASAdvertiseAddress = apiServerServiceIP.String() + c.SkipKASInterface = false + } else { + c.SkipKASInterface = true + } + if len(c.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with @@ -455,12 +485,6 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string, flags *pflag.FlagS } } - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) - _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) - if err != nil { - return fmt.Errorf("error getting apiserver IP: %v", err) - } if stringSliceContains( c.SubjectAltNames, "kubernetes", @@ -471,7 +495,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string, flags *pflag.FlagS "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - apiServerServiceIP.String(), + c.KASAdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index fa3415563a..d1ee3cb126 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -71,8 +71,9 @@ type KubeAPIServer struct { verbosity int configureErr error // todo: report configuration errors immediately - masterURL string - servingCAPath string + masterURL string + servingCAPath string + advertiseAddress string } func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { @@ -84,7 +85,7 @@ func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { } func (s *KubeAPIServer) Name() string { return "kube-apiserver" } -func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd"} } +func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { s.verbosity = cfg.LogVLevel @@ -112,10 +113,11 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { s.masterURL = cfg.Cluster.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) + s.advertiseAddress = cfg.KASAdvertiseAddress overrides := &kubecontrolplanev1.KubeAPIServerConfig{ APIServerArguments: map[string]kubecontrolplanev1.Arguments{ - "advertise-address": {cfg.NodeIP}, + "advertise-address": {s.advertiseAddress}, "audit-policy-file": {microshiftDataDir + "/resources/kube-apiserver-audit-policies/default.yaml"}, "client-ca-file": {clientCABundlePath}, "etcd-cafile": {cryptomaterial.CACertPath(cryptomaterial.EtcdSignerDir(certsDir))}, diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go new file mode 100644 index 0000000000..e43813d7f8 --- /dev/null +++ b/pkg/node/netconfig.go @@ -0,0 +1,120 @@ +/* +Copyright © 2023 MicroShift Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package node + +import ( + "context" + "fmt" + + "k8s.io/klog/v2" + + "github.com/vishvananda/netlink" + + "github.com/openshift/microshift/pkg/config" +) + +const ( + // Network configuration component name + componentNetworkConfiguration = "network-configuration" + // Interface name where to add service IP + loopbackInterface = "lo" +) + +type NetworkConfiguration struct { + kasAdvertiseAddress string + skipInterfaceConfiguration bool +} + +func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration { + n := &NetworkConfiguration{} + n.configure(cfg) + return n +} + +func (n *NetworkConfiguration) Name() string { return componentNetworkConfiguration } +func (n *NetworkConfiguration) Dependencies() []string { return []string{} } + +func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { + n.kasAdvertiseAddress = cfg.KASAdvertiseAddress + n.skipInterfaceConfiguration = cfg.SkipKASInterface +} + +func (n *NetworkConfiguration) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { + defer close(stopped) + + stopChan := make(chan struct{}) + + if !n.skipInterfaceConfiguration { + if err := n.addServiceIPLoopback(); err != nil { + return err + } + go func() { + select { + case <-ctx.Done(): + if err := n.removeServiceIPLoopback(); err != nil { + klog.Warningf("failed to remove IP from interface: %v", err) + } + close(stopChan) + } + }() + } + klog.Infof("%q is ready", n.Name()) + close(ready) + <-stopChan + return ctx.Err() +} + +func (n *NetworkConfiguration) addServiceIPLoopback() error { + link, err := netlink.LinkByName(loopbackInterface) + if err != nil { + return err + } + address, err := netlink.ParseAddr(fmt.Sprintf("%s/32", n.kasAdvertiseAddress)) + if err != nil { + return err + } + existing, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return err + } + for _, existingAddress := range existing { + if address.Equal(existingAddress) { + return nil + } + } + return netlink.AddrAdd(link, address) +} + +func (n *NetworkConfiguration) removeServiceIPLoopback() error { + link, err := netlink.LinkByName(loopbackInterface) + if err != nil { + return err + } + address, err := netlink.ParseAddr(fmt.Sprintf("%s/32", n.kasAdvertiseAddress)) + if err != nil { + return err + } + existing, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return err + } + for _, existingAddress := range existing { + if address.Equal(existingAddress) { + return netlink.AddrDel(link, address) + } + } + return nil +}