[release-4.12] OCPBUGS-7467: add configurable KAS advertise address

docs/howto_config.md

@@ -39,7 +39,7 @@ The configuration settings alongside with the supported command line arguments a
 | nodeIP                | --node-ip                 | MICROSHIFT_NODEIP                       | The IP address of the node, defaults to IP of the default route
 | hostnameOverride      | --hostname-override       | MICROSHIFT_HOSTNAMEOVERRIDE             | The name of the node, defaults to hostname
 | logLevel              | --v                       | MICROSHIFT_LOGVLEVEL                    | Log verbosity (Normal, Debug, Trace, TraceAll)
-| subjectAltNames     | --subject-alt-names       | MICROSHIFT_SUBJECTALTNAMES              | Subject Alternative Names for apiserver certificates
+| subjectAltNames       | --subject-alt-names       | MICROSHIFT_SUBJECTALTNAMES              | Subject Alternative Names for apiserver certificates
 
 ## Default Settings
 

pkg/cmd/init.go

@@ -70,6 +70,21 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err
 		return nil, err
 	}
 
+	externalCertNames := []string{
+		cfg.NodeName,
+		"api." + cfg.BaseDomain,
+	}
+	externalCertNames = append(externalCertNames, cfg.SubjectAltNames...)
+	// When Kube apiserver advertise address matches the node IP we can not add
+	// it to the certificates or else the internal pod access to apiserver is
+	// broken. Because of client-go not using SNI and the way apiserver handles
+	// which certificate to serve which destination IP, internal pods start
+	// getting the external certificate, which is signed by a different CA and
+	// does not match the hostname.
+	if cfg.KASAdvertiseAddress != cfg.NodeIP {
+		externalCertNames = append(externalCertNames, cfg.NodeIP)
+	}
+
 	certsDir := cryptomaterial.CertsDirectory(microshiftDataDir)
 
 	certChains, err := certchains.NewCertificateChains(
@@ -231,11 +246,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err
 					Name:         "kube-external-serving",
 					ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays,
 				},
-				Hostnames: append(
-					cfg.SubjectAltNames,
-					cfg.NodeName,
-					"api."+cfg.BaseDomain,
-				),
+				Hostnames: externalCertNames,
 			},
 		),
 

pkg/cmd/run.go

@@ -93,6 +93,7 @@ func RunMicroshift(cfg *config.MicroshiftConfig, flags *pflag.FlagSet) error {
 	}
 
 	m := servicemanager.NewServiceManager()
+	util.Must(m.AddService(node.NewNetworkConfiguration(cfg)))
 	util.Must(m.AddService(controllers.NewEtcd(cfg)))
 	util.Must(m.AddService(sysconfwatch.NewSysConfWatchController(cfg)))
 	util.Must(m.AddService(controllers.NewKubeAPIServer(cfg)))

pkg/config/config.go

@@ -59,11 +59,20 @@ type IngressConfig struct {
 type MicroshiftConfig struct {
 	LogVLevel int `json:"logVLevel"`
 
-	SubjectAltNames []string      `json:"subjectAltNames"`
-	NodeName        string        `json:"nodeName"`
-	NodeIP          string        `json:"nodeIP"`
-	BaseDomain      string        `json:"baseDomain"`
-	Cluster         ClusterConfig `json:"cluster"`
+	SubjectAltNames []string `json:"subjectAltNames"`
+	NodeName        string   `json:"nodeName"`
+	NodeIP          string   `json:"nodeIP"`
+	// Kube apiserver advertise address to work around the certificates issue
+	// when requiring external access using the node IP. This will turn into
+	// the IP configured in the endpoint slice for kubernetes service. Must be
+	// a reachable IP from pods. Defaults to service network CIDR first
+	// address.
+	KASAdvertiseAddress string `json:"kasAdvertiseAddress"`
+	// Determines if kube-apiserver controller should configure the
+	// KASAdvertiseAddress in the loopback interface. Automatically computed.
+	SkipKASInterface bool          `json:"-"`
+	BaseDomain       string        `json:"baseDomain"`
+	Cluster          ClusterConfig `json:"cluster"`
 
 	Ingress IngressConfig `json:"-"`
 }
@@ -118,6 +127,9 @@ type DNS struct {
 type ApiServer struct {
 	// SubjectAltNames added to API server certs
 	SubjectAltNames []string `json:"subjectAltNames"`
+	// AdvertiseAddress for endpoint slices in kubernetes service. Developer
+	// only parameter, wont show in show-config commands or docs.
+	AdvertiseAddress string `json:"advertiseAddress,omitempty"`
 }
 
 type Node struct {
@@ -363,6 +375,9 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error {
 	if len(config.ApiServer.SubjectAltNames) > 0 {
 		c.SubjectAltNames = config.ApiServer.SubjectAltNames
 	}
+	if len(config.ApiServer.AdvertiseAddress) > 0 {
+		c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress
+	}
 
 	return nil
 }
@@ -425,6 +440,21 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string, flags *pflag.FlagS
 	}
 	c.Cluster.DNS = clusterDNS
 
+	// If KAS advertise address is not configured then grab it from the service
+	// CIDR automatically.
+	if len(c.KASAdvertiseAddress) == 0 {
+		// unchecked error because this was done when getting cluster DNS
+		_, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR)
+		_, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet)
+		if err != nil {
+			return fmt.Errorf("error getting apiserver IP: %v", err)
+		}
+		c.KASAdvertiseAddress = apiServerServiceIP.String()
+		c.SkipKASInterface = false
+	} else {
+		c.SkipKASInterface = true
+	}
+
 	if len(c.SubjectAltNames) > 0 {
 		// Any entry in SubjectAltNames will be included in the external access certificates.
 		// Any of the hostnames and IPs (except the node IP) listed below conflicts with
@@ -455,12 +485,6 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string, flags *pflag.FlagS
 			}
 		}
 
-		// unchecked error because this was done when getting cluster DNS
-		_, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR)
-		_, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet)
-		if err != nil {
-			return fmt.Errorf("error getting apiserver IP: %v", err)
-		}
 		if stringSliceContains(
 			c.SubjectAltNames,
 			"kubernetes",
@@ -471,7 +495,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string, flags *pflag.FlagS
 			"openshift.default",
 			"openshift.default.svc",
 			"openshift.default.svc.cluster.local",
-			apiServerServiceIP.String(),
+			c.KASAdvertiseAddress,
 		) {
 			return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs")
 		}

pkg/controllers/kube-apiserver.go

@@ -71,8 +71,9 @@ type KubeAPIServer struct {
 	verbosity      int
 	configureErr   error // todo: report configuration errors immediately
 
-	masterURL     string
-	servingCAPath string
+	masterURL        string
+	servingCAPath    string
+	advertiseAddress string
 }
 
 func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer {
@@ -84,7 +85,7 @@ func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer {
 }
 
 func (s *KubeAPIServer) Name() string           { return "kube-apiserver" }
-func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd"} }
+func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} }
 
 func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error {
 	s.verbosity = cfg.LogVLevel
@@ -112,10 +113,11 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error {
 
 	s.masterURL = cfg.Cluster.URL
 	s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir)
+	s.advertiseAddress = cfg.KASAdvertiseAddress
 
 	overrides := &kubecontrolplanev1.KubeAPIServerConfig{
 		APIServerArguments: map[string]kubecontrolplanev1.Arguments{
-			"advertise-address": {cfg.NodeIP},
+			"advertise-address": {s.advertiseAddress},
 			"audit-policy-file": {microshiftDataDir + "/resources/kube-apiserver-audit-policies/default.yaml"},
 			"client-ca-file":    {clientCABundlePath},
 			"etcd-cafile":       {cryptomaterial.CACertPath(cryptomaterial.EtcdSignerDir(certsDir))},

pkg/node/netconfig.go

@@ -0,0 +1,120 @@
+/*
+Copyright © 2023 MicroShift Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package node
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	"github.com/vishvananda/netlink"
+
+	"github.com/openshift/microshift/pkg/config"
+)
+
+const (
+	// Network configuration component name
+	componentNetworkConfiguration = "network-configuration"
+	// Interface name where to add service IP
+	loopbackInterface = "lo"
+)
+
+type NetworkConfiguration struct {
+	kasAdvertiseAddress        string
+	skipInterfaceConfiguration bool
+}
+
+func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration {
+	n := &NetworkConfiguration{}
+	n.configure(cfg)
+	return n
+}
+
+func (n *NetworkConfiguration) Name() string           { return componentNetworkConfiguration }
+func (n *NetworkConfiguration) Dependencies() []string { return []string{} }
+
+func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) {
+	n.kasAdvertiseAddress = cfg.KASAdvertiseAddress
+	n.skipInterfaceConfiguration = cfg.SkipKASInterface
+}
+
+func (n *NetworkConfiguration) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error {
+	defer close(stopped)
+
+	stopChan := make(chan struct{})
+
+	if !n.skipInterfaceConfiguration {
+		if err := n.addServiceIPLoopback(); err != nil {
+			return err
+		}
+		go func() {
+			select {
+			case <-ctx.Done():
+				if err := n.removeServiceIPLoopback(); err != nil {
+					klog.Warningf("failed to remove IP from interface: %v", err)
+				}
+				close(stopChan)
+			}
+		}()
+	}
+	klog.Infof("%q is ready", n.Name())
+	close(ready)
+	<-stopChan
+	return ctx.Err()
+}
+
+func (n *NetworkConfiguration) addServiceIPLoopback() error {
+	link, err := netlink.LinkByName(loopbackInterface)
+	if err != nil {
+		return err
+	}
+	address, err := netlink.ParseAddr(fmt.Sprintf("%s/32", n.kasAdvertiseAddress))
+	if err != nil {
+		return err
+	}
+	existing, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return err
+	}
+	for _, existingAddress := range existing {
+		if address.Equal(existingAddress) {
+			return nil
+		}
+	}
+	return netlink.AddrAdd(link, address)
+}
+
+func (n *NetworkConfiguration) removeServiceIPLoopback() error {
+	link, err := netlink.LinkByName(loopbackInterface)
+	if err != nil {
+		return err
+	}
+	address, err := netlink.ParseAddr(fmt.Sprintf("%s/32", n.kasAdvertiseAddress))
+	if err != nil {
+		return err
+	}
+	existing, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return err
+	}
+	for _, existingAddress := range existing {
+		if address.Equal(existingAddress) {
+			return netlink.AddrDel(link, address)
+		}
+	}
+	return nil
+}