apache · DaanHoogland · Jul 2, 2021 · Jun 30, 2021
diff --git a/...cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/...cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
@@ -142,8 +142,8 @@ protected void init() {
     private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
         String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
         final String sshPubKey = "{{ k8s.ssh.pub.key }}";
-        final String joinIpKey = "{{ k8s_master.join_ip }}";
-        final String clusterTokenKey = "{{ k8s_master.cluster.token }}";
+        final String joinIpKey = "{{ k8s_control_node.join_ip }}";
+        final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
         final String ejectIsoKey = "{{ k8s.eject.iso }}";
         String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
         String sshKeyPair = kubernetesCluster.getKeyPair();

diff --git a/...rc/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/...rc/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
@@ -89,8 +89,8 @@ public KubernetesSupportedVersion getKubernetesClusterVersion() {
         return kubernetesClusterVersion;
     }
 
-    private Pair<String, Map<Long, Network.IpAddresses>> getKubernetesControlIpAddresses(final DataCenter zone, final Network network, final Account account) throws InsufficientAddressCapacityException {
-        String controlIp = null;
+    private Pair<String, Map<Long, Network.IpAddresses>> getKubernetesControlNodeIpAddresses(final DataCenter zone, final Network network, final Account account) throws InsufficientAddressCapacityException {
+        String controlNodeIp = null;
         Map<Long, Network.IpAddresses> requestedIps = null;
         if (Network.GuestType.Shared.equals(network.getGuestType())) {
             List<Long> vlanIds = new ArrayList<>();
@@ -100,16 +100,16 @@ private Pair<String, Map<Long, Network.IpAddresses>> getKubernetesControlIpAddre
             }
             PublicIp ip = ipAddressManager.getAvailablePublicIpAddressFromVlans(zone.getId(), null, account, Vlan.VlanType.DirectAttached, vlanIds,network.getId(), null, false);
             if (ip != null) {
-                controlIp = ip.getAddress().toString();
+                controlNodeIp = ip.getAddress().toString();
             }
             requestedIps = new HashMap<>();
             Ip ipAddress = ip.getAddress();
             boolean isIp6 = ipAddress.isIp6();
             requestedIps.put(network.getId(), new Network.IpAddresses(ipAddress.isIp4() ? ip.getAddress().addr() : null, null));
         } else {
-            controlIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null);
+            controlNodeIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null);
         }
-        return new Pair<>(controlIp, requestedIps);
+        return new Pair<>(controlNodeIp, requestedIps);
     }
 
     private boolean isKubernetesVersionSupportsHA() {
@@ -127,20 +127,20 @@ private boolean isKubernetesVersionSupportsHA() {
         return haSupported;
     }
 
-    private String getKubernetesControlConfig(final String controlIp, final String serverIp,
-                                              final String hostName, final boolean haSupported,
-                                              final boolean ejectIso) throws IOException {
-        String k8sControlConfig = readResourceFile("/conf/k8s-control-node.yml");
-        final String apiServerCert = "{{ k8s_master.apiserver.crt }}";
-        final String apiServerKey = "{{ k8s_master.apiserver.key }}";
-        final String caCert = "{{ k8s_master.ca.crt }}";
+    private String getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp,
+                                                  final String hostName, final boolean haSupported,
+                                                  final boolean ejectIso) throws IOException {
+        String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml");
+        final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}";
+        final String apiServerKey = "{{ k8s_control_node.apiserver.key }}";
+        final String caCert = "{{ k8s_control_node.ca.crt }}";
         final String sshPubKey = "{{ k8s.ssh.pub.key }}";
-        final String clusterToken = "{{ k8s_master.cluster.token }}";
-        final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}";
+        final String clusterToken = "{{ k8s_control_node.cluster.token }}";
+        final String clusterInitArgsKey = "{{ k8s_control_node.cluster.initargs }}";
         final String ejectIsoKey = "{{ k8s.eject.iso }}";
         final List<String> addresses = new ArrayList<>();
-        addresses.add(controlIp);
-        if (!serverIp.equals(controlIp)) {
+        addresses.add(controlNodeIp);
+        if (!serverIp.equals(controlNodeIp)) {
             addresses.add(serverIp);
         }
         final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes",
@@ -149,9 +149,9 @@ private String getKubernetesControlConfig(final String controlIp, final String s
         final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate());
         final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey());
         final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates());
-        k8sControlConfig = k8sControlConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n      "));
-        k8sControlConfig = k8sControlConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n      "));
-        k8sControlConfig = k8sControlConfig.replace(caCert, tlsCaCert.replace("\n", "\n      "));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n      "));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n      "));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n      "));
         String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
         String sshKeyPair = kubernetesCluster.getKeyPair();
         if (!Strings.isNullOrEmpty(sshKeyPair)) {
@@ -160,8 +160,8 @@ private String getKubernetesControlConfig(final String controlIp, final String s
                 pubKey += "\n  - \"" + sshkp.getPublicKey() + "\"";
             }
         }
-        k8sControlConfig = k8sControlConfig.replace(sshPubKey, pubKey);
-        k8sControlConfig = k8sControlConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey);
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
         String initArgs = "";
         if (haSupported) {
             initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ",
@@ -171,9 +171,9 @@ private String getKubernetesControlConfig(final String controlIp, final String s
         }
         initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp);
         initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion());
-        k8sControlConfig = k8sControlConfig.replace(clusterInitArgsKey, initArgs);
-        k8sControlConfig = k8sControlConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
-        return k8sControlConfig;
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs);
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
+        return k8sControlNodeConfig;
     }
 
     private UserVm createKubernetesControlNode(final Network network, String serverIp) throws ManagementServerException,
@@ -183,13 +183,13 @@ private UserVm createKubernetesControlNode(final Network network, String serverI
         ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
         List<Long> networkIds = new ArrayList<Long>();
         networkIds.add(kubernetesCluster.getNetworkId());
-        Pair<String, Map<Long, Network.IpAddresses>> ipAddresses = getKubernetesControlIpAddresses(zone, network, owner);
-        String controlIp = ipAddresses.first();
+        Pair<String, Map<Long, Network.IpAddresses>> ipAddresses = getKubernetesControlNodeIpAddresses(zone, network, owner);
+        String controlNodeIp = ipAddresses.first();
         Map<Long, Network.IpAddresses> requestedIps = ipAddresses.second();
         if (Network.GuestType.Shared.equals(network.getGuestType()) && Strings.isNullOrEmpty(serverIp)) {
-            serverIp = controlIp;
+            serverIp = controlNodeIp;
         }
-        Network.IpAddresses addrs = new Network.IpAddresses(controlIp, null);
+        Network.IpAddresses addrs = new Network.IpAddresses(controlNodeIp, null);
         long rootDiskSize = kubernetesCluster.getNodeRootDiskSize();
         Map<String, String> customParameterMap = new HashMap<String, String>();
         if (rootDiskSize > 0) {
@@ -201,13 +201,13 @@ private UserVm createKubernetesControlNode(final Network network, String serverI
         }
         hostName = getKubernetesClusterNodeAvailableName(hostName);
         boolean haSupported = isKubernetesVersionSupportsHA();
-        String k8sControlConfig = null;
+        String k8sControlNodeConfig = null;
         try {
-            k8sControlConfig = getKubernetesControlConfig(controlIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
+            k8sControlNodeConfig = getKubernetesControlNodeConfig(controlNodeIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
         } catch (IOException e) {
-            logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e);
+            logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e);
         }
-        String base64UserData = Base64.encodeBase64String(k8sControlConfig.getBytes(StringUtils.getPreferredCharset()));
+        String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(StringUtils.getPreferredCharset()));
         controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner,
                 hostName, hostName, null, null, null,
                 Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(),
@@ -218,12 +218,12 @@ private UserVm createKubernetesControlNode(final Network network, String serverI
         return controlVm;
     }
 
-    private String getKubernetesAdditionalControlConfig(final String joinIp, final boolean ejectIso) throws IOException {
-        String k8sControlConfig = readResourceFile("/conf/k8s-control-node-add.yml");
-        final String joinIpKey = "{{ k8s_master.join_ip }}";
-        final String clusterTokenKey = "{{ k8s_master.cluster.token }}";
+    private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
+        String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml");
+        final String joinIpKey = "{{ k8s_control_node.join_ip }}";
+        final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
         final String sshPubKey = "{{ k8s.ssh.pub.key }}";
-        final String clusterHACertificateKey = "{{ k8s_master.cluster.ha.certificate.key }}";
+        final String clusterHACertificateKey = "{{ k8s_control_node.cluster.ha.certificate.key }}";
         final String ejectIsoKey = "{{ k8s.eject.iso }}";
         String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
         String sshKeyPair = kubernetesCluster.getKeyPair();
@@ -233,12 +233,12 @@ private String getKubernetesAdditionalControlConfig(final String joinIp, final b
                 pubKey += "\n  - \"" + sshkp.getPublicKey() + "\"";
             }
         }
-        k8sControlConfig = k8sControlConfig.replace(sshPubKey, pubKey);
-        k8sControlConfig = k8sControlConfig.replace(joinIpKey, joinIp);
-        k8sControlConfig = k8sControlConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
-        k8sControlConfig = k8sControlConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
-        k8sControlConfig = k8sControlConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
-        return k8sControlConfig;
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey);
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpKey, joinIp);
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
+        return k8sControlNodeConfig;
     }
 
     private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance) throws ManagementServerException,
@@ -255,13 +255,13 @@ private UserVm createKubernetesAdditionalControlNode(final String joinIp, final
             customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize));
         }
         String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-control-%d", kubernetesClusterNodeNamePrefix, additionalControlNodeInstance + 1));
-        String k8sControlConfig = null;
+        String k8sControlNodeConfig = null;
         try {
-            k8sControlConfig = getKubernetesAdditionalControlConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
+            k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
         } catch (IOException e) {
             logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e);
         }
-        String base64UserData = Base64.encodeBase64String(k8sControlConfig.getBytes(StringUtils.getPreferredCharset()));
+        String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(StringUtils.getPreferredCharset()));
         additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner,
                 hostName, hostName, null, null, null,
                 Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(),

diff --git a/...s-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/...s-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java
@@ -101,9 +101,9 @@ public class KubernetesClusterResponse extends BaseResponse implements Controlle
     @Param(description = "keypair details")
     private String keypair;
 
-    @Deprecated
+    @Deprecated(since = "4.16")
     @SerializedName(ApiConstants.MASTER_NODES)
-    @Param(description = "the master nodes count for the Kubernetes cluster")
+    @Param(description = "the master nodes count for the Kubernetes cluster. This parameter is deprecated, please use 'controlnodes' parameter.")
     private Long masterNodes;
 
     @SerializedName(ApiConstants.CONTROL_NODES)

diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
@@ -196,7 +196,7 @@ write-files:
       if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
         export PATH=$PATH:/opt/bin
       fi
-      kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --control-plane --certificate-key {{ k8s_master.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification
+      kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification
 
       sudo touch /home/core/success
       echo "true" > /home/core/success
@@ -229,7 +229,7 @@ coreos:
         Type=simple
         StartLimitInterval=0
         Restart=on-failure
-        ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version
+        ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version
         ExecStart=/opt/bin/deploy-kube-system
 
   update:

diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
@@ -29,17 +29,17 @@ write-files:
   - path: /etc/kubernetes/pki/cloudstack/ca.crt
     permissions: '0644'
     content: |
-      {{ k8s_master.ca.crt }}
+      {{ k8s_control_node.ca.crt }}
 
   - path: /etc/kubernetes/pki/cloudstack/apiserver.crt
     permissions: '0644'
     content: |
-      {{ k8s_master.apiserver.crt }}
+      {{ k8s_control_node.apiserver.crt }}
 
   - path: /etc/kubernetes/pki/cloudstack/apiserver.key
     permissions: '0600'
     content: |
-      {{ k8s_master.apiserver.key }}
+      {{ k8s_control_node.apiserver.key }}
 
   - path: /opt/bin/setup-kube-system
     permissions: 0700
@@ -204,7 +204,7 @@ write-files:
         fi
         retval=0
         set +e
-        kubeadm init --token {{ k8s_master.cluster.token }} --token-ttl 0 {{ k8s_master.cluster.initargs }}
+        kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }}
         retval=$?
         set -e
         if [ $retval -eq 0 ]; then

diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
@@ -196,7 +196,7 @@ write-files:
       if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
         export PATH=$PATH:/opt/bin
       fi
-      kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification
+      kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification
 
       sudo touch /home/core/success
       echo "true" > /home/core/success
@@ -229,7 +229,7 @@ coreos:
         Type=simple
         StartLimitInterval=0
         Restart=on-failure
-        ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version
+        ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version
         ExecStart=/opt/bin/deploy-kube-system
 
   update:

diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js
@@ -430,7 +430,7 @@ export default {
         fields.push('zonename')
         return fields
       },
-      details: ['name', 'description', 'zonename', 'kubernetesversionname', 'size', 'masternodes', 'cpunumber', 'memory', 'keypair', 'associatednetworkname', 'account', 'domain', 'zonename'],
+      details: ['name', 'description', 'zonename', 'kubernetesversionname', 'size', 'controlnodes', 'cpunumber', 'memory', 'keypair', 'associatednetworkname', 'account', 'domain', 'zonename'],
       tabs: [{
         name: 'k8s',
         component: () => import('@/views/compute/KubernetesServiceTab.vue')