diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 4e3733bb5a49..98b1618a8da8 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -121,4 +121,6 @@ UserAccount createUserAccount(String userName, String password, String firstName UserAccount getUserAccountById(Long userId); public Map getKeys(GetUserKeysCmd cmd); + + public Map getKeys(Long userId); } diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 56a6dfd25a27..d9a1eb86be7a 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -379,7 +379,7 @@ UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffe String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, String sshKeyPair, Map requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, - Map templateOvfPropertiesMap) + Map templateOvfPropertiesMap, String type) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 0087fee33402..ba797196fdc3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -818,10 +818,14 @@ public class ApiConstants { public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; public static final String MASTER_NODES = "masternodes"; + public static final String NODE_IDS = "nodeids"; public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; public static final String SUPPORTS_HA = "supportsha"; + public static final String AUTOSCALING_ENABLED = "autoscalingenabled"; + public static final String MIN_SIZE = "minsize"; + public static final String MAX_SIZE = "maxsize"; public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 8e9ec450b1ed..2c5f6b94cc92 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1674,8 +1674,10 @@ protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachinePr guru.finalizeStop(profile, answer); if (vm.getType() == VirtualMachine.Type.User) { final UserVmVO userVm = _userVmDao.findById(vm.getId()); - userVm.setPowerState(PowerState.PowerOff); - _userVmDao.update(userVm.getId(), userVm); + if (userVm != null) { + userVm.setPowerState(PowerState.PowerOff); + _userVmDao.update(userVm.getId(), userVm); + } } } else { s_logger.error("Invalid answer received in response to a StopCommand for " + vm.getInstanceName()); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 05732e584113..4279331f2b9a 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -824,7 +824,7 @@ public List allocateTemplatedVolumes(Type type, String name, DiskOf int volumesNumber = 1; List templateAsIsDisks = null; String configurationId = null; - if (template.isDeployAsIs()) { + if (template.isDeployAsIs() && vm.getType() != VirtualMachine.Type.SecondaryStorageVm) { UserVmDetailVO configurationDetail = userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); if (configurationDetail != null) { configurationId = configurationDetail.getValue(); @@ -849,7 +849,7 @@ public List allocateTemplatedVolumes(Type type, String name, DiskOf String volumeName = name; Long volumeSize = rootDisksize; long deviceId = type.equals(Type.ROOT) ? 0L : 1L; - if (template.isDeployAsIs()) { + if (template.isDeployAsIs() && vm.getType() != VirtualMachine.Type.SecondaryStorageVm) { int volumeNameSuffix = templateAsIsDisks.get(number).getDiskNumber(); volumeName = String.format("%s-%d", volumeName, volumeNameSuffix); volumeSize = templateAsIsDisks.get(number).getVirtualSize(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41400to41500.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41400to41500.java index e3fe6022331c..204125cbac82 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41400to41500.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41400to41500.java @@ -140,9 +140,9 @@ private void updateSystemVmTemplates(final Connection conn) { final Map newTemplateChecksum = new HashMap() { { - put(Hypervisor.HypervisorType.KVM, "81b3e48bb934784a13555a43c5ef5ffb"); - put(Hypervisor.HypervisorType.XenServer, "1b178a5dbdbe090555515340144c6017"); - put(Hypervisor.HypervisorType.VMware, "e6a88e518c57d6f36c096c4204c3417f"); + put(Hypervisor.HypervisorType.KVM, "0d95bb3d9385097dec8b485e46eae34b"); + put(Hypervisor.HypervisorType.XenServer, "8dd2df4eba815711e68cf204a8df7650"); + put(Hypervisor.HypervisorType.VMware, "6d18a4e41589fa32217adb0d0be3d286"); put(Hypervisor.HypervisorType.Hyperv, "5c94da45337cf3e1910dcbe084d4b9ad"); put(Hypervisor.HypervisorType.LXC, "81b3e48bb934784a13555a43c5ef5ffb"); put(Hypervisor.HypervisorType.Ovm3, "875c5c65455fc06c4a012394410db375"); diff --git a/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java b/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java index e3950340469a..5d9399d7b2a5 100644 --- a/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java @@ -48,6 +48,9 @@ public class UserVmVO extends VMInstanceVO implements UserVm { @Column(name = "update_parameters", updatable = true) protected boolean updateParameters = true; + @Column(name = "user_vm_type", updatable = true) + private String userVmType; + transient String password; @Override @@ -125,4 +128,12 @@ public void setUpdateParameters(boolean updateParameters) { public boolean isUpdateParameters() { return updateParameters; } + + public String getUserVmType() { + return userVmType; + } + + public void setUserVmType(String userVmType) { + this.userVmType = userVmType; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index e87ac077ce68..915b56d4a84a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -833,3 +833,15 @@ INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_vers -- Fix OS category for Guest OS 'Other PV Virtio-SCSI (64-bit)' UPDATE `cloud`.`guest_os` SET category_id = 7 WHERE id = 275 AND display_name = 'Other PV Virtio-SCSI (64-bit)'; + +-- TODO : Move to 4.16 +ALTER TABLE `cloud`.`user_vm` ADD COLUMN `user_vm_type` varchar(255) DEFAULT "UserVM" COMMENT 'Defines the type of UserVM'; + +UPDATE `cloud`.`vm_template` set deploy_as_is = 1 where id = 8; + +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `autoscaling_enabled` tinyint(1) unsigned NOT NULL DEFAULT 0; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `minsize` bigint; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `maxsize` bigint; + +ALTER TABLE `cloud`.`kubernetes_cluster_vm_map` ADD COLUMN `is_master` tinyint(1) unsigned NOT NULL DEFAULT 0; + diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index dbb9571cea31..54ffe17f68c6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -88,14 +88,7 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource libvirtComputingResource.applyDefaultNetworkRules(conn, vmSpec, false); // pass cmdline info to system vms - if (vmSpec.getType() != VirtualMachine.Type.User) { - String controlIp = null; - for (final NicTO nic : vmSpec.getNics()) { - if (nic.getType() == TrafficType.Control) { - controlIp = nic.getIp(); - break; - } - } + if (vmSpec.getType() != VirtualMachine.Type.User || (vmSpec.getBootArgs() != null && vmSpec.getBootArgs().contains("CKSNode"))) { // try to patch and SSH into the systemvm for up to 5 minutes for (int count = 0; count < 10; count++) { // wait and try passCmdLine for 30 seconds at most for CLOUDSTACK-2823 @@ -104,12 +97,22 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource } } - final VirtualRoutingResource virtRouterResource = libvirtComputingResource.getVirtRouterResource(); - // check if the router is up? - for (int count = 0; count < 60; count++) { - final boolean result = virtRouterResource.connect(controlIp, 1, 5000); - if (result) { - break; + if (vmSpec.getType() != VirtualMachine.Type.User) { + String controlIp = null; + for (final NicTO nic : vmSpec.getNics()) { + if (nic.getType() == TrafficType.Control) { + controlIp = nic.getIp(); + break; + } + } + + final VirtualRoutingResource virtRouterResource = libvirtComputingResource.getVirtRouterResource(); + // check if the router is up? + for (int count = 0; count < 60; count++) { + final boolean result = virtRouterResource.connect(controlIp, 1, 5000); + if (result) { + break; + } } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index aef304a03d36..a06424d283b2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -37,6 +37,7 @@ enum Event { StopRequested, DestroyRequested, RecoveryRequested, + AutoscaleRequested, ScaleUpRequested, ScaleDownRequested, UpgradeRequested, @@ -81,6 +82,7 @@ enum State { s_fsm.addTransition(State.Running, Event.FaultsDetected, State.Alert); + s_fsm.addTransition(State.Running, Event.AutoscaleRequested, State.Scaling); s_fsm.addTransition(State.Running, Event.ScaleUpRequested, State.Scaling); s_fsm.addTransition(State.Running, Event.ScaleDownRequested, State.Scaling); s_fsm.addTransition(State.Scaling, Event.OperationSucceeded, State.Running); @@ -131,4 +133,7 @@ enum State { @Override State getState(); Date getCreated(); + boolean getAutoscalingEnabled(); + Long getMinSize(); + Long getMaxSize(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 8b8be1cf4032..bc3ce6e88a80 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -53,6 +54,7 @@ import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; @@ -134,7 +136,11 @@ import com.cloud.user.AccountManager; import com.cloud.user.AccountService; import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.user.UserVO; import com.cloud.user.dao.SSHKeyPairDao; +import com.cloud.user.dao.UserDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; @@ -194,6 +200,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected TemplateJoinDao templateJoinDao; @Inject + protected UserDao userDao; + @Inject protected AccountService accountService; @Inject protected AccountManager accountManager; @@ -261,60 +269,6 @@ private void logAndThrow(final Level logLevel, final String message, final Excep logTransitStateAndThrow(logLevel, message, null, null, ex); } - private boolean isKubernetesServiceTemplateConfigured(DataCenter zone) { - // Check Kubernetes VM template for zone - boolean isHyperVAvailable = false; - boolean isKVMAvailable = false; - boolean isVMwareAvailable = false; - boolean isXenserverAvailable = false; - List clusters = clusterDao.listByZoneId(zone.getId()); - for (ClusterVO clusterVO : clusters) { - if (Hypervisor.HypervisorType.Hyperv.equals(clusterVO.getHypervisorType())) { - isHyperVAvailable = true; - } - if (Hypervisor.HypervisorType.KVM.equals(clusterVO.getHypervisorType())) { - isKVMAvailable = true; - } - if (Hypervisor.HypervisorType.VMware.equals(clusterVO.getHypervisorType())) { - isVMwareAvailable = true; - } - if (Hypervisor.HypervisorType.XenServer.equals(clusterVO.getHypervisorType())) { - isXenserverAvailable = true; - } - } - List> templatePairs = new ArrayList<>(); - if (isHyperVAvailable) { - templatePairs.add(new Pair<>(KubernetesClusterHyperVTemplateName.key(), KubernetesClusterHyperVTemplateName.value())); - } - if (isKVMAvailable) { - templatePairs.add(new Pair<>(KubernetesClusterKVMTemplateName.key(), KubernetesClusterKVMTemplateName.value())); - } - if (isVMwareAvailable) { - templatePairs.add(new Pair<>(KubernetesClusterVMwareTemplateName.key(), KubernetesClusterVMwareTemplateName.value())); - } - if (isXenserverAvailable) { - templatePairs.add(new Pair<>(KubernetesClusterXenserverTemplateName.key(), KubernetesClusterXenserverTemplateName.value())); - } - for (Pair templatePair : templatePairs) { - String templateKey = templatePair.first(); - String templateName = templatePair.second(); - if (Strings.isNullOrEmpty(templateName)) { - LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", templateKey)); - return false; - } - final VMTemplateVO template = templateDao.findValidByTemplateName(templateName); - if (template == null) { - LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster nodes", templateName)); - return false; - } - if (CollectionUtils.isEmpty(templateJoinDao.newTemplateView(template, zone.getId(), true))) { - LOGGER.warn(String.format("The template ID: %s, name: %s is not available for use in zone ID: %s provisioning Kubernetes cluster nodes", template.getUuid(), templateName, zone.getUuid())); - return false; - } - } - return true; - } - private boolean isKubernetesServiceNetworkOfferingConfigured(DataCenter zone) { // Check network offering String networkOfferingName = KubernetesClusterNetworkOffering.value(); @@ -362,9 +316,6 @@ private boolean isKubernetesServiceNetworkOfferingConfigured(DataCenter zone) { } private boolean isKubernetesServiceConfigured(DataCenter zone) { - if (!isKubernetesServiceTemplateConfigured(zone)) { - return false; - } if (!isKubernetesServiceNetworkOfferingConfigured(zone)) { return false; } @@ -384,23 +335,12 @@ private IpAddress getSourceNatIp(Network network) { return null; } - private VMTemplateVO getKubernetesServiceTemplate(Hypervisor.HypervisorType hypervisorType) { - String templateName = null; - switch (hypervisorType) { - case Hyperv: - templateName = KubernetesClusterHyperVTemplateName.value(); - break; - case KVM: - templateName = KubernetesClusterKVMTemplateName.value(); - break; - case VMware: - templateName = KubernetesClusterVMwareTemplateName.value(); - break; - case XenServer: - templateName = KubernetesClusterXenserverTemplateName.value(); - break; + private VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) { + VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType); + if (template == null) { + throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenter.getId()); } - return templateDao.findValidByTemplateName(templateName); + return template; } private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) { @@ -470,7 +410,7 @@ private boolean validateServiceOffering(final ServiceOffering serviceOffering, f throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); } if (serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", serviceOffering.getUuid(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); + throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template needs minimum %d vCPUs and %d MB RAM", serviceOffering.getUuid(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); } if (serviceOffering.getCpu() < version.getMinimumCpu()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes version ID: %s needs minimum %d vCPUs", serviceOffering.getUuid(), version.getUuid(), version.getMinimumCpu())); @@ -621,6 +561,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes response.setIpAddressId(ipAddresses.get(0).getUuid()); } } + List vmResponses = new ArrayList(); List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); ResponseView respView = ResponseView.Restricted; @@ -640,10 +581,22 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes } } response.setVirtualMachines(vmResponses); + response.setAutoscalingEnabled(kubernetesCluster.getAutoscalingEnabled()); + response.setMinSize(kubernetesCluster.getMinSize()); + response.setMaxSize(kubernetesCluster.getMaxSize()); return response; } + private void validateEndpointUrl() { + String csUrl = ApiServiceConfiguration.ApiServletPath.value(); + if (csUrl == null || csUrl.contains("localhost")) { + throw new InvalidParameterValueException("Global setting endpointe.url has to be set to the Management Server's API end point"); + } + } + private void validateKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws CloudRuntimeException { + validateEndpointUrl(); + final String name = cmd.getName(); final Long zoneId = cmd.getZoneId(); final Long kubernetesVersionId = cmd.getKubernetesVersionId(); @@ -653,6 +606,7 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu final String sshKeyPair = cmd.getSSHKeyPairName(); final Long masterNodeCount = cmd.getMasterNodes(); final Long clusterSize = cmd.getClusterSize(); + final long totalNodeCount = masterNodeCount + clusterSize; final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); @@ -664,14 +618,20 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name:" + name); } - if (masterNodeCount < 1 || masterNodeCount > 100) { + if (masterNodeCount < 1) { throw new InvalidParameterValueException("Invalid cluster master nodes count: " + masterNodeCount); } - if (clusterSize < 1 || clusterSize > 100) { + if (clusterSize < 1) { throw new InvalidParameterValueException("Invalid cluster size: " + clusterSize); } + int maxClusterSize = KubernetesMaxClusterSize.valueIn(owner.getId()); + if (totalNodeCount > maxClusterSize) { + throw new InvalidParameterValueException( + String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize)); + } + DataCenter zone = dataCenterDao.findById(zoneId); if (zone == null) { throw new InvalidParameterValueException("Unable to find zone by ID: " + zoneId); @@ -845,30 +805,92 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd final Long kubernetesClusterId = cmd.getId(); final Long serviceOfferingId = cmd.getServiceOfferingId(); final Long clusterSize = cmd.getClusterSize(); + final List nodeIds = cmd.getNodeIds(); + final Boolean isAutoscalingEnabled = cmd.isAutoscalingEnabled(); + final Long minSize = cmd.getMinSize(); + final Long maxSize = cmd.getMaxSize(); + if (kubernetesClusterId == null || kubernetesClusterId < 1L) { throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); } + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); if (zone == null) { logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster : %s", kubernetesCluster.getName())); } + if (serviceOfferingId == null && clusterSize == null && nodeIds == null && isAutoscalingEnabled == null) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s cannot be scaled, either service offering or cluster size or nodeids to be removed or autoscaling must be passed", kubernetesCluster.getName())); + } + Account caller = CallContext.current().getCallingAccount(); accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); - if (serviceOfferingId == null && clusterSize == null) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getName())); - } - final KubernetesSupportedVersion clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (clusterVersion == null) { throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster : %s", kubernetesCluster.getName())); } + if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and can not be scaled", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + + if (isAutoscalingEnabled != null && isAutoscalingEnabled) { + if (clusterSize != null || serviceOfferingId != null || nodeIds != null) { + throw new InvalidParameterValueException("autoscaling can not be passed along with nodeids or clustersize or service offering"); + } + + if (!KubernetesVersionManagerImpl.versionSupportsAutoscaling(clusterVersion)) { + throw new InvalidParameterValueException(String.format("Autoscaling requires Kubernetes Version %s or above", + KubernetesVersionManagerImpl.MINIMUN_AUTOSCALER_SUPPORTED_VERSION )); + } + + validateEndpointUrl(); + + if (minSize == null || maxSize == null) { + throw new InvalidParameterValueException("autoscaling requires minsize and maxsize to be passed"); + } + if (minSize < 1) { + throw new InvalidParameterValueException("minsize must be at least than 1"); + } + if (maxSize <= minSize) { + throw new InvalidParameterValueException("maxsize must be greater than or equal to minsize"); + } + int maxClusterSize = KubernetesMaxClusterSize.valueIn(kubernetesCluster.getAccountId()); + if (maxSize + kubernetesCluster.getMasterNodeCount() > maxClusterSize) { + throw new InvalidParameterValueException( + String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize)); + } + } + + if (nodeIds != null) { + if (clusterSize != null || serviceOfferingId != null) { + throw new InvalidParameterValueException("nodeids can not be passed along with clustersize or service offering"); + } + List nodes = kubernetesClusterVmMapDao.listByClusterIdAndVmIdsIn(kubernetesCluster.getId(), nodeIds); + // Do all the nodes exist ? + if (nodes == null || nodes.size() != nodeIds.size()) { + throw new InvalidParameterValueException("Invalid node ids"); + } + // Ensure there's always a master + long mastersToRemove = nodes.stream().filter(x -> x.isMaster()).count(); + if (mastersToRemove >= kubernetesCluster.getMasterNodeCount()) { + throw new InvalidParameterValueException("Can not remove all masters from a cluster"); + } + // Ensure there's always a node + long nodesToRemove = nodes.stream().filter(x -> !x.isMaster()).count(); + if (nodesToRemove >= kubernetesCluster.getNodeCount()) { + throw new InvalidParameterValueException("Can not remove all nodes from a cluster"); + } + } + ServiceOffering serviceOffering = null; if (serviceOfferingId != null) { serviceOffering = serviceOfferingDao.findById(serviceOfferingId); @@ -899,12 +921,6 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd } } - if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || - kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || - kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { - throw new PermissionDeniedException(String.format("Kubernetes cluster : %s is in %s state", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); - } - if (clusterSize != null) { if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size throw new PermissionDeniedException(String.format("Kubernetes cluster : %s is in %s state", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); @@ -925,6 +941,8 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd } private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { + validateEndpointUrl(); + // Validate parameters final Long kubernetesClusterId = cmd.getId(); final Long upgradeVersionId = cmd.getKubernetesVersionId(); @@ -955,8 +973,8 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster } KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (clusterVersion == null || clusterVersion.getRemoved() != null) { - throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", - kubernetesCluster.getUuid())); + throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster : %s", + kubernetesCluster.getName())); } final ServiceOffering serviceOffering = serviceOfferingDao.findByIdIncludingRemoved(kubernetesCluster.getServiceOfferingId()); if (serviceOffering == null) { @@ -1023,7 +1041,7 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) } final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)masterNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); - final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(deployDestination.getCluster().getHypervisorType()); + final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType()); final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); @@ -1088,10 +1106,13 @@ public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster : %s", kubernetesCluster.getName())); } KubernetesClusterStartWorker startWorker = - new KubernetesClusterStartWorker(kubernetesCluster, this); + new KubernetesClusterStartWorker(kubernetesCluster, this); startWorker = ComponentContext.inject(startWorker); if (onCreate) { // Start for Kubernetes cluster in 'Created' state + Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId()); + String[] keys = getServiceUserKeys(owner); + startWorker.setKeys(keys); return startWorker.startKubernetesClusterOnCreate(); } else { // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started @@ -1220,15 +1241,49 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC return response; } + private String[] getServiceUserKeys(Account owner) { + if (owner == null) { + owner = CallContext.current().getCallingAccount(); + } + String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME; + UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId()); + String[] keys = null; + if (kubeadmin == null) { + User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(), + KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN)); + keys = accountService.createApiKeyAndSecretKey(kube.getId()); + } else { + String apiKey = kubeadmin.getApiKey(); + String secretKey = kubeadmin.getSecretKey(); + if (Strings.isNullOrEmpty(apiKey) || Strings.isNullOrEmpty(secretKey)) { + keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId()); + } else { + keys = new String[]{apiKey, secretKey}; + } + } + return keys; + } + @Override public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws CloudRuntimeException { if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } validateKubernetesClusterScaleParameters(cmd); + + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId()); + String[] keys = getServiceUserKeys(owner); KubernetesClusterScaleWorker scaleWorker = - new KubernetesClusterScaleWorker(kubernetesClusterDao.findById(cmd.getId()), - serviceOfferingDao.findById(cmd.getServiceOfferingId()), cmd.getClusterSize(), this); + new KubernetesClusterScaleWorker(kubernetesClusterDao.findById(cmd.getId()), + serviceOfferingDao.findById(cmd.getServiceOfferingId()), + cmd.getClusterSize(), + cmd.getNodeIds(), + cmd.isAutoscalingEnabled(), + cmd.getMinSize(), + cmd.getMaxSize(), + this); + scaleWorker.setKeys(keys); scaleWorker = ComponentContext.inject(scaleWorker); return scaleWorker.scaleCluster(); } @@ -1238,10 +1293,14 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } + validateKubernetesClusterUpgradeParameters(cmd); + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId()); + String[] keys = getServiceUserKeys(owner); KubernetesClusterUpgradeWorker upgradeWorker = - new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()), - kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this); + new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()), + kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this, keys); upgradeWorker = ComponentContext.inject(upgradeWorker); return upgradeWorker.upgradeCluster(); } @@ -1445,8 +1504,8 @@ boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualM // check cluster is running at desired capacity include master nodes as well if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster ID: %s while expected %d VMs to be in state: %s", - clusterVMs.size(), kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), state.toString())); + LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster %s while expected %d VMs to be in state: %s", + clusterVMs.size(), kubernetesCluster.getName(), kubernetesCluster.getTotalNodeCount(), state.toString())); } return false; } @@ -1522,16 +1581,13 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { - KubernetesServiceEnabled, - KubernetesClusterHyperVTemplateName, - KubernetesClusterKVMTemplateName, - KubernetesClusterVMwareTemplateName, - KubernetesClusterXenserverTemplateName, - KubernetesClusterNetworkOffering, - KubernetesClusterStartTimeout, - KubernetesClusterScaleTimeout, - KubernetesClusterUpgradeTimeout, - KubernetesClusterExperimentalFeaturesEnabled + KubernetesServiceEnabled, + KubernetesClusterNetworkOffering, + KubernetesClusterStartTimeout, + KubernetesClusterScaleTimeout, + KubernetesClusterUpgradeTimeout, + KubernetesClusterExperimentalFeaturesEnabled, + KubernetesMaxClusterSize }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index db5ab91b3d11..138889a2fb37 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -34,32 +34,13 @@ public interface KubernetesClusterService extends PluggableService, Configurable static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0"; static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2; static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048; + static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin"; static final ConfigKey KubernetesServiceEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.service.enabled", "false", "Indicates whether Kubernetes Service plugin is enabled or not. Management server restart needed on change", false); - static final ConfigKey KubernetesClusterHyperVTemplateName = new ConfigKey("Advanced", String.class, - "cloud.kubernetes.cluster.template.name.hyperv", - "Kubernetes-Service-Template-HyperV", - "Name of the template to be used for creating Kubernetes cluster nodes on HyperV", - true); - static final ConfigKey KubernetesClusterKVMTemplateName = new ConfigKey("Advanced", String.class, - "cloud.kubernetes.cluster.template.name.kvm", - "Kubernetes-Service-Template-KVM", - "Name of the template to be used for creating Kubernetes cluster nodes on KVM", - true); - static final ConfigKey KubernetesClusterVMwareTemplateName = new ConfigKey("Advanced", String.class, - "cloud.kubernetes.cluster.template.name.vmware", - "Kubernetes-Service-Template-VMware", - "Name of the template to be used for creating Kubernetes cluster nodes on VMware", - true); - static final ConfigKey KubernetesClusterXenserverTemplateName = new ConfigKey("Advanced", String.class, - "cloud.kubernetes.cluster.template.name.xenserver", - "Kubernetes-Service-Template-Xenserver", - "Name of the template to be used for creating Kubernetes cluster nodes on Xenserver", - true); static final ConfigKey KubernetesClusterNetworkOffering = new ConfigKey("Advanced", String.class, "cloud.kubernetes.cluster.network.offering", "DefaultNetworkOfferingforKubernetesService", @@ -85,6 +66,12 @@ public interface KubernetesClusterService extends PluggableService, Configurable "false", "Indicates whether experimental feature for Kubernetes cluster such as Docker private registry are enabled or not", true); + static final ConfigKey KubernetesMaxClusterSize = new ConfigKey("Advanced", Integer.class, + "cloud.kubernetes.cluster.max.size", + "10", + "Maximum size of the kubernetes cluster.", + true, ConfigKey.Scope.Account); + KubernetesCluster findById(final Long id); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 9ff0be335f37..805f44d4db93 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -93,6 +93,15 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "endpoint") private String endpoint; + @Column(name = "autoscaling_enabled") + private boolean autoscalingEnabled; + + @Column(name = "minsize") + private Long minSize; + + @Column(name = "maxsize") + private Long maxSize; + @Column(name = GenericDao.CREATED_COLUMN) private Date created; @@ -303,13 +312,40 @@ public Date getCreated() { return created; } + @Override + public boolean getAutoscalingEnabled() { + return autoscalingEnabled; + } + + public void setAutoscalingEnabled(boolean enabled) { + this.autoscalingEnabled = enabled; + } + + @Override + public Long getMinSize() { + return minSize; + } + + public void setMinSize(Long minSize) { + this.minSize = minSize; + } + + @Override + public Long getMaxSize() { + return maxSize; + } + + public void setMaxSize(Long maxSize) { + this.maxSize = maxSize; + } + public KubernetesClusterVO() { this.uuid = UUID.randomUUID().toString(); } public KubernetesClusterVO(String name, String description, long zoneId, long kubernetesVersionId, long serviceOfferingId, long templateId, - long networkId, long domainId, long accountId, long masterNodeCount, long nodeCount, State state, - String keyPair, long cores, long memory, Long nodeRootDiskSize, String endpoint) { + long networkId, long domainId, long accountId, long masterNodeCount, long nodeCount, State state, String keyPair, long cores, + long memory, Long nodeRootDiskSize, String endpoint) { this.uuid = UUID.randomUUID().toString(); this.name = name; this.description = description; @@ -333,6 +369,16 @@ public KubernetesClusterVO(String name, String description, long zoneId, long ku this.checkForGc = false; } + public KubernetesClusterVO(String name, String description, long zoneId, long kubernetesVersionId, long serviceOfferingId, long templateId, + long networkId, long domainId, long accountId, long masterNodeCount, long nodeCount, State state, String keyPair, long cores, + long memory, Long nodeRootDiskSize, String endpoint, boolean autoscalingEnabled, Long minSize, Long maxSize) { + this(name, description, zoneId, kubernetesVersionId, serviceOfferingId, templateId, networkId, domainId, accountId, masterNodeCount, + nodeCount, state, keyPair, cores, memory, nodeRootDiskSize, endpoint); + this.autoscalingEnabled = autoscalingEnabled; + this.minSize = minSize; + this.maxSize = maxSize; + } + @Override public Class getEntityType() { return KubernetesCluster.class; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java index c7399202348f..b20cf0451a6d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java @@ -27,4 +27,5 @@ public interface KubernetesClusterVmMap { long getId(); long getClusterId(); long getVmId(); + boolean isMaster(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java index edb06e79534a..abbd90a949a9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java @@ -28,6 +28,30 @@ @Table(name = "kubernetes_cluster_vm_map") public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "cluster_id") + long clusterId; + + @Column(name = "vm_id") + long vmId; + + @Column(name = "is_master") + boolean master; + + public KubernetesClusterVmMapVO() { + } + + public KubernetesClusterVmMapVO(long clusterId, long vmId, boolean master) { + this.vmId = vmId; + this.clusterId = clusterId; + this.master = master; + } + + @Override public long getId() { return id; @@ -36,11 +60,9 @@ public long getId() { @Override public long getClusterId() { return clusterId; - } public void setClusterId(long clusterId) { - this.clusterId = clusterId; } @@ -50,27 +72,15 @@ public long getVmId() { } public void setVmId(long vmId) { - this.vmId = vmId; } - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "id") - long id; - - @Column(name = "cluster_id") - long clusterId; - - @Column(name = "vm_id") - long vmId; - - public KubernetesClusterVmMapVO() { - + @Override + public boolean isMaster() { + return master; } - public KubernetesClusterVmMapVO(long clusterId, long vmId) { - this.vmId = vmId; - this.clusterId = clusterId; + public void setMaster(boolean master) { + this.master = master; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index e5c811878ced..ad07a004f70e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -17,17 +17,21 @@ package com.cloud.kubernetes.cluster.actionworkers; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import javax.inject.Inject; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.commons.collections.CollectionUtils; @@ -55,6 +59,7 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; @@ -70,13 +75,15 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; +import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.UserVmService; +import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.UserVmDao; import com.google.common.base.Strings; public class KubernetesClusterActionWorker { - public static final String CLUSTER_NODE_VM_USER = "core"; + public static final String CLUSTER_NODE_VM_USER = "root"; public static final int CLUSTER_API_PORT = 6443; public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; @@ -114,6 +121,10 @@ public class KubernetesClusterActionWorker { protected UserVmService userVmService; @Inject protected VlanDao vlanDao; + @Inject + protected VirtualMachineManager itMgr; + @Inject + protected LaunchPermissionDao launchPermissionDao; protected KubernetesClusterDao kubernetesClusterDao; protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @@ -127,6 +138,13 @@ public class KubernetesClusterActionWorker { protected String publicIpAddress; protected int sshPort; + protected final String autoscaleScriptFilename = "autoscale-kube-cluster"; + protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret"; + protected File autoscaleScriptFile; + protected File deploySecretsScriptFile; + + protected String[] keys; + protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { this.kubernetesCluster = kubernetesCluster; this.kubernetesClusterDao = clusterManager.kubernetesClusterDao; @@ -178,7 +196,7 @@ protected void logMessage(final Level logLevel, final String message, final Exce } protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, - final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { logMessage(logLevel, message, e); stateTransitTo(kubernetesCluster.getId(), event); detachIsoKubernetesVMs(clusterVMs); @@ -188,11 +206,19 @@ protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final Stri throw new CloudRuntimeException(message, e); } + protected void deleteTemplateLaunchPermission() { + if (clusterTemplate != null && owner != null) { + LOGGER.info("Revoking launch permission for systemVM template"); + launchPermissionDao.removePermissions(clusterTemplate.getId(), Collections.singletonList(owner.getId())); + } + } + protected void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { logMessage(logLevel, message, e); if (kubernetesClusterId != null && event != null) { stateTransitTo(kubernetesClusterId, event); } + deleteTemplateLaunchPermission(); if (e == null) { throw new CloudRuntimeException(message); } @@ -220,11 +246,11 @@ protected File getManagementServerSshPublicKeyFile() { return new File(keyFile); } - protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId) { + protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isMaster) { return Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { - KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId); + KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId, isMaster); kubernetesClusterVmMapDao.persist(newClusterVmMap); return newClusterVmMap; } @@ -257,6 +283,13 @@ protected String getMasterVmPrivateIp() { return ip; } + private boolean containsMasterNode(List clusterVMs) { + List nodeNames = clusterVMs.stream().map(vm -> vm.getHostName()).collect(Collectors.toList()); + boolean present = false; + present = nodeNames.stream().anyMatch(s -> s.contains("master")); + return present; + } + protected Pair getKubernetesClusterServerIpSshPort(UserVm masterVm) { int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT; KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS); @@ -295,6 +328,7 @@ protected Pair getKubernetesClusterServerIpSshPort(UserVm maste } protected void attachIsoKubernetesVMs(List clusterVMs, final KubernetesSupportedVersion kubernetesSupportedVersion) throws CloudRuntimeException { + //final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000; KubernetesSupportedVersion version = kubernetesSupportedVersion; if (kubernetesSupportedVersion == null) { version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); @@ -317,6 +351,7 @@ protected void attachIsoKubernetesVMs(List clusterVMs, final KubernetesS if (!iso.getState().equals(VirtualMachineTemplate.State.Active)) { logTransitStateAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster : %s. Binaries ISO not active.", kubernetesCluster.getName()), kubernetesCluster.getId(), failedEvent); } + for (UserVm vm : clusterVMs) { try { templateService.attachIso(iso.getId(), vm.getId()); @@ -359,6 +394,10 @@ protected List getKubernetesClusterVMMaps() { return clusterVMs; } + protected List getKubernetesClusterVMMapsForNodes(List nodeIds) { + return kubernetesClusterVmMapDao.listByClusterIdAndVmIdsIn(kubernetesCluster.getId(), nodeIds); + } + protected List getKubernetesClusterVMs() { List vmList = new ArrayList<>(); List clusterVMs = getKubernetesClusterVMMaps(); @@ -380,4 +419,59 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve return false; } } + + protected boolean createCloudStackSecret(String[] keys) { + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + + try { + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + pkFile, null, String.format("sudo /opt/bin/deploy-cloudstack-secret -u '%s' -k '%s' -s '%s'", + ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]), + 10000, 10000, 60000); + return result.first(); + } catch (Exception e) { + String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName()); + LOGGER.warn(msg, e); + } + return false; + } + + protected File retrieveScriptFile(String filename) { + File file = null; + try { + String data = readResourceFile("/script/" + filename); + file = File.createTempFile(filename, ".sh"); + BufferedWriter writer = new BufferedWriter(new FileWriter(file)); + writer.write(data); + writer.close(); + } catch (IOException e) { + logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster %s, unable to prepare upgrade script %s", kubernetesCluster.getName(), filename), e); + } + return file; + } + + protected void retrieveScriptFiles() { + autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename); + deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename); + } + + protected void copyAutoscalerScripts(String nodeAddress, final int sshPort) throws Exception { + SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + "~/", autoscaleScriptFile.getAbsolutePath(), "0755"); + SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + "~/", deploySecretsScriptFile.getAbsolutePath(), "0755"); + String cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", autoscaleScriptFile.getName(), autoscaleScriptFilename); + SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + cmdStr, 10000, 10000, 10 * 60 * 1000); + cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", deploySecretsScriptFile.getName(), deploySecretsScriptFilename); + SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + cmdStr, 10000, 10000, 10 * 60 * 1000); + } + + public void setKeys(String[] keys) { + this.keys = keys; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 87b454a9bd97..d25ef59dd61a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -37,6 +37,7 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVmMap; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.network.IpAddress; +import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; import com.cloud.network.rules.FirewallRule; import com.cloud.user.Account; @@ -239,7 +240,10 @@ public boolean destroy() throws CloudRuntimeException { } } else { try { - deleteKubernetesClusterNetworkRules(); + NetworkVO kubernetesClusterNetwork = networkDao.findById(kubernetesCluster.getNetworkId()); + if (kubernetesClusterNetwork != null && kubernetesClusterNetwork.getGuestType() != Network.GuestType.Shared) { + deleteKubernetesClusterNetworkRules(); + } } catch (ManagementServerException e) { String msg = String.format("Failed to remove network rules of Kubernetes cluster : %s", kubernetesCluster.getName()); LOGGER.warn(msg, e); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index b715f09d7b88..e6805ac79744 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -17,6 +17,7 @@ package com.cloud.kubernetes.cluster.actionworkers; +import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.util.ArrayList; @@ -46,7 +47,7 @@ import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -55,6 +56,7 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; @@ -70,6 +72,7 @@ import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; +import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.user.Account; import com.cloud.user.SSHKeyPairVO; import com.cloud.uservm.UserVm; @@ -77,11 +80,13 @@ import com.cloud.utils.StringUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionStatus; -import com.cloud.utils.exception.ExecutionException; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.utils.net.NetUtils; +import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.Nic; import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; @@ -118,6 +123,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected VMInstanceDao vmInstanceDao; @Inject protected UserVmManager userVmManager; + @Inject + protected LaunchPermissionDao launchPermissionDao; protected String kubernetesClusterNodeNamePrefix; @@ -172,9 +179,9 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI if (!Strings.isNullOrEmpty(dockerUserName) && !Strings.isNullOrEmpty(dockerPassword)) { // do write file for /.docker/config.json through the code instead of k8s-node.yml as we can no make a section // optional or conditionally applied - String dockerConfigString = "write-files:\n" + + String dockerConfigString = "write_files:\n" + " - path: /.docker/config.json\n" + - " owner: core:core\n" + + " owner: root:root\n" + " permissions: '0644'\n" + " content: |\n" + " {\n" + @@ -185,7 +192,7 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI " }\n" + " }\n" + " }"; - k8sNodeConfig = k8sNodeConfig.replace("write-files:", dockerConfigString); + k8sNodeConfig = k8sNodeConfig.replace("write_files:", dockerConfigString); final String dockerUrlKey = "{{docker.url}}"; final String dockerAuthKey = "{{docker.secret}}"; final String dockerEmailKey = "{{docker.email}}"; @@ -275,12 +282,11 @@ protected void startKubernetesVM(final UserVm vm) throws ManagementServerExcepti Field f = startVm.getClass().getDeclaredField("id"); f.setAccessible(true); f.set(startVm, vm.getId()); - userVmService.startVirtualMachine(startVm); + itMgr.advanceStart(vm.getUuid(), null, null); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Started VM : %s in the Kubernetes cluster : %s", vm.getDisplayName(), kubernetesCluster.getName())); } - } catch (IllegalAccessException | NoSuchFieldException | ExecutionException | - ResourceUnavailableException | ResourceAllocationException | InsufficientCapacityException ex) { + } catch (IllegalAccessException | NoSuchFieldException | OperationTimedoutException | ResourceUnavailableException | InsufficientCapacityException ex) { throw new ManagementServerException(String.format("Failed to start VM in the Kubernetes cluster : %s", kubernetesCluster.getName()), ex); } @@ -294,8 +300,8 @@ protected List provisionKubernetesClusterNodeVms(final long nodeCount, f ResourceUnavailableException, InsufficientCapacityException { List nodes = new ArrayList<>(); for (int i = offset + 1; i <= nodeCount; i++) { - UserVm vm = createKubernetesNode(publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); + UserVm vm = createKubernetesNode(publicIpAddress); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false); startKubernetesVM(vm); vm = userVmDao.findById(vm.getId()); if (vm == null) { @@ -314,7 +320,7 @@ protected List provisionKubernetesClusterNodeVms(final long nodeCount, f return provisionKubernetesClusterNodeVms(nodeCount, 0, publicIpAddress); } - protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws ManagementServerException, + protected UserVm createKubernetesNode(String joinIp) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { UserVm nodeVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); @@ -328,7 +334,8 @@ protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws Ma if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, nodeInstance)); + String suffix = Long.toHexString(System.currentTimeMillis()); + String hostName = String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sNodeConfig = null; try { k8sNodeConfig = getKubernetesNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); @@ -339,7 +346,7 @@ protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws Ma nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), - null, addrs, null, null, null, customParameterMap, null, null, null, null); + null, addrs, null, null, null, customParameterMap, null, null, null, null, String.valueOf(UserVmManager.UserVmType.CKSNode)); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Created node VM : %s, %s in the Kubernetes cluster : %s", hostName, nodeVm.getUuid(), kubernetesCluster.getName())); } @@ -418,7 +425,6 @@ protected void provisionSshPortForwardingRules(IpAddress publicIp, Network netwo final Ip vmIp = new Ip(vmNic.getIPv4Address()); final long vmIdFinal = vmId; final int srcPortFinal = firewallRuleSourcePortStart + i; - PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { @Override public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { @@ -484,6 +490,17 @@ protected void removePortForwardingRules(final IpAddress publicIp, final Network } } + protected void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, int startPort, int endPort) + throws ResourceUnavailableException { + List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); + for (PortForwardingRuleVO pfRule : pfRules) { + if (startPort <= pfRule.getSourcePortStart() && pfRule.getSourcePortStart() <= endPort) { + portForwardingRulesDao.remove(pfRule.getId()); + } + } + rulesService.applyPortForwardingRules(publicIp.getId(), account); + } + protected void removeLoadBalancingRule(final IpAddress publicIp, final Network network, final Account account, final int port) throws ResourceUnavailableException { List rules = loadBalancerDao.listByIpAddress(publicIp.getId()); @@ -513,13 +530,96 @@ protected String getKubernetesClusterNodeNamePrefix() { return prefix; } - protected String getKubernetesClusterNodeAvailableName(final String hostName) { - String name = hostName; - int suffix = 1; - while (vmInstanceDao.findVMByHostName(name) != null) { - name = String.format("%s-%d", hostName, suffix); - suffix++; + protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, + final Long size, final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) { + return Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesCluster.getId()); + if (cores != null) { + updatedCluster.setCores(cores); + } + if (memory != null) { + updatedCluster.setMemory(memory); + } + if (size != null) { + updatedCluster.setNodeCount(size); + } + if (serviceOfferingId != null) { + updatedCluster.setServiceOfferingId(serviceOfferingId); + } + if (autoscaleEnabled != null) { + updatedCluster.setAutoscalingEnabled(autoscaleEnabled.booleanValue()); + } + updatedCluster.setMinSize(minSize); + updatedCluster.setMaxSize(maxSize); + return kubernetesClusterDao.persist(updatedCluster); + } + }); + } + + private KubernetesClusterVO updateKubernetesClusterEntry(final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) throws CloudRuntimeException { + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(null, null, null, null, autoscaleEnabled, minSize, maxSize); + if (kubernetesClusterVO == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + return kubernetesClusterVO; + } + + protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AutoscaleRequested); + } + + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + + try { + if (enable) { + String command = String.format("sudo /opt/bin/autoscale-kube-cluster -i %s -e -M %d -m %d", + kubernetesCluster.getUuid(), maxSize, minSize); + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + pkFile, null, command, 10000, 10000, 60000); + + // Maybe the file isn't present. Try and copy it + if (!result.first()) { + logMessage(Level.INFO, "Autoscaling files missing. Adding them now", null); + retrieveScriptFiles(); + copyAutoscalerScripts(publicIpAddress, sshPort); + + if (!createCloudStackSecret(keys)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + + // If at first you don't succeed ... + result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + pkFile, null, command, 10000, 10000, 60000); + if (!result.first()) { + throw new CloudRuntimeException(result.second()); + } + } + updateKubernetesClusterEntry(true, minSize, maxSize); + } else { + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + pkFile, null, String.format("sudo /opt/bin/autoscale-kube-cluster -d"), + 10000, 10000, 60000); + if (!result.first()) { + throw new CloudRuntimeException(result.second()); + } + updateKubernetesClusterEntry(false, null, null); + } + return true; + } catch (Exception e) { + String msg = String.format("Failed to autoscale Kubernetes cluster: %s : %s", kubernetesCluster.getName(), e.getMessage()); + logAndThrow(Level.ERROR, msg); + return false; + } finally { + // Deploying the autoscaler might fail but it can be deployed manually too, so no need to go to an alert state + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } - return name; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 1fce00ba81d5..8833907bc92d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -19,8 +19,10 @@ import java.io.File; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -46,10 +48,9 @@ import com.cloud.network.Network; import com.cloud.network.rules.FirewallRule; import com.cloud.offering.ServiceOffering; +import com.cloud.storage.LaunchPermissionVO; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.UserVmVO; @@ -65,18 +66,35 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private ServiceOffering serviceOffering; private Long clusterSize; + private List nodeIds; private KubernetesCluster.State originalState; private Network network; + private Long minSize; + private Long maxSize; + private Boolean isAutoscalingEnabled; private long scaleTimeoutTime; public KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final ServiceOffering serviceOffering, final Long clusterSize, + final List nodeIds, + final Boolean isAutoscalingEnabled, + final Long minSize, + final Long maxSize, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); this.serviceOffering = serviceOffering; - this.clusterSize = clusterSize; + this.nodeIds = nodeIds; + this.isAutoscalingEnabled = isAutoscalingEnabled; + this.minSize = minSize; + this.maxSize = maxSize; this.originalState = kubernetesCluster.getState(); + if (this.nodeIds != null) { + this.clusterSize = kubernetesCluster.getNodeCount() - this.nodeIds.size(); + } else { + this.clusterSize = clusterSize; + } + } protected void init() { @@ -100,13 +118,12 @@ private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final /** * Scale network rules for an existing Kubernetes cluster while scaling it * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n. - * Also remove port forwarding rules for removed virtual machines and create port-forwarding rule + * Also remove port forwarding rules for all virtual machines and re-create port-forwarding rule * to forward public IP traffic to all node VMs' private IP. * @param clusterVMIds - * @param removedVMIds * @throws ManagementServerException */ - private void scaleKubernetesClusterNetworkRules(final List clusterVMIds, final List removedVMIds) throws ManagementServerException { + private void scaleKubernetesClusterNetworkRules(final List clusterVMIds) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); @@ -124,48 +141,31 @@ private void scaleKubernetesClusterNetworkRules(final List clusterVMIds, f throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned"); } int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); - final int scaledTotalNodeCount = clusterSize == null ? (int)kubernetesCluster.getTotalNodeCount() : (int)(clusterSize + kubernetesCluster.getMasterNodeCount()); + int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; // Provision new SSH firewall rules try { - provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + scaledTotalNodeCount - 1); + provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", - CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + scaledTotalNodeCount - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster %s", + CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName())); } } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { throw new ManagementServerException(String.format("Failed to activate SSH firewall rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } try { - removePortForwardingRules(publicIp, network, owner, removedVMIds); + removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, existingFirewallRuleSourcePortEnd); } catch (ResourceUnavailableException e) { throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } try { - provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, existingFirewallRuleSourcePortEnd + 1); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } } - private KubernetesClusterVO updateKubernetesClusterEntry(final long cores, final long memory, - final Long size, final Long serviceOfferingId) { - return Transaction.execute((TransactionCallback) status -> { - KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesCluster.getId()); - updatedCluster.setCores(cores); - updatedCluster.setMemory(memory); - if (size != null) { - updatedCluster.setNodeCount(size); - } - if (serviceOfferingId != null) { - updatedCluster.setServiceOfferingId(serviceOfferingId); - } - kubernetesClusterDao.persist(updatedCluster); - return updatedCluster; - }); - } - private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, final ServiceOffering newServiceOffering) throws CloudRuntimeException { final ServiceOffering serviceOffering = newServiceOffering == null ? serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()) : newServiceOffering; @@ -173,10 +173,11 @@ private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, fin final long size = newSize == null ? kubernetesCluster.getTotalNodeCount() : (newSize + kubernetesCluster.getMasterNodeCount()); final long cores = serviceOffering.getCpu() * size; final long memory = serviceOffering.getRamSize() * size; - KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, newSize, serviceOfferingId); + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, newSize, serviceOfferingId, + kubernetesCluster.getAutoscalingEnabled(), kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize()); if (kubernetesClusterVO == null) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster", - kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } return kubernetesClusterVO; } @@ -192,13 +193,13 @@ private boolean removeKubernetesClusterNode(final String ipAddress, final int po retryCounter++; try { Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), + pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); if (!result.first()) { LOGGER.warn(String.format("Draining node: %s on VM : %s in Kubernetes cluster : %s unsuccessful", hostName, userVm.getDisplayName(), kubernetesCluster.getName())); } else { result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - pkFile, null, String.format("sudo kubectl delete node %s", hostName), + pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), 10000, 10000, 30000); if (result.first()) { return true; @@ -302,72 +303,78 @@ private void scaleKubernetesClusterOffering() throws CloudRuntimeException { kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); } - private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); - } - final List originalVmList = getKubernetesClusterVMMaps(); - int i = originalVmList.size() - 1; - List removedVmIds = new ArrayList<>(); - while (i >= kubernetesCluster.getMasterNodeCount() + clusterSize) { - KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); + private void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { + for (KubernetesClusterVmMapVO vmMapVO : vmMaps) { UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + LOGGER.info(String.format("Removing vm : %s from cluster %s", userVM.getDisplayName(), kubernetesCluster.getName())); if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, failed to remove Kubernetes node: %s running on VM : %s", kubernetesCluster.getName(), userVM.getHostName(), userVM.getDisplayName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - // For removing port-forwarding network rules - removedVmIds.add(userVM.getId()); try { UserVm vm = userVmService.destroyVm(userVM.getId(), true); if (!userVmManager.expunge(userVM, CallContext.current().getCallingUserId(), CallContext.current().getCallingAccount())) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to expunge VM '%s'." - , kubernetesCluster.getUuid() - , vm.getInstanceName()), - kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to expunge VM '%s'." + , kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } } catch (ResourceUnavailableException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to remove VM ID: %s" - , kubernetesCluster.getUuid() , userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to remove VM ID: %s", + kubernetesCluster.getName() , userVM.getDisplayName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); if (System.currentTimeMillis() > scaleTimeoutTime) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - i--; } + // Scale network rules to update firewall rule try { - scaleKubernetesClusterNetworkRules(null, removedVmIds); + List clusterVMIds = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + scaleKubernetesClusterNetworkRules(clusterVMIds); } catch (ManagementServerException e) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to update network rules", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } } + private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); + } + List vmList; + if (this.nodeIds != null) { + vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds); + } else { + vmList = getKubernetesClusterVMMaps(); + vmList = vmList.subList((int) (kubernetesCluster.getMasterNodeCount() + clusterSize), vmList.size()); + } + Collections.reverse(vmList); + removeNodesFromCluster(vmList); + } + private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRuntimeException { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); } List clusterVMs = new ArrayList<>(); - List clusterVMIds = new ArrayList<>(); + LaunchPermissionVO launchPermission = new LaunchPermissionVO(clusterTemplate.getId(), owner.getId()); + launchPermissionDao.persist(launchPermission); try { clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to provision node VM in the cluster", kubernetesCluster.getName()), e); } - attachIsoKubernetesVMs(clusterVMs); - for (UserVm vm : clusterVMs) { - clusterVMIds.add(vm.getId()); - } try { - scaleKubernetesClusterNetworkRules(clusterVMIds, null); + List clusterVMIds = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + scaleKubernetesClusterNetworkRules(clusterVMIds); } catch (ManagementServerException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to update network rules", kubernetesCluster.getName()), e); } + attachIsoKubernetesVMs(clusterVMs); KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, scaleTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); + deleteTemplateLaunchPermission(); if (!readyNodesCountValid) { // Scaling failed logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster : %s as it does not have desired number of nodes in ready state", kubernetesCluster.getName())); } @@ -409,6 +416,10 @@ public boolean scaleCluster() throws CloudRuntimeException { if (existingServiceOffering == null) { logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getName())); } + + if (this.isAutoscalingEnabled != null) { + return autoscaleCluster(this.isAutoscalingEnabled, minSize, maxSize); + } final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalClusterSize; final long newVMRequired = clusterSize == null ? 0 : clusterSize - originalClusterSize; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 855c264d6906..d14a054339da 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -60,6 +60,7 @@ import com.cloud.network.addr.PublicIp; import com.cloud.network.rules.LoadBalancer; import com.cloud.offering.ServiceOffering; +import com.cloud.storage.LaunchPermissionVO; import com.cloud.user.Account; import com.cloud.user.SSHKeyPairVO; import com.cloud.uservm.UserVm; @@ -71,6 +72,7 @@ import com.cloud.vm.Nic; import com.cloud.vm.ReservationContext; import com.cloud.vm.ReservationContextImpl; +import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; import com.google.common.base.Strings; @@ -195,11 +197,8 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = kubernetesClusterNodeNamePrefix + "-master"; - if (kubernetesCluster.getMasterNodeCount() > 1) { - hostName += "-1"; - } - hostName = getKubernetesClusterNodeAvailableName(hostName); + String suffix = Long.toHexString(System.currentTimeMillis()); + String hostName = String.format("%s-master-%s", kubernetesClusterNodeNamePrefix, suffix); boolean haSupported = isKubernetesVersionSupportsHA(); String k8sMasterConfig = null; try { @@ -211,7 +210,7 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), - requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null); + requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null, String.valueOf(UserVmManager.UserVmType.CKSNode)); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Created master VM ID: %s, %s in the Kubernetes cluster : %s", masterVm.getUuid(), hostName, kubernetesCluster.getName())); } @@ -254,7 +253,8 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-master-%d", kubernetesClusterNodeNamePrefix, additionalMasterNodeInstance + 1)); + String suffix = Long.toHexString(System.currentTimeMillis()); + String hostName = String.format("%s-master-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sMasterConfig = null; try { k8sMasterConfig = getKubernetesAdditionalMasterConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); @@ -265,7 +265,7 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a additionalMasterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), - null, addrs, null, null, null, customParameterMap, null, null, null, null); + null, addrs, null, null, null, customParameterMap, null, null, null, null, String.valueOf(UserVmManager.UserVmType.CKSNode)); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Created master VM ID : %s, %s in the Kubernetes cluster : %s", additionalMasterVm.getUuid(), hostName, kubernetesCluster.getName())); } @@ -276,7 +276,7 @@ private UserVm provisionKubernetesClusterMasterVm(final Network network, final S ManagementServerException, InsufficientCapacityException, ResourceUnavailableException { UserVm k8sMasterVM = null; k8sMasterVM = createKubernetesMaster(network, publicIpAddress); - addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); + addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId(), true); startKubernetesVM(k8sMasterVM); k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); if (k8sMasterVM == null) { @@ -295,7 +295,7 @@ private List provisionKubernetesClusterAdditionalMasterVms(final String for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { UserVm vm = null; vm = createKubernetesAdditionalMaster(publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true); startKubernetesVM(vm); vm = userVmDao.findById(vm.getId()); if (vm == null) { @@ -378,17 +378,18 @@ private void setupKubernetesClusterNetworkRules(Network network, List cl throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); } - + // Firewall rule fo API access for master node VMs try { provisionFirewallRules(publicIp, owner, CLUSTER_API_PORT, CLUSTER_API_PORT); if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", - CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster %s", + CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getName())); } } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } + // Firewall rule fo SSH access on each node VM try { int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMs.size() - 1; provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); @@ -500,6 +501,10 @@ public boolean startKubernetesClusterOnCreate() { (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getMasterNodeCount() > 1)) { // Shared network, single-master cluster won't have an IP yet logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as no public IP found for the cluster" , kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } + // Allow account creating the kubernetes cluster to access systemVM template + LaunchPermissionVO launchPermission = new LaunchPermissionVO(clusterTemplate.getId(), owner.getId()); + launchPermissionDao.persist(launchPermission); + List clusterVMs = new ArrayList<>(); UserVm k8sMasterVM = null; try { @@ -565,7 +570,21 @@ public boolean startKubernetesClusterOnCreate() { if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); } + retrieveScriptFiles(); + for (int i = 0; i < clusterVMs.size(); ++i) { + try { + copyAutoscalerScripts(publicIpAddress, CLUSTER_NODES_DEFAULT_START_SSH_PORT + i); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + if (!createCloudStackSecret(keys)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", + kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); + } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + // remove launch permissions + deleteTemplateLaunchPermission(); return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index f408292a2e52..43f5a175e5d1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -17,10 +17,7 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; -import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -45,26 +42,22 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke private List clusterVMs = new ArrayList<>(); private KubernetesSupportedVersion upgradeVersion; + private final String upgradeScriptFilename = "upgrade-kubernetes.sh"; private File upgradeScriptFile; private long upgradeTimeoutTime; public KubernetesClusterUpgradeWorker(final KubernetesCluster kubernetesCluster, final KubernetesSupportedVersion upgradeVersion, - final KubernetesClusterManagerImpl clusterManager) { + final KubernetesClusterManagerImpl clusterManager, + final String[] keys) { super(kubernetesCluster, clusterManager); this.upgradeVersion = upgradeVersion; + this.keys = keys; } - private void retrieveUpgradeScriptFile() { - try { - String upgradeScriptData = readResourceFile("/script/upgrade-kubernetes.sh"); - upgradeScriptFile = File.createTempFile("upgrade-kuberntes", ".sh"); - BufferedWriter upgradeScriptFileWriter = new BufferedWriter(new FileWriter(upgradeScriptFile)); - upgradeScriptFileWriter.write(upgradeScriptData); - upgradeScriptFileWriter.close(); - } catch (IOException e) { - logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to prepare upgrade script", kubernetesCluster.getName()), e); - } + protected void retrieveScriptFiles() { + super.retrieveScriptFiles(); + upgradeScriptFile = retrieveScriptFile(upgradeScriptFilename); } private Pair runInstallScriptOnVM(final UserVm vm, final int index) throws Exception { @@ -93,12 +86,12 @@ private void upgradeKubernetesClusterNodes() { } result = null; if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Upgrading node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", - vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); + LOGGER.info(String.format("Upgrading node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s", + vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); } try { result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, - String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), + String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); } catch (Exception e) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); @@ -110,12 +103,21 @@ private void upgradeKubernetesClusterNodes() { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } try { + copyAutoscalerScripts(publicIpAddress, CLUSTER_NODES_DEFAULT_START_SSH_PORT + i); + if (!createCloudStackSecret(keys)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", + kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); + } result = runInstallScriptOnVM(vm, i); } catch (Exception e) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); } if (!result.first()) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + String message = String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", + kubernetesCluster.getName(), vm.getDisplayName()); + String messageWithLogs = String.format("%s. Logs :\n%s", message, result.second()); + logMessage(Level.ERROR, messageWithLogs, null); + logTransitStateDetachIsoAndThrow(Level.ERROR, message, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (System.currentTimeMillis() > upgradeTimeoutTime) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); @@ -129,8 +131,8 @@ private void upgradeKubernetesClusterNodes() { } } if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Successfully upgraded node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", - vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); + LOGGER.info(String.format("Successfully upgraded node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s", + vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); } } } @@ -151,7 +153,7 @@ public boolean upgradeCluster() throws CloudRuntimeException { if (CollectionUtils.isEmpty(clusterVMs)) { logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster : %s, unable to retrieve VMs for cluster", kubernetesCluster.getName())); } - retrieveUpgradeScriptFile(); + retrieveScriptFiles(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); attachIsoKubernetesVMs(clusterVMs, upgradeVersion); upgradeKubernetesClusterNodes(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java index 8b08dd37d553..42061cde1f0f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java @@ -23,4 +23,5 @@ public interface KubernetesClusterVmMapDao extends GenericDao { public List listByClusterId(long clusterId); + public List listByClusterIdAndVmIdsIn(long clusterId, List vmIds); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java index 0b86b2c1a622..0f6ebfa6909a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java @@ -34,6 +34,7 @@ public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase listByClusterId(long clusterId) { sc.setParameters("clusterId", clusterId); return listBy(sc, null); } + + @Override + public List listByClusterIdAndVmIdsIn(long clusterId, List vmIds) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + sc.setParameters("vmIdsIN", vmIds.toArray()); + return listBy(sc); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index b06cc00c9229..abb9fbf662d7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -41,7 +41,7 @@ public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kuber String user, File sshKeyFile, String nodeName) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, - String.format("sudo kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName.toLowerCase()), + String.format("sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName.toLowerCase()), 10000, 10000, 20000); if (result.first() && nodeName.equals(result.second().trim())) { return true; @@ -102,7 +102,7 @@ public static boolean uncordonKubernetesClusterNode(final KubernetesCluster kube Pair result = null; try { result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, - String.format("sudo kubectl uncordon %s", hostName), + String.format("sudo /opt/bin/kubectl uncordon %s", hostName), 10000, 10000, 30000); if (result.first()) { return true; @@ -125,9 +125,9 @@ public static boolean isKubernetesClusterAddOnServiceRunning(final KubernetesClu final int port, final String user, final File sshKeyFile, final String namespace, String serviceName) { try { - String cmd = "sudo kubectl get pods --all-namespaces"; + String cmd = "sudo /opt/bin/kubectl get pods --all-namespaces"; if (!Strings.isNullOrEmpty(namespace)) { - cmd = String.format("sudo kubectl get pods --namespace=%s", namespace); + cmd = String.format("sudo /opt/bin/kubectl get pods --namespace=%s", namespace); } Pair result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, cmd, @@ -203,7 +203,7 @@ public static int getKubernetesClusterReadyNodesCount(final KubernetesCluster ku final int port, final String user, final File sshKeyFile) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, - "sudo kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", + "sudo /opt/bin/kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", 10000, 10000, 20000); if (result.first()) { return Integer.parseInt(result.second().trim().replace("\"", "")); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 72a1c3794871..41ef095c7ab6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -78,6 +78,8 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne @Inject private TemplateApiService templateService; + public static final String MINIMUN_AUTOSCALER_SUPPORTED_VERSION = "1.15.0"; + private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) { KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); response.setObjectName("kubernetessupportedversion"); @@ -202,6 +204,10 @@ public static int compareSemanticVersions(String v1, String v2) throws IllegalAr return 0; } + public static boolean versionSupportsAutoscaling(KubernetesSupportedVersion clusterVersion) { + return clusterVersion.getSemanticVersion().compareTo(MINIMUN_AUTOSCALER_SUPPORTED_VERSION) >= 0; + } + /** * Returns a boolean value whether Kubernetes cluster upgrade can be carried from a given currentVersion to upgradeVersion * Kubernetes clusters can only be upgraded from one MINOR version to the next MINOR version, or between PATCH versions of the same MINOR. @@ -214,9 +220,7 @@ public static int compareSemanticVersions(String v1, String v2) throws IllegalAr */ public static boolean canUpgradeKubernetesVersion(final String currentVersion, final String upgradeVersion) throws IllegalArgumentException { int versionDiff = compareSemanticVersions(upgradeVersion, currentVersion); - if (versionDiff == 0) { - throw new IllegalArgumentException(String.format("Kubernetes clusters can not be upgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); - } else if (versionDiff < 0) { + if (versionDiff < 0) { throw new IllegalArgumentException(String.format("Kubernetes clusters can not be downgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); } String[] thisParts = currentVersion.split("\\."); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 574d8a70395e..11b74441bbde 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.api.command.user.kubernetes.cluster; +import java.util.List; + import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; @@ -30,6 +32,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; @@ -58,19 +61,38 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, - entityType = KubernetesClusterResponse.class, - description = "the ID of the Kubernetes cluster") + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster") private Long id; @ACL(accessType = SecurityChecker.AccessType.UseEntry) @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, - description = "the ID of the service offering for the virtual machines in the cluster.") + description = "the ID of the service offering for the virtual machines in the cluster.") private Long serviceOfferingId; @Parameter(name=ApiConstants.SIZE, type = CommandType.LONG, - description = "number of Kubernetes cluster nodes") + description = "number of Kubernetes cluster nodes") private Long clusterSize; + @Parameter(name = ApiConstants.NODE_IDS, + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType = UserVmResponse.class, + description = "the IDs of the nodes to be removed") + private List nodeIds; + + @Parameter(name=ApiConstants.AUTOSCALING_ENABLED, type = CommandType.BOOLEAN, + description = "Whether autoscaling is enabled for the cluster") + private Boolean isAutoscalingEnabled; + + @Parameter(name=ApiConstants.MIN_SIZE, type = CommandType.LONG, + description = "Minimum number of worker nodes in the cluster") + private Long minSize; + + @Parameter(name=ApiConstants.MAX_SIZE, type = CommandType.LONG, + description = "Maximum number of worker nodes in the cluster") + private Long maxSize; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -87,6 +109,22 @@ public Long getClusterSize() { return clusterSize; } + public List getNodeIds() { + return nodeIds; + } + + public Boolean isAutoscalingEnabled() { + return isAutoscalingEnabled; + } + + public Long getMinSize() { + return minSize; + } + + public Long getMaxSize() { + return maxSize; + } + @Override public String getEventType() { return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_SCALE; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index bb3f14f56891..f8a054333829 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -141,6 +141,18 @@ public class KubernetesClusterResponse extends BaseResponse implements Controlle @Param(description = "Public IP Address ID of the cluster") private String ipAddressId; + @SerializedName(ApiConstants.AUTOSCALING_ENABLED) + @Param(description = "Whether autoscaling is enabled for the cluster") + private boolean isAutoscalingEnabled; + + @SerializedName(ApiConstants.MIN_SIZE) + @Param(description = "Minimum size of the cluster") + private Long minSize; + + @SerializedName(ApiConstants.MAX_SIZE) + @Param(description = "Maximum size of the cluster") + private Long maxSize; + public KubernetesClusterResponse() { } @@ -340,4 +352,16 @@ public void setIpAddress(String ipAddress) { public void setIpAddressId(String ipAddressId) { this.ipAddressId = ipAddressId; } + + public void setAutoscalingEnabled(boolean isAutoscalingEnabled) { + this.isAutoscalingEnabled = isAutoscalingEnabled; + } + + public void setMinSize(Long minSize) { + this.minSize = minSize; + } + + public void setMaxSize(Long maxSize) { + this.maxSize = maxSize; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index 787ea97491ce..9e395a3f5673 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -20,14 +20,14 @@ ssh_authorized_keys: {{ k8s.ssh.pub.key }} -write-files: +write_files: - path: /opt/bin/setup-kube-system - permissions: 0700 + permissions: '0700' owner: root:root content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/debian/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -96,7 +96,7 @@ write-files: mkdir -p /opt/bin cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . chmod +x {kubeadm,kubelet,kubectl} sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service @@ -125,6 +125,10 @@ write-files: done <<< "$output" setup_complete=true fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" @@ -175,14 +179,14 @@ write-files: fi - path: /opt/bin/deploy-kube-system - permissions: 0700 + permissions: '0700' owner: root:root content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then - echo "Already provisioned!" - exit 0 + if [[ -f "/home/debian/success" ]]; then + echo "Already provisioned!" + exit 0 fi if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then @@ -192,46 +196,43 @@ write-files: modprobe ip_vs modprobe ip_vs_wrr modprobe ip_vs_sh - modprobe nf_conntrack_ipv4 + modprobe nf_conntrack if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then export PATH=$PATH:/opt/bin fi kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --control-plane --certificate-key {{ k8s_master.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success - -coreos: - units: - - name: docker.service - command: start - enable: true - - - name: setup-kube-system.service - command: start - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - name: deploy-kube-system.service - command: start - content: | - [Unit] - After=setup-kube-system.service - - [Service] - Type=simple - StartLimitInterval=0 - Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version - ExecStart=/opt/bin/deploy-kube-system - - update: - group: stable - reboot-strategy: off + sudo touch /home/debian/success + echo "true" > /home/debian/success + + - path: /etc/systemd/system/setup-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - path: /etc/systemd/system/deploy-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version + ExecStart=/opt/bin/deploy-kube-system + +runcmd: + - [ systemctl, start, setup-kube-system ] + - [ systemctl, start, deploy-kube-system ] + diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index 14828578ed8c..d17adc664057 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -20,7 +20,7 @@ ssh_authorized_keys: {{ k8s.ssh.pub.key }} -write-files: +write_files: - path: /etc/conf.d/nfs permissions: '0644' content: | @@ -42,12 +42,12 @@ write-files: {{ k8s_master.apiserver.key }} - path: /opt/bin/setup-kube-system - permissions: 0700 + permissions: '0700' owner: root:root content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/debian/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -116,7 +116,7 @@ write-files: mkdir -p /opt/bin cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . chmod +x {kubeadm,kubelet,kubectl} sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service @@ -147,6 +147,10 @@ write-files: fi mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" @@ -214,12 +218,12 @@ write-files: done - path: /opt/bin/deploy-kube-system - permissions: 0700 + permissions: '0700' owner: root:root content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/debian/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -255,40 +259,37 @@ write-files: kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - sudo touch /home/core/success - echo "true" > /home/core/success - -coreos: - units: - - name: docker.service - command: start - enable: true - - - name: setup-kube-system.service - command: start - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - name: deploy-kube-system.service - command: start - content: | - [Unit] - After=setup-kube-system.service - - [Service] - Type=simple - StartLimitInterval=0 - Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://127.0.0.1:6443/version - ExecStart=/opt/bin/deploy-kube-system - - update: - group: stable - reboot-strategy: off + sudo touch /home/debian/success + echo "true" > /home/debian/success + + - path: /etc/systemd/system/setup-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - path: /etc/systemd/system/deploy-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://127.0.0.1:6443/version + ExecStart=/opt/bin/deploy-kube-system + +runcmd: + - [ systemctl, start, setup-kube-system ] + - [ systemctl, start, deploy-kube-system ] + diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index d2f5454a669d..74cd18b5ea7d 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -20,14 +20,14 @@ ssh_authorized_keys: {{ k8s.ssh.pub.key }} -write-files: +write_files: - path: /opt/bin/setup-kube-system - permissions: 0700 + permissions: '0700' owner: root:root content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/debian/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -96,7 +96,7 @@ write-files: mkdir -p /opt/bin cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . chmod +x {kubeadm,kubelet,kubectl} sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service @@ -125,6 +125,10 @@ write-files: done <<< "$output" setup_complete=true fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" @@ -175,12 +179,12 @@ write-files: fi - path: /opt/bin/deploy-kube-system - permissions: 0700 + permissions: '0700' owner: root:root content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/debian/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -192,46 +196,42 @@ write-files: modprobe ip_vs modprobe ip_vs_wrr modprobe ip_vs_sh - modprobe nf_conntrack_ipv4 + modprobe nf_conntrack if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then export PATH=$PATH:/opt/bin fi kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success - -coreos: - units: - - name: docker.service - command: start - enable: true - - - name: setup-kube-system.service - command: start - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - name: deploy-kube-system.service - command: start - content: | - [Unit] - After=setup-kube-system.service - - [Service] - Type=simple - StartLimitInterval=0 - Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version - ExecStart=/opt/bin/deploy-kube-system - - update: - group: stable - reboot-strategy: off + sudo touch /home/debian/success + echo "true" > /home/debian/success + + - path: /etc/systemd/system/setup-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - path: /etc/systemd/system/deploy-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version + ExecStart=/opt/bin/deploy-kube-system + +runcmd: + - [ systemctl, start, setup-kube-system ] + - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/autoscale-kube-cluster b/plugins/integrations/kubernetes-service/src/main/resources/script/autoscale-kube-cluster new file mode 100755 index 000000000000..8d234b394617 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/autoscale-kube-cluster @@ -0,0 +1,76 @@ +#! /bin/bash +function usage() { + cat << USAGE +Usage: ./autoscale-kube-cluster [OPTIONS]... +Enables autoscaling for the kubernetes cluster. +Arguments: + -i, --id string ID of the cluster + -e, --enable Enables autoscaling + -d, --disable Disables autoscaling + -M, --maxsize number Maximum size of the cluster + -m, --minsize number Minimum size of the cluster +Other arguments: + -h, --help Display this help message and exit +Examples: + ./autoscale-kube-cluster -e -M 3 -m 1 + ./autoscale-kube-cluster -d +USAGE + exit 0 +} +ID="" +ENABLE="" +MINSIZE="" +MAXSIZE="" +while [ -n "$1" ]; do + case "$1" in + -h | --help) + usage + ;; + -i | --id) + ID=$2 + shift 2 + ;; + -e | --enable) + ENABLE="true" + shift 1 + ;; + -d | --enable) + ENABLE="false" + shift 1 + ;; + -M | --maxsize) + MAXSIZE=$2 + shift 2 + ;; + -m | --minsize) + MINSIZE=$2 + shift 2 + ;; + -*|*) + echo "ERROR: no such option $1. -h or --help for help" + exit 1 + ;; + esac +done +if [ $ENABLE == "true" ] ; then + if [ -e /opt/autoscaler/autoscaler_tmpl.yaml ]; then + sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml + kubectl apply -f /opt/autoscaler/autoscaler_now.yaml + exit 0 + else + mkdir -p /opt/autoscaler + AUTOSCALER_URL="https://raw.githubusercontent.com/shapeblue/autoscaler/add-acs/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" + autoscaler_conf_file="/opt/autoscaler/autoscaler_tmpl.yaml" + curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} + if [ $? -ne 0 ]; then + echo "Unable to connect to the internet to download the autoscaler deployment and image" + exit 1 + else + sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml + kubectl apply -f /opt/autoscaler/autoscaler_now.yaml + exit 0 + fi + fi +else + kubectl delete deployment -n kube-system cluster-autoscaler +fi diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret new file mode 100755 index 000000000000..e734e0436bbf --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret @@ -0,0 +1,51 @@ +#! /bin/bash +function usage() { + cat << USAGE +Usage: ./deploy-cloudstack-secret [OPTIONS]... +Enables autoscaling for the kubernetes cluster. +Arguments: + -u, --url string ID of the cluster + -k, --key string Enables autoscaling + -s, --secret string Disables autoscaling +Other arguments: + -h, --help Display this help message and exit +Examples: + ./deploy-cloudstack-secret -u http://localhost:8080 -k abcd -s efgh +USAGE + exit 0 +} +API_URL="" +API_KEY="" +SECRET_KEY="" +while [ -n "$1" ]; do + case "$1" in + -h | --help) + usage + ;; + -u | --url) + API_URL=$2 + shift 2 + ;; + -k | --key) + API_KEY=$2 + shift 2 + ;; + -s | --secret) + SECRET_KEY=$2 + shift 2 + ;; + -*|*) + echo "ERROR: no such option $1. -h or --help for help" + exit 1 + ;; + esac +done +cat > /tmp/cloud-config < getKeys(GetUserKeysCmd cmd){ return null; } + @Override + public Map getKeys(Long userId) { + return null; + } + @Override public void checkAccess(User user, ControlledEntity entity) throws PermissionDeniedException { diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index d7d9c16310d9..d2fd6ed392c0 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -70,6 +70,12 @@ echo "Downloading dashboard config ${DASHBORAD_CONFIG_URL}" dashboard_conf_file="${working_dir}/dashboard.yaml" curl -sSL ${DASHBORAD_CONFIG_URL} -o ${dashboard_conf_file} +# TODO : Change the url once merged +AUTOSCALER_URL="https://raw.githubusercontent.com/shapeblue/autoscaler/add-acs/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" +echo "Downloading kubernetes cluster autoscaler ${AUTOSCALER_URL}" +autoscaler_conf_file="${working_dir}/autoscaler.yaml" +curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} + echo "Fetching k8s docker images..." docker -v if [ $? -ne 0 ]; then @@ -87,6 +93,10 @@ if [ $? -ne 0 ]; then fi mkdir -p "${working_dir}/docker" output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` + +# Don't forget about the autoscaler image ! +autoscaler_image=`grep "image:" ${autoscaler_conf_file} | cut -d ':' -f2- | tr -d ' '` +output=`printf "%s\n" ${output} ${autoscaler_image}` while read -r line; do echo "Downloading docker image $line ---" sudo docker pull "$line" diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index b0eac2bcf448..2949ad92f71f 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -1483,7 +1483,6 @@ public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile prof if(profile.getHypervisorType() == HypervisorType.Hyperv) { controlNic = managementNic; } - CheckSshCommand check = new CheckSshCommand(profile.getInstanceName(), controlNic.getIPv4Address(), 3922); cmds.addCommand("checkSsh", check); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index c71054951619..f0f2b3a5a1fd 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1336,7 +1336,7 @@ private long createNewVM(AutoScaleVmGroupVO asGroup) { } else { vm = _userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), - null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, addrs, true, null, null, null, null, null, null, null); + null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, addrs, true, null, null, null, null, null, null, null, String.valueOf(UserVmManager.UserVmType.AutoScaleVM)); } } diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index a20090cf7c14..37d49a685730 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -2431,7 +2431,11 @@ public Pair findUserByApiKey(String apiKey) { @Override public Map getKeys(GetUserKeysCmd cmd) { final long userId = cmd.getID(); + return getKeys(userId); + } + @Override + public Map getKeys(Long userId) { User user = getActiveUser(userId); if (user == null) { throw new InvalidParameterValueException("Unable to find user by id"); diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index 6edf7e6caec8..47787f99a8cf 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -55,6 +55,10 @@ public interface UserVmManager extends UserVmService { static final int MAX_USER_DATA_LENGTH_BYTES = 2048; + public static enum UserVmType { + UserVM, AutoScaleVM, CKSNode + } + /** * @param hostId get all of the virtual machines that belong to one host. * @return collection of VirtualMachine. diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index a931159ea211..0e686dd9295e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -3178,7 +3178,7 @@ public UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOff return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, userData, sshKeyPair, hypervisor, caller, requestedIps, defaultIps, displayVm, keyboard, affinityGroupIdList, customParametes, customId, dhcpOptionMap, - dataDiskTemplateToDiskOfferingMap, userVmOVFProperties); + dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, null); } @@ -3289,7 +3289,7 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, userData, sshKeyPair, hypervisor, caller, requestedIps, defaultIps, displayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, dataDiskTemplateToDiskOfferingMap, - userVmOVFProperties); + userVmOVFProperties, null); } @Override @@ -3298,7 +3298,7 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, String sshKeyPair, Map requestedIps, IpAddresses defaultIps, Boolean displayvm, String keyboard, List affinityGroupIdList, Map customParametrs, String customId, Map> dhcpOptionsMap, Map dataDiskTemplateToDiskOfferingMap, - Map userVmOVFPropertiesMap) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, + Map userVmOVFPropertiesMap, String type) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); @@ -3350,7 +3350,7 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, networkList, null, group, httpmethod, userData, sshKeyPair, hypervisor, caller, requestedIps, defaultIps, displayvm, keyboard, affinityGroupIdList, customParametrs, customId, dhcpOptionsMap, - dataDiskTemplateToDiskOfferingMap, userVmOVFPropertiesMap); + dataDiskTemplateToDiskOfferingMap, userVmOVFPropertiesMap, type); } private NetworkVO getNetworkToAddToNetworkList(VirtualMachineTemplate template, Account owner, HypervisorType hypervisor, @@ -3469,7 +3469,7 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe String sshKeyPair, HypervisorType hypervisor, Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, - Map userVmOVFPropertiesMap) throws InsufficientCapacityException, ResourceUnavailableException, + Map userVmOVFPropertiesMap, String type) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, StorageUnavailableException, ResourceAllocationException { _accountMgr.checkAccess(caller, null, true, owner); @@ -3661,7 +3661,7 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe } } - if (template.getTemplateType().equals(TemplateType.SYSTEM)) { + if (template.getTemplateType().equals(TemplateType.SYSTEM) && !String.valueOf(UserVmType.CKSNode).equals(type)) { throw new InvalidParameterValueException("Unable to use system template " + template.getId() + " to deploy a user vm"); } List listZoneTemplate = _templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); @@ -3841,7 +3841,7 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe UserVmVO vm = commitUserVm(zone, template, hostName, displayName, owner, diskOfferingId, diskSize, userData, caller, isDisplayVm, keyboard, accountId, userId, offering, isIso, sshPublicKey, networkNicMap, id, instanceName, uuidName, hypervisorType, customParameters, dhcpOptionMap, - datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap); + datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, type); // Assign instance to the group try { @@ -3903,7 +3903,7 @@ private UserVmVO commitUserVm(final boolean isImport, final DataCenter zone, fin final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, - final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState) throws InsufficientCapacityException { + final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, String type) throws InsufficientCapacityException { return Transaction.execute(new TransactionCallbackWithException() { @Override public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCapacityException { @@ -3989,6 +3989,7 @@ public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCap } } + vm.setUserVmType(type); _vmDao.persist(vm); for (String key : customParameters.keySet()) { if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || @@ -4090,13 +4091,13 @@ private UserVmVO commitUserVm(final DataCenter zone, final VirtualMachineTemplat final long accountId, final long userId, final ServiceOfferingVO offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, - Map userVmOVFPropertiesMap) throws InsufficientCapacityException { + Map userVmOVFPropertiesMap, String type) throws InsufficientCapacityException { return commitUserVm(false, zone, null, null, template, hostName, displayName, owner, diskOfferingId, diskSize, userData, caller, isDisplayVm, keyboard, accountId, userId, offering, isIso, sshPublicKey, networkNicMap, id, instanceName, uuidName, hypervisorType, customParameters, extraDhcpOptionMap, dataDiskTemplateToDiskOfferingMap, - userVmOVFPropertiesMap, null); + userVmOVFPropertiesMap, null, type); } public void validateRootDiskResize(final HypervisorType hypervisorType, Long rootDiskSize, VMTemplateVO templateVO, UserVmVO vm, final Map customParameters) throws InvalidParameterValueException @@ -4403,12 +4404,54 @@ private UserVm startVirtualMachine(long vmId, Long podId, Long clusterId, Long h return vm; } + private void addUserVMCmdlineArgs(Long vmId, VirtualMachineProfile profile, DeployDestination dest, StringBuilder buf) { + UserVmVO k8sVM = _vmDao.findById(vmId); + buf.append(" template=domP"); + buf.append(" name=").append(profile.getHostName()); + buf.append(" type=").append(k8sVM.getUserVmType()); + for (NicProfile nic : profile.getNics()) { + int deviceId = nic.getDeviceId(); + if (nic.getIPv4Address() == null) { + buf.append(" eth").append(deviceId).append("ip=").append("0.0.0.0"); + buf.append(" eth").append(deviceId).append("mask=").append("0.0.0.0"); + } else { + buf.append(" eth").append(deviceId).append("ip=").append(nic.getIPv4Address()); + buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask()); + } + + if (nic.isDefaultNic()) { + buf.append(" gateway=").append(nic.getIPv4Gateway()); + } + + if (nic.getTrafficType() == TrafficType.Management) { + String mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); + if (NetUtils.isValidIp4Cidr(mgmt_cidr)) { + buf.append(" mgmtcidr=").append(mgmt_cidr); + } + buf.append(" localgw=").append(dest.getPod().getGateway()); + } + } + DataCenterVO dc = _dcDao.findById(profile.getVirtualMachine().getDataCenterId()); + buf.append(" internaldns1=").append(dc.getInternalDns1()); + if (dc.getInternalDns2() != null) { + buf.append(" internaldns2=").append(dc.getInternalDns2()); + } + buf.append(" dns1=").append(dc.getDns1()); + if (dc.getDns2() != null) { + buf.append(" dns2=").append(dc.getDns2()); + } + s_logger.info("cmdline details: "+ buf.toString()); + } + @Override public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) { UserVmVO vm = _vmDao.findById(profile.getId()); Map details = userVmDetailsDao.listDetailsKeyPairs(vm.getId()); vm.setDetails(details); - + StringBuilder buf = profile.getBootArgsBuilder(); + if (String.valueOf(UserVmType.CKSNode).equals(vm.getUserVmType())) { + addUserVMCmdlineArgs(vm.getId(), profile, dest, buf); + } // add userdata info into vm profile Nic defaultNic = _networkModel.getDefaultNic(vm.getId()); if(defaultNic != null) { @@ -5251,7 +5294,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } vm = createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, name, displayName, diskOfferingId, size, group, cmd.getHypervisor(), cmd.getHttpMethod(), userData, sshKeyPairName, cmd.getIpToNetworkMap(), addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), - cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties); + cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, null); } } // check if this templateId has a child ISO @@ -7251,7 +7294,7 @@ public UserVm importVM(final DataCenter zone, final Host host, final VirtualMach null, null, userData, caller, isDisplayVm, keyboard, accountId, userId, serviceOffering, template.getFormat().equals(ImageFormat.ISO), sshPublicKey, null, id, instanceName, uuidName, hypervisorType, customParameters, - null, null, null, powerState); + null, null, null, powerState, null); } @Override diff --git a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java index ea6287d2bcc1..7916007c4065 100644 --- a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java +++ b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java @@ -425,6 +425,11 @@ public Map getKeys(GetUserKeysCmd cmd) { return null; } + @Override + public Map getKeys(Long userId) { + return null; + } + @Override public void checkAccess(User user, ControlledEntity entity) throws PermissionDeniedException { diff --git a/systemvm/debian/opt/cloud/bin/setup/CKSNode.sh b/systemvm/debian/opt/cloud/bin/setup/CKSNode.sh new file mode 100755 index 000000000000..bd79e885096a --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/setup/CKSNode.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +. /opt/cloud/bin/setup/common.sh + +setup_k8s_node() { + log_it "Setting up k8s node" + + # set default ssh port and restart sshd service + sed -i 's/3922/22/g' /etc/ssh/sshd_config + + swapoff -a + sudo sed -i '/ swap / s/^/#/' /etc/fstab + log_it "Swap disabled" + + log_it "Setting up interfaces" + setup_common eth0 + setup_system_rfc1918_internal + + log_it "Setting up entry in hosts" + sed -i /$NAME/d /etc/hosts + echo "$ETH0_IP $NAME" >> /etc/hosts + + public_ip=`getPublicIp` + echo "$public_ip $NAME" >> /etc/hosts + + echo "export PATH='$PATH:/opt/bin/'">> ~/.bashrc + + disable_rpfilter + enable_fwding 1 + enable_irqbalance 0 + setup_ntp + dhclient + + rm -f /etc/logrotate.d/cloud + + log_it "Starting cloud-init services" + systemctl enable --now --no-block containerd + systemctl enable --now --no-block docker.socket + systemctl enable --now --no-block docker.service + systemctl enable --now --no-block cloud-init + systemctl enable --now --no-block cloud-config + systemctl enable --now --no-block cloud-final +} + +setup_k8s_node \ No newline at end of file diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh index 0fb317b32035..7a42b0c7f963 100755 --- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh +++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh @@ -170,6 +170,7 @@ patch_systemvm() { patch() { local PATCH_MOUNT=/media/cdrom local logfile="/var/log/patchsystemvm.log" + if [ "$TYPE" == "consoleproxy" ] || [ "$TYPE" == "secstorage" ] && [ -f ${PATCH_MOUNT}/agent.zip ] && [ -f /var/cache/cloud/patch.required ] then echo "Patching systemvm for cloud service with mount=$PATCH_MOUNT for type=$TYPE" >> $logfile diff --git a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config index 02593a37affb..58302ef3a82c 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config +++ b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config @@ -61,7 +61,6 @@ patch() { [ -f ${md5file} ] && oldmd5=$(cat ${md5file}) local newmd5= [ -f ${patchfile} ] && newmd5=$(md5sum ${patchfile} | awk '{print $1}') - log_it "Scripts checksum detected: oldmd5=$oldmd5 newmd5=$newmd5" if [ "$oldmd5" != "$newmd5" ] && [ -f ${patchfile} ] && [ "$newmd5" != "" ] then diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index e24642fc6035..9b406b1e2b25 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -543,7 +543,7 @@ setup_system_rfc1918_internal() { public_ip=`getPublicIp` echo "$public_ip" | grep -E "^((127\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(192\.168\.))" if [ "$?" == "0" ]; then - log_it "Not setting up route of RFC1918 space to $LOCAL_GW befause $public_ip is RFC1918." + log_it "Not setting up route of RFC1918 space to $LOCAL_GW because $public_ip is RFC1918." else log_it "Setting up route of RFC1918 space to $LOCAL_GW" # Setup general route for RFC 1918 space, as otherwise it will be sent to diff --git a/systemvm/debian/opt/cloud/bin/setup/postinit.sh b/systemvm/debian/opt/cloud/bin/setup/postinit.sh index 5e7e4c01a228..0ebd73a23ca6 100755 --- a/systemvm/debian/opt/cloud/bin/setup/postinit.sh +++ b/systemvm/debian/opt/cloud/bin/setup/postinit.sh @@ -18,8 +18,17 @@ # # This scripts before ssh.service but after cloud-early-config +log_it() { + echo "$(date) $@" >> /var/log/cloud.log + log_action_msg "$@" +} + # Eject cdrom if any -eject || true +CMDLINE=/var/cache/cloud/cmdline +export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) +if [ "$TYPE" != "CKSNode" ]; then + eject || true +fi # Restart journald for setting changes to apply systemctl restart systemd-journald @@ -33,6 +42,10 @@ then fi fi +if [ "$TYPE" == "CKSNode" ]; then + pkill -9 dhclient +fi + [ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs for svc in $(cat /var/cache/cloud/enabled_svcs) do diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index f2f0471cfbf4..31a772f3d117 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -30,6 +30,7 @@ deleteKubernetesCluster, upgradeKubernetesCluster, scaleKubernetesCluster, + getKubernetesClusterConfig, destroyVirtualMachine, deleteNetwork) from marvin.cloudstackException import CloudstackAPIException @@ -47,7 +48,8 @@ from nose.plugins.attrib import attr from marvin.lib.decoratorGenerators import skipTestIf -import time +from kubernetes import client, config +import time, io, yaml _multiprocess_shared_ = True @@ -63,7 +65,6 @@ def setUpClass(cls): cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) cls.hypervisor = cls.testClient.getHypervisorInfo() cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ - cls.cks_template_name_key = "cloud.kubernetes.cluster.template.name." + cls.hypervisor.lower() cls.hypervisorNotSupported = False if cls.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: @@ -73,8 +74,13 @@ def setUpClass(cls): cls.kubernetes_version_ids = [] if cls.hypervisorNotSupported == False: - cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, - name="cloud.kubernetes.service.enabled")[0].value + cls.endpoint_url = Configurations.list(cls.apiclient, name="endpointe.url")[0].value + if "localhost" in cls.endpoint_url: + endpoint_url = "http://%s:%d/client/api " %(cls.mgtSvrDetails["mgtSvrIp"], cls.mgtSvrDetails["port"]) + cls.debug("Setting endpointe.url to %s" %(endpoint_url)) + Configurations.update(cls.apiclient, "endpointe.url", endpoint_url) + + cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value if cls.initial_configuration_cks_enabled not in ["true", True]: cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") Configurations.update(cls.apiclient, @@ -82,8 +88,6 @@ def setUpClass(cls): "true") cls.restartServer() - cls.cks_template = None - cls.initial_configuration_cks_template_name = None cls.cks_service_offering = None if cls.setup_failed == False: @@ -120,20 +124,6 @@ def setUpClass(cls): (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) if cls.setup_failed == False: - cls.cks_template = cls.getKubernetesTemplate() - if cls.cks_template == FAILED: - assert False, "getKubernetesTemplate() failed to return template for hypervisor %s" % cls.hypervisor - cls.setup_failed = True - else: - cls._cleanup.append(cls.cks_template) - - if cls.setup_failed == False: - cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, - name=cls.cks_template_name_key)[0].value - Configurations.update(cls.apiclient, - cls.cks_template_name_key, - cls.cks_template.name) - cks_offering_data = cls.services["cks_service_offering"] cks_offering_data["name"] = 'CKS-Instance-' + random_gen() cls.cks_service_offering = ServiceOffering.create( @@ -161,13 +151,6 @@ def tearDownClass(cls): version_delete_failed = True cls.debug("Error: Exception during cleanup for added Kubernetes supported versions: %s" % e) try: - # Restore original CKS template - if cls.cks_template != None: - cls.cks_template.delete(cls.apiclient) - if cls.hypervisorNotSupported == False and cls.initial_configuration_cks_template_name != None: - Configurations.update(cls.apiclient, - cls.cks_template_name_key, - cls.initial_configuration_cks_template_name) # Restore CKS enabled if cls.initial_configuration_cks_enabled not in ["true", True]: cls.debug("Restoring Kubernetes Service enabled value") @@ -217,41 +200,6 @@ def isManagementUp(cls): except Exception: return False - @classmethod - def getKubernetesTemplate(cls, cks_templates=None): - - if cks_templates is None: - cks_templates = cls.services["cks_templates"] - - hypervisor = cls.hypervisor.lower() - - if hypervisor not in cks_templates.keys(): - cls.debug("Provided hypervisor has no CKS template") - return FAILED - - cks_template = cks_templates[hypervisor] - - cmd = listTemplates.listTemplatesCmd() - cmd.name = cks_template['name'] - cmd.templatefilter = 'all' - cmd.zoneid = cls.zone.id - cmd.hypervisor = hypervisor - templates = cls.apiclient.listTemplates(cmd) - - if validateList(templates)[0] != PASS: - details = None - if hypervisor in ["vmware"]: - details = [{"keyboard": "us"}] - template = Template.register(cls.apiclient, cks_template, zoneid=cls.zone.id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) - template.download(cls.apiclient) - return template - - for template in templates: - if template.isready and template.ispublic: - return Template(template.__dict__) - - return FAILED - @classmethod def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=30, interval=60): """Check if Kubernetes supported version ISO is in Ready state""" @@ -359,7 +307,7 @@ def test_02_invalid_upgrade_kubernetes_cluster(self): try: k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1.id) - self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kubernetes_version_1.id) + self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % self.kubernetes_version_1.id) self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") except Exception as e: @@ -543,6 +491,36 @@ def test_09_delete_kubernetes_ha_cluster(self): return + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_10_deploy_and_autoscale_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and check for failure while tying to autoscale it + + # Validate the following: + # 1. scaleKubernetesCluster should return valid info for the cluster when it is autoscaled + # 2. cluster-autoscaler pod should be running + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 1, True) + + self.debug("Autoscaling Kubernetes cluster with ID: %s" % k8s_cluster.id) + + try: + k8s_cluster = self.autoscaleKubernetesCluster(k8s_cluster.id, 1, 2) + self.verifyKubernetesClusterAutocale(k8s_cluster, 1, 2) + + up = self.waitForAutoscalerPodInRunningState(k8s_cluster.id) + self.assertTrue(up, "Autoscaler pod failed to run") + self.debug("Kubernetes cluster with ID: %s has autoscaler running" % k8s_cluster.id) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to autoscale Kubernetes cluster due to: %s" % e) + + return + def listKubernetesCluster(self, cluster_id = None): listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd() if cluster_id != None: @@ -601,12 +579,52 @@ def scaleKubernetesCluster(self, cluster_id, size): response = self.apiclient.scaleKubernetesCluster(scaleKubernetesClusterCmd) return response - def getValidKubernetesCluster(self, size=1, master_nodes=1): + def autoscaleKubernetesCluster(self, cluster_id, minsize, maxsize): + scaleKubernetesClusterCmd = scaleKubernetesCluster.scaleKubernetesClusterCmd() + scaleKubernetesClusterCmd.id = cluster_id + scaleKubernetesClusterCmd.autoscalingenabled = True + scaleKubernetesClusterCmd.minsize = minsize + scaleKubernetesClusterCmd.maxsize = maxsize + response = self.apiclient.scaleKubernetesCluster(scaleKubernetesClusterCmd) + return response + + def fetchKubernetesClusterConfig(self, cluster_id): + getKubernetesClusterConfigCmd = getKubernetesClusterConfig.getKubernetesClusterConfigCmd() + getKubernetesClusterConfigCmd.id = cluster_id + response = self.apiclient.getKubernetesClusterConfig(getKubernetesClusterConfigCmd) + return response + + def waitForAutoscalerPodInRunningState(self, cluster_id, retries=5, interval=60): + k8s_config = self.fetchKubernetesClusterConfig(cluster_id) + cfg = io.StringIO(k8s_config.configdata) + cfg = yaml.load(cfg) + # Adding this so we don't get certificate exceptions + cfg['clusters'][0]['cluster']['insecure-skip-tls-verify']=True + config.load_kube_config_from_dict(cfg) + v1 = client.CoreV1Api() + + while retries > 0: + time.sleep(interval) + pods = v1.list_pod_for_all_namespaces(watch=False, label_selector="app=cluster-autoscaler").items + if len(pods) == 0 : + self.debug("Autoscaler pod still not up") + continue + pod = pods[0] + if pod.status.phase == 'Running' : + self.debug("Autoscaler pod %s up and running!" % pod.metadata.name) + return True + self.debug("Autoscaler pod %s up but not running on retry %d. State is : %s" %(pod.metadata.name, retries, pod.status.phase)) + retries = retries - 1 + return False + + def getValidKubernetesCluster(self, size=1, master_nodes=1, autoscaling=False): cluster = k8s_cluster version = self.kubernetes_version_2 if master_nodes != 1: version = self.kubernetes_version_3 valid = True + if autoscaling: + version = self.kubernetes_version_4 if cluster == None: valid = False self.debug("No existing cluster available, k8s_cluster: %s" % cluster) @@ -715,6 +733,21 @@ def verifyKubernetesClusterScale(self, cluster_response, size=1, master_nodes=1) self.verifyKubernetesClusterState(cluster_response, 'Running') self.verifyKubernetesClusterSize(cluster_response, size, master_nodes) + def verifyKubernetesClusterAutocale(self, cluster_response, minsize, maxsize): + """Check if Kubernetes cluster state and node sizes are valid after upgrade""" + + self.verifyKubernetesClusterState(cluster_response, 'Running') + self.assertEqual( + cluster_response.minsize, + minsize, + "Check KubernetesCluster minsize {}, {}".format(cluster_response.minsize, minsize) + ) + self.assertEqual( + cluster_response.maxsize, + maxsize, + "Check KubernetesCluster maxsize {}, {}".format(cluster_response.maxsize, maxsize) + ) + def stopAndVerifyKubernetesCluster(self, cluster_id): """Stop Kubernetes cluster and check if it is really stopped""" diff --git a/tools/appliance/systemvmtemplate/http/preseed.cfg b/tools/appliance/systemvmtemplate/http/preseed.cfg index ce51f746c300..0a07e97b15dd 100644 --- a/tools/appliance/systemvmtemplate/http/preseed.cfg +++ b/tools/appliance/systemvmtemplate/http/preseed.cfg @@ -56,13 +56,13 @@ d-i partman-auto/disk string /dev/vda d-i partman-auto/method string regular d-i partman-auto/expert_recipe string \ boot-root :: \ - 100 60 100 ext2 \ + 512 60 512 ext2 \ $primary{ } $bootable{ } \ method{ format } format{ } \ use_filesystem{ } filesystem{ ext2 } \ mountpoint{ /boot } \ . \ - 2240 40 2500 ext4 \ + 5000 40 10000 ext4 \ method{ format } format{ } \ use_filesystem{ } filesystem{ ext4 } \ mountpoint{ / } \ diff --git a/tools/appliance/systemvmtemplate/scripts/cleanup.sh b/tools/appliance/systemvmtemplate/scripts/cleanup.sh index 8f2408a325a3..ab0ceb628611 100644 --- a/tools/appliance/systemvmtemplate/scripts/cleanup.sh +++ b/tools/appliance/systemvmtemplate/scripts/cleanup.sh @@ -17,11 +17,10 @@ # under the License. set -e -set -x function cleanup_apt() { export DEBIAN_FRONTEND=noninteractive - apt-get -y remove --purge dictionaries-common busybox isc-dhcp-client isc-dhcp-common \ + apt-get -y remove --purge dictionaries-common busybox \ task-english task-ssh-server tasksel tasksel-data laptop-detect wamerican sharutils \ nano util-linux-locales krb5-locales diff --git a/tools/appliance/systemvmtemplate/scripts/configure_conntrack.sh b/tools/appliance/systemvmtemplate/scripts/configure_conntrack.sh index 7202717d73b5..63016a98d003 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_conntrack.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_conntrack.sh @@ -34,8 +34,6 @@ function load_conntrack_modules() { grep nf_conntrack_ipv4 /etc/modules && return cat >> /etc/modules << EOF -nf_conntrack_ipv4 -nf_conntrack_ipv6 nf_conntrack nf_conntrack_ftp nf_conntrack_pptp diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 4e8605d4caf1..d4769bc1f7d4 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -124,6 +124,27 @@ function configure_services() { systemctl disable hyperv-daemons.hv-vss-daemon.service systemctl disable qemu-guest-agent + # Disable container services + systemctl disable containerd + systemctl disable docker.service + systemctl stop docker.service + systemctl disable docker.socket + systemctl stop docker.socket + + # Disable cloud init by default +cat < /etc/cloud/cloud.cfg.d/cloudstack.cfg +datasource_list: ['CloudStack'] +datasource: + CloudStack: + max_wait: 120 + timeout: 50 +EOF + + sed -i 's/\(disable_root: \)\(.*\)/\1false/' /etc/cloud/cloud.cfg + touch /etc/cloud/cloud-init.disabled + systemctl stop cloud-init + systemctl disable cloud-init + configure_apache2 configure_strongswan configure_issue diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index 2e788f3ced59..1cbdfea56308 100644 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -35,6 +35,12 @@ function debconf_packages() { echo "libc6 libraries/restart-without-asking boolean false" | debconf-set-selections } +function apt_clean() { + apt-get -y autoremove --purge + apt-get clean + apt-get autoclean +} + function install_packages() { export DEBIAN_FRONTEND=noninteractive export DEBIAN_PRIORITY=critical @@ -70,20 +76,29 @@ function install_packages() { radvd \ sharutils genisoimage aria2 \ strongswan libcharon-extra-plugins libstrongswan-extra-plugins strongswan-charon strongswan-starter \ - virt-what open-vm-tools qemu-guest-agent hyperv-daemons - - apt-get -y autoremove --purge - apt-get clean - apt-get autoclean + virt-what open-vm-tools qemu-guest-agent hyperv-daemons \ + apt-transport-https ca-certificates curl gnupg gnupg-agent software-properties-common cloud-init + apt_clean ${apt_get} install links + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - + apt-key fingerprint 0EBFCD88 + #32 bit architecture support for vhd-util: not required for 32 bit template if [ "${arch}" != "i386" ]; then dpkg --add-architecture i386 apt-get update ${apt_get} install libuuid1:i386 libc6:i386 + + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" + apt-get update + ${apt_get} install docker-ce docker-ce-cli containerd.io fi + apt_clean install_vhd_util # Install xenserver guest utilities as debian repos don't have it diff --git a/tools/appliance/systemvmtemplate/template.json b/tools/appliance/systemvmtemplate/template.json index e87effc3d516..22279d72a3e7 100644 --- a/tools/appliance/systemvmtemplate/template.json +++ b/tools/appliance/systemvmtemplate/template.json @@ -33,11 +33,11 @@ [ "-smp", "1" ] ], "format": "qcow2", - "disk_size": 2500, + "disk_size": 10000, "disk_interface": "virtio", "net_device": "virtio-net", - "iso_url": "https://download.cloudstack.org/systemvm/debian/debian-10.5.0-amd64-netinst.iso", - "iso_checksum": "0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e", + "iso_url": "http://sbjenkins-stagingrepo.jenkins.lon/flatcar/debian-10.6.0-amd64-netinst.iso", + "iso_checksum": "cb74dcb7f3816da4967c727839bdaa5efb2f912cab224279f4a31f0c9e35f79621b32afe390195d5e142d66cedc03d42f48874eba76eae23d1fac22d618cb669", "iso_checksum_type": "sha512", "output_directory": "../dist", "http_directory": "http", diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 436c656509d9..d3791cc201d4 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -2018,64 +2018,30 @@ "cks_kubernetes_versions": { "1.14.9": { "semanticversion": "1.14.9", - "url": "http://download.cloudstack.org/cks/setup-1.14.9.iso", + "url": "http://sbjenkins-stagingrepo.jenkins.lon/flatcar/setup-v1.14.9.iso", "mincpunumber": 2, "minmemory": 2048 }, "1.15.0": { "semanticversion": "1.15.0", - "url": "http://download.cloudstack.org/cks/setup-1.15.0.iso", + "url": "http://sbjenkins-stagingrepo.jenkins.lon/flatcar/setup-v1.15.0.iso", "mincpunumber": 2, "minmemory": 2048 }, "1.16.0": { "semanticversion": "1.16.0", - "url": "http://download.cloudstack.org/cks/setup-1.16.0.iso", + "url": "http://sbjenkins-stagingrepo.jenkins.lon/flatcar/setup-v1.16.0.iso", "mincpunumber": 2, "minmemory": 2048 }, "1.16.3": { "semanticversion": "1.16.3", - "url": "http://download.cloudstack.org/cks/setup-1.16.3.iso", + #"url": "http://sbjenkins-stagingrepo.jenkins.lon/flatcar/setup-v1.16.3.iso", + "url": "http://sbjenkins-stagingrepo.jenkins.lon/cks/binaries-iso/as-1.16.3.iso", "mincpunumber": 2, "minmemory": 2048 } }, - "cks_templates": { - "kvm": { - "name": "Kubernetes-Service-Template-kvm", - "displaytext": "Kubernetes-Service-Template kvm", - "format": "qcow2", - "hypervisor": "kvm", - "ostype": "CoreOS", - "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2", - "requireshvm": "True", - "ispublic": "True", - "isextractable": "True" - }, - "xenserver": { - "name": "Kubernetes-Service-Template-xen", - "displaytext": "Kubernetes-Service-Template xen", - "format": "vhd", - "hypervisor": "xenserver", - "ostype": "CoreOS", - "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2", - "requireshvm": "True", - "ispublic": "True", - "isextractable": "True" - }, - "vmware": { - "name": "Kubernetes-Service-Template-vmware", - "displaytext": "Kubernetes-Service-Template vmware", - "format": "ova", - "hypervisor": "vmware", - "ostype": "CoreOS", - "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova", - "requireshvm": "True", - "ispublic": "True", - "details": [{"keyboard":"us","nicAdapter":"Vmxnet3","rootDiskController":"pvscsi"}] - } - }, "cks_service_offering": { "name": "CKS-Instance", "displaytext": "CKS Instance",