Skip to content

Commit 09700de

Browse files
davidjumanidavidjumani
committed
Deploying the autoscaler
1 parent 1d9cf54 commit 09700de

File tree

17 files changed

+632
-39
lines changed

17 files changed

+632
-39
lines changed

‎api/src/main/java/com/cloud/user/AccountService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,4 +121,6 @@ UserAccount createUserAccount(String userName, String password, String firstName
121121
UserAccount getUserAccountById(Long userId);
122122

123123
public Map<String, String> getKeys(GetUserKeysCmd cmd);
124+
125+
public Map<String, String> getKeys(Long userId);
124126
}

‎api/src/main/java/org/apache/cloudstack/api/ApiConstants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,9 @@ public class ApiConstants {
822822
public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid";
823823
public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize";
824824
public static final String SUPPORTS_HA = "supportsha";
825+
public static final String AUTOSCALING_ENABLED = "autoscaling_enabled";
826+
public static final String MIN_SIZE = "minsize";
827+
public static final String MAX_SIZE = "maxsize";
825828

826829
public static final String BOOT_TYPE = "boottype";
827830
public static final String BOOT_MODE = "bootmode";

‎engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,4 +222,8 @@ ALTER VIEW `cloud`.`image_store_view` AS
222222
`cloud`.`image_store_details` ON image_store_details.store_id = image_store.id;
223223

224224
-- TODO : Move to 416
225+
ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `autoscaling_enabled` tinyint(1) NOT NULL DEFAULT '0';
226+
ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `minsize` bigint;
227+
ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `maxsize` bigint;
228+
225229
ALTER TABLE `cloud`.`kubernetes_cluster_vm_map` ADD COLUMN `is_master` tinyint(1) NOT NULL DEFAULT '0';

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java‎

Lines changed: 145 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,25 @@
2626
import java.util.HashMap;
2727
import java.util.List;
2828
import java.util.Map;
29+
import java.util.UUID;
2930
import java.util.concurrent.ConcurrentHashMap;
3031
import java.util.concurrent.Executors;
3132
import java.util.concurrent.ScheduledExecutorService;
3233
import java.util.concurrent.TimeUnit;
3334
import java.util.regex.Matcher;
3435
import java.util.regex.Pattern;
36+
import java.util.stream.Collectors;
3537

3638
import javax.inject.Inject;
3739
import javax.naming.ConfigurationException;
3840

3941
import org.apache.cloudstack.acl.ControlledEntity;
42+
import org.apache.cloudstack.acl.Role;
43+
import org.apache.cloudstack.acl.RoleService;
44+
import org.apache.cloudstack.acl.RoleType;
45+
import org.apache.cloudstack.acl.Rule;
4046
import org.apache.cloudstack.acl.SecurityChecker;
47+
import org.apache.cloudstack.acl.RolePermissionEntity.Permission;
4148
import org.apache.cloudstack.api.ApiConstants;
4249
import org.apache.cloudstack.api.ApiConstants.VMDetails;
4350
import org.apache.cloudstack.api.ResponseObject.ResponseView;
@@ -53,6 +60,7 @@
5360
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
5461
import org.apache.cloudstack.api.response.ListResponse;
5562
import org.apache.cloudstack.api.response.UserVmResponse;
63+
import org.apache.cloudstack.config.ApiServiceConfiguration;
5664
import org.apache.cloudstack.context.CallContext;
5765
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
5866
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -131,7 +139,11 @@
131139
import com.cloud.user.Account;
132140
import com.cloud.user.AccountManager;
133141
import com.cloud.user.AccountService;
142+
import com.cloud.user.AccountVO;
134143
import com.cloud.user.SSHKeyPairVO;
144+
import com.cloud.user.User;
145+
import com.cloud.user.UserAccount;
146+
import com.cloud.user.dao.AccountDao;
135147
import com.cloud.user.dao.SSHKeyPairDao;
136148
import com.cloud.utils.Pair;
137149
import com.cloud.utils.Ternary;
@@ -167,6 +179,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
167179
ScheduledExecutorService _gcExecutor;
168180
ScheduledExecutorService _stateScanner;
169181

182+
Account kubeadmin;
183+
170184
@Inject
171185
public KubernetesClusterDao kubernetesClusterDao;
172186
@Inject
@@ -192,6 +206,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
192206
@Inject
193207
protected TemplateJoinDao templateJoinDao;
194208
@Inject
209+
protected AccountDao accountDao;
210+
@Inject
211+
protected RoleService roleService;
212+
@Inject
195213
protected AccountService accountService;
196214
@Inject
197215
protected AccountManager accountManager;
@@ -222,6 +240,45 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
222240
@Inject
223241
protected FirewallRulesDao firewallRulesDao;
224242

243+
private Role createKubeadminRole() {
244+
Role kubeadminRole = roleService.createRole(KUBEADMIN_ACCOUNT_NAME, RoleType.Admin, "Default Kubeadmin role");
245+
roleService.createRolePermission(kubeadminRole, new Rule("listKubernetesClusters"), Permission.ALLOW, "");
246+
roleService.createRolePermission(kubeadminRole, new Rule("scaleKubernetesCluster"), Permission.ALLOW, "");
247+
roleService.createRolePermission(kubeadminRole, new Rule("*"), Permission.DENY, "");
248+
return kubeadminRole;
249+
}
250+
251+
private void init() {
252+
253+
// Check and create Kubeadmin role
254+
Role kubeadminRole = null;
255+
List<Role> roles = roleService.findRolesByName(KUBEADMIN_ACCOUNT_NAME);
256+
if (roles == null) {
257+
kubeadminRole = createKubeadminRole();
258+
} else {
259+
roles = roles.stream().filter(x -> x.getDescription() == "Default Kubeadmin role").collect(Collectors.toList());
260+
if (roles.size() != 1 ) {
261+
kubeadminRole = createKubeadminRole();
262+
} else {
263+
kubeadminRole = roles.get(0);
264+
}
265+
}
266+
267+
// Check and create Kubeadmin account
268+
if (accountManager.getActiveAccountByName(KUBEADMIN_ACCOUNT_NAME, 1L) != null) {
269+
return;
270+
}
271+
272+
AccountVO kubeadmin = new AccountVO();
273+
kubeadmin.setAccountName(KUBEADMIN_ACCOUNT_NAME);
274+
kubeadmin.setUuid(UUID.randomUUID().toString());
275+
kubeadmin.setState(Account.State.enabled);
276+
kubeadmin.setDomainId(1);
277+
kubeadmin.setType(RoleType.Admin.getAccountType());
278+
kubeadmin.setRoleId(kubeadminRole.getId());
279+
kubeadmin = accountDao.persist(kubeadmin);
280+
}
281+
225282
private void logMessage(final Level logLevel, final String message, final Exception e) {
226283
if (logLevel == Level.WARN) {
227284
if (e != null) {
@@ -628,6 +685,13 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes
628685
}
629686
}
630687
response.setVirtualMachineIds(vmIds);
688+
Boolean isAutoscalingEnabled = kubernetesCluster.isAutoscalingEnabled();
689+
LOGGER.warn("Autoscaling enabled : " + isAutoscalingEnabled);
690+
if (isAutoscalingEnabled != null) {
691+
response.setAutoscalingEnabled(isAutoscalingEnabled);
692+
}
693+
response.setMinSize(kubernetesCluster.getMinSize());
694+
response.setMaxSize(kubernetesCluster.getMaxSize());
631695
return response;
632696
}
633697

@@ -834,17 +898,59 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd
834898
final Long serviceOfferingId = cmd.getServiceOfferingId();
835899
final Long clusterSize = cmd.getClusterSize();
836900
final List<Long> nodeIds = cmd.getNodeIds();
837-
if (kubernetesClusterId == null || kubernetesClusterId < 1L) {
838-
throw new InvalidParameterValueException("Invalid Kubernetes cluster ID");
839-
}
901+
final Boolean isAutoscalingEnabled = cmd.isAutoscalingEnabled();
902+
final Long minSize = cmd.getMinSize();
903+
final Long maxSize = cmd.getMaxSize();
904+
840905
KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId);
841906
if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) {
842907
throw new InvalidParameterValueException("Invalid Kubernetes cluster ID");
843908
}
909+
844910
final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
845911
if (zone == null) {
846912
logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()));
847913
}
914+
915+
if (serviceOfferingId == null && clusterSize == null && nodeIds == null && isAutoscalingEnabled == null) {
916+
throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size or nodeids to be removed or autoscaling must be passed", kubernetesCluster.getUuid()));
917+
}
918+
919+
Account caller = CallContext.current().getCallingAccount();
920+
accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster);
921+
922+
final KubernetesSupportedVersion clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId());
923+
if (clusterVersion == null) {
924+
throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster ID: %s", kubernetesCluster.getUuid()));
925+
}
926+
927+
if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) ||
928+
kubernetesCluster.getState().equals(KubernetesCluster.State.Running) ||
929+
kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) {
930+
throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString()));
931+
}
932+
933+
if (isAutoscalingEnabled != null && isAutoscalingEnabled) {
934+
if (clusterSize != null || serviceOfferingId != null || nodeIds != null) {
935+
throw new InvalidParameterValueException("autoscaling can not be passed along with nodeids or clustersize or service offering");
936+
}
937+
938+
String csUrl = ApiServiceConfiguration.ApiServletPath.value();
939+
if (csUrl == null || csUrl.contains("localhost")) {
940+
throw new InvalidParameterValueException("Global setting endpointe.url has to be set to the Management Server's API end point");
941+
}
942+
943+
if (minSize == null || maxSize == null) {
944+
throw new InvalidParameterValueException("autoscaling requires minsize and maxsize to be passed");
945+
}
946+
if (minSize < 1) {
947+
throw new InvalidParameterValueException("minsize must be more than 1");
948+
}
949+
if (maxSize <= minSize) {
950+
throw new InvalidParameterValueException("maxsize must be greater than minsize");
951+
}
952+
}
953+
848954
if (nodeIds != null) {
849955
if (clusterSize != null || serviceOfferingId != null) {
850956
throw new InvalidParameterValueException("nodeids can not be passed along with clustersize or service offering");
@@ -859,19 +965,6 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd
859965
if (mastersToRemove >= kubernetesCluster.getMasterNodeCount()) {
860966
throw new InvalidParameterValueException("Can not remove all masters from a cluster");
861967
}
862-
863-
} else {
864-
if (serviceOfferingId == null && clusterSize == null) {
865-
throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size or nodeids to be removed must be passed", kubernetesCluster.getUuid()));
866-
}
867-
}
868-
869-
Account caller = CallContext.current().getCallingAccount();
870-
accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster);
871-
872-
final KubernetesSupportedVersion clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId());
873-
if (clusterVersion == null) {
874-
throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster ID: %s", kubernetesCluster.getUuid()));
875968
}
876969

877970
ServiceOffering serviceOffering = null;
@@ -904,12 +997,6 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd
904997
}
905998
}
906999

907-
if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) ||
908-
kubernetesCluster.getState().equals(KubernetesCluster.State.Running) ||
909-
kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) {
910-
throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString()));
911-
}
912-
9131000
if (clusterSize != null) {
9141001
if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size
9151002
throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString()));
@@ -1226,16 +1313,50 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC
12261313
return response;
12271314
}
12281315

1316+
private String[] createServiceAccount() {
1317+
// TODO : Maybe create a service account kubeadmin with restricted permissions and add the user to that instead
1318+
Account caller = CallContext.current().getCallingAccount();
1319+
String username = caller.getAccountName() + "-kubeadmin";
1320+
UserAccount kubeadmin = accountService.getActiveUserAccount(username, caller.getDomainId());
1321+
String[] keys = null;
1322+
if (kubeadmin == null) {
1323+
User kube = accountService.createUser(username, "password", "kube", "admin", "kubeadmin", null, caller.getAccountName(), caller.getDomainId(), null);
1324+
keys = accountService.createApiKeyAndSecretKey(kube.getId());
1325+
} else {
1326+
String apiKey = kubeadmin.getApiKey();
1327+
String secretKey = kubeadmin.getSecretKey();
1328+
if (Strings.isNullOrEmpty(apiKey) || Strings.isNullOrEmpty(secretKey)) {
1329+
keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId());
1330+
} else {
1331+
keys = new String[]{apiKey, secretKey};
1332+
}
1333+
}
1334+
return keys;
1335+
}
1336+
12291337
@Override
12301338
public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws CloudRuntimeException {
12311339
if (!KubernetesServiceEnabled.value()) {
12321340
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
12331341
}
12341342
validateKubernetesClusterScaleParameters(cmd);
1343+
1344+
Boolean isAutoscalingEnabled = cmd.isAutoscalingEnabled();
1345+
String[] keys = null;
1346+
if (isAutoscalingEnabled != null && isAutoscalingEnabled) {
1347+
keys = createServiceAccount();
1348+
}
1349+
12351350
KubernetesClusterScaleWorker scaleWorker =
12361351
new KubernetesClusterScaleWorker(kubernetesClusterDao.findById(cmd.getId()),
1237-
serviceOfferingDao.findById(cmd.getServiceOfferingId()), cmd.getClusterSize(),
1238-
cmd.getNodeIds(), this);
1352+
serviceOfferingDao.findById(cmd.getServiceOfferingId()),
1353+
cmd.getClusterSize(),
1354+
cmd.getNodeIds(),
1355+
cmd.isAutoscalingEnabled(),
1356+
cmd.getMinSize(),
1357+
cmd.getMaxSize(),
1358+
keys,
1359+
this);
12391360
scaleWorker = ComponentContext.inject(scaleWorker);
12401361
return scaleWorker.scaleCluster();
12411362
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable
3434
static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
3535
static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
3636
static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
37+
static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";
3738

3839
static final ConfigKey<Boolean> KubernetesServiceEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
3940
"cloud.kubernetes.service.enabled",

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,15 @@ public class KubernetesClusterVO implements KubernetesCluster {
9393
@Column(name = "endpoint")
9494
private String endpoint;
9595

96+
@Column(name = "autoscaling_enabled")
97+
private boolean isAutoscalingEnabled;
98+
99+
@Column(name = "minsize")
100+
private Long minSize;
101+
102+
@Column(name = "maxsize")
103+
private Long maxSize;
104+
96105
@Column(name = GenericDao.CREATED_COLUMN)
97106
private Date created;
98107

@@ -303,6 +312,30 @@ public Date getCreated() {
303312
return created;
304313
}
305314

315+
public boolean isAutoscalingEnabled() {
316+
return isAutoscalingEnabled;
317+
}
318+
319+
public void setAutoscalingEnabled(boolean isAutoscalingEnabled) {
320+
this.isAutoscalingEnabled = isAutoscalingEnabled;
321+
}
322+
323+
public Long getMinSize() {
324+
return minSize;
325+
}
326+
327+
public void setMinSize(Long minSize) {
328+
this.minSize = minSize;
329+
}
330+
331+
public Long getMaxSize() {
332+
return maxSize;
333+
}
334+
335+
public void setMaxSize(Long maxSize) {
336+
this.maxSize = maxSize;
337+
}
338+
306339
public KubernetesClusterVO() {
307340
this.uuid = UUID.randomUUID().toString();
308341
}

0 commit comments

Comments
 (0)