From 6eb43c92236b90c17c6310a28ef5d76babe0755b Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Fri, 31 Aug 2018 19:07:13 +0200 Subject: [PATCH 01/11] Suqash commits into 1 in order to rebase master --- .../diagnostics/GetDiagnosticsDataCmd.java | 152 ++++++++++++++++++ .../response/GetDiagnosticsDataResponse.java | 43 +++++ .../diagnostics/DiagnosticsService.java | 18 +++ .../resource/virtualnetwork/VRScripts.java | 1 + .../VirtualRoutingResource.java | 15 +- .../CopyToSecondaryStorageCommand.java | 33 ++++ .../diagnostics/CopyZipFilesCommand.java | 45 ++++++ .../diagnostics/PrepareFilesAnswer.java | 34 ++++ .../diagnostics/PrepareFilesCommand.java | 42 +++++ ...tCopyToSecondaryStorageCommandWrapper.java | 36 +++++ .../LibvirtCopyZipFilesCommandWrapper.java | 52 ++++++ .../diagnostics/DiagnosticsServiceImpl.java | 134 ++++++++++++++- .../cloud/bin/retrieve_diagnostics_files.py | 45 ++++++ 13 files changed, 647 insertions(+), 3 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/GetDiagnosticsDataResponse.java create mode 100644 core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/diagnostics/CopyZipFilesCommand.java create mode 100644 core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyToSecondaryStorageCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java create mode 100755 systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java new file mode 100644 index 000000000000..5d7a503b032f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.diagnostics; + +import java.util.List; + +import javax.inject.Inject; + +import com.cloud.event.EventTypes; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GetDiagnosticsDataResponse; +import org.apache.cloudstack.api.response.SystemVmResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.diagnostics.DiagnosticsService; +import org.apache.log4j.Logger; + +@APICommand(name = GetDiagnosticsDataCmd.APINAME, + responseObject = GetDiagnosticsDataResponse.class, + entityType = {VirtualMachine.class}, + responseHasSensitiveInfo = false, + requestHasSensitiveInfo = false, + description = "Get diagnostics data files", + since = "4.12.0.0", + authorized = {RoleType.Admin}) +public class GetDiagnosticsDataCmd extends BaseAsyncCmd { + private static final Logger LOGGER = Logger.getLogger(GetDiagnosticsDataCmd.class); + public static final String APINAME = "getDiagnosticsData"; + + @Inject + private DiagnosticsService diagnosticsService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.TARGET_ID, + type = BaseCmd.CommandType.UUID, + entityType = SystemVmResponse.class, + required = true, + validations = {ApiArgValidator.PositiveNumber}, + description = "The ID of the system VM instance to retrieve diagnostics data files from") + private Long id; + + @Parameter(name = ApiConstants.DETAILS, + type = BaseCmd.CommandType.LIST, + collectionType = BaseCmd.CommandType.STRING, + description = "Optional list of additional files to retrieve and must correspond with the diagnostics file dataTypeList. Can be specified as file name or path.") + private List additionalFilesList; + + @Parameter(name = ApiConstants.TYPE, + type = BaseCmd.CommandType.LIST, + collectionType = BaseCmd.CommandType.STRING, + description = "The diagnostics data dataTypeList required, examples are dhcp, iptables, dns or log files. Defaults are taken from the database if none has been provided.") + private List dataTypeList; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public List getAdditionalFilesList() { + return additionalFilesList; + } + + public List getDataTypeList() { + return dataTypeList; + } + + ///////////////////////////////////////////////////// + /////////////////// Implementation ////////////////// + ///////////////////////////////////////////////////// + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + if (account != null) { + return account.getId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException { + String result = diagnosticsService.getDiagnosticsDataCommand(this); + GetDiagnosticsDataResponse response = new GetDiagnosticsDataResponse(); + response.setUrl(result); + response.setObjectName("diagnostics-data"); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + //TODO rebase after Run Diagnostics get merged to get the return types + @Override + public String getEventType() { + VirtualMachine.Type vmType = _entityMgr.findById(VirtualMachine.class, getId()).getType(); + String eventType = ""; + switch (vmType) { + case ConsoleProxy: + eventType = EventTypes.EVENT_PROXY_DIAGNOSTICS; + break; + case SecondaryStorageVm: + eventType = EventTypes.EVENT_SSVM_DIAGNOSTICS; + break; + case DomainRouter: + eventType = EventTypes.EVENT_ROUTER_DIAGNOSTICS; + break; + } + return eventType; + } + + @Override + public String getEventDescription() { + return "Getting diagnostics data files from system vm: " + this._uuidMgr.getUuid(VirtualMachine.class, getId()); + } + + @Override + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.SystemVm; + } +} \ No newline at end of file diff --git a/api/src/main/java/org/apache/cloudstack/api/response/GetDiagnosticsDataResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/GetDiagnosticsDataResponse.java new file mode 100644 index 000000000000..9f50e37b764f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/GetDiagnosticsDataResponse.java @@ -0,0 +1,43 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import com.cloud.serializer.Param; +import com.cloud.vm.VirtualMachine; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = VirtualMachine.class) +public class GetDiagnosticsDataResponse extends BaseResponse { + @SerializedName(ApiConstants.URL) + @Param(description = "Storage URL to download retrieve diagnostics data files") + private String url; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsService.java b/api/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsService.java index a9177af7e0c9..f91203a70699 100644 --- a/api/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsService.java +++ b/api/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsService.java @@ -18,12 +18,30 @@ // package org.apache.cloudstack.diagnostics; +import org.apache.cloudstack.api.command.admin.diagnostics.GetDiagnosticsDataCmd; import org.apache.cloudstack.api.command.admin.diagnostics.RunDiagnosticsCmd; import java.util.Map; public interface DiagnosticsService { + /** + * network utility method to execute ICMP commands in system vms + * @params to method acquired from API command + * + * @TODO method is too long, need to refactor a lot of code to smaller methods or helper class + */ Map runDiagnosticsCommand(RunDiagnosticsCmd cmd); + + /** + * method to retrieve diagnostics data files from system vms + * + * @params to method are passed by the API command + * + * @TODO split the implementation to distinct method + * method needs to do one thing, and one thing only + */ + String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd); + } \ No newline at end of file diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java index 2c75a78b1a38..b2515f227362 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java @@ -70,4 +70,5 @@ public class VRScripts { public static final String VR_CFG = "vr_cfg.sh"; public static final String DIAGNOSTICS = "diagnostics.py"; + public static final String RETRIEVE_DIAGNOSTICS = "retrieve_diagnostics_files.py"; } \ No newline at end of file diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 112d9209349e..10a06cb9126e 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -25,6 +25,8 @@ import org.apache.cloudstack.diagnostics.DiagnosticsAnswer; import org.apache.cloudstack.diagnostics.DiagnosticsCommand; +import org.apache.cloudstack.diagnostics.PrepareFilesCommand; +import org.apache.cloudstack.diagnostics.PrepareFilesAnswer; import org.joda.time.Duration; import java.util.ArrayList; import java.util.HashMap; @@ -196,7 +198,9 @@ private Answer executeQueryCommand(NetworkElementCommand cmd) { } else if (cmd instanceof GetRouterAlertsCommand) { return execute((GetRouterAlertsCommand)cmd); } else if (cmd instanceof DiagnosticsCommand) { - return execute((DiagnosticsCommand)cmd); + return execute((DiagnosticsCommand) cmd); + } else if (cmd instanceof PrepareFilesCommand){ + return execute((PrepareFilesCommand)cmd); } else { s_logger.error("Unknown query command in VirtualRoutingResource!"); return Answer.createUnsupportedCommandAnswer(cmd); @@ -306,6 +310,15 @@ private Answer execute(DiagnosticsCommand cmd) { return new DiagnosticsAnswer(cmd, result.isSuccess(), result.getDetails()); } + private Answer execute(PrepareFilesCommand cmd) { + String fileList = String.join(" ", cmd.getFilesToRetrieveList()); + final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.RETRIEVE_DIAGNOSTICS, fileList); + if (result.isSuccess()){ + return new PrepareFilesAnswer(cmd, true, result.getDetails()); + } + return new PrepareFilesAnswer(cmd, false, result.getDetails()); + } + private Answer execute(GetDomRVersionCmd cmd) { final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.VERSION, null); if (!result.isSuccess()) { diff --git a/core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java b/core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java new file mode 100644 index 000000000000..550065fa274e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java @@ -0,0 +1,33 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.diagnostics; + +import com.cloud.agent.api.storage.StorageNfsVersionCommand; +import com.cloud.agent.api.to.DataStoreTO; + +public class CopyToSecondaryStorageCommand extends StorageNfsVersionCommand { + private DataStoreTO destStore; + + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/diagnostics/CopyZipFilesCommand.java b/core/src/main/java/org/apache/cloudstack/diagnostics/CopyZipFilesCommand.java new file mode 100644 index 000000000000..95c0f6532542 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/diagnostics/CopyZipFilesCommand.java @@ -0,0 +1,45 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.diagnostics; + +import com.cloud.agent.api.Command; + +public class CopyZipFilesCommand extends Command { + String zipFilesDir; + String routerIP; + + public CopyZipFilesCommand(String routerIp, String zipFilesDir) { + this.routerIP = routerIp; + this.zipFilesDir = zipFilesDir; + } + + public String getZipFilesDir() { + return zipFilesDir; + } + + public String getRouterIP() { + return routerIP; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesAnswer.java b/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesAnswer.java new file mode 100644 index 000000000000..90257eda780f --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesAnswer.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.diagnostics; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; + +public class PrepareFilesAnswer extends Answer { + + public PrepareFilesAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public String getExecDetails(){ + return details; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java b/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java new file mode 100644 index 000000000000..e8f8fd74b9ed --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java @@ -0,0 +1,42 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.diagnostics; + +import java.util.List; + +import com.cloud.agent.api.routing.NetworkElementCommand; + +public class PrepareFilesCommand extends NetworkElementCommand { + List filesToRetrieveList; + + public PrepareFilesCommand(List filesToRetrieve) { + this.filesToRetrieveList = filesToRetrieve; + } + + public List getFilesToRetrieveList() { + return filesToRetrieveList; + } + + @Override + public boolean isQuery() { + return true; + } + +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyToSecondaryStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyToSecondaryStorageCommandWrapper.java new file mode 100644 index 000000000000..fa4fc7509b40 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyToSecondaryStorageCommandWrapper.java @@ -0,0 +1,36 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageCommand; + +@ResourceWrapper(handles = CopyToSecondaryStorageCommand.class) +public class LibvirtCopyToSecondaryStorageCommandWrapper extends CommandWrapper { + + + @Override + public Answer execute(CopyToSecondaryStorageCommand command, LibvirtComputingResource libvirtComputingResource) { + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java new file mode 100644 index 000000000000..540660a2f1c8 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java @@ -0,0 +1,52 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script2; +import org.apache.cloudstack.diagnostics.CopyZipFilesCommand; +import org.apache.log4j.Logger; + +@ResourceWrapper(handles = CopyZipFilesCommand.class) +public class LibvirtCopyZipFilesCommandWrapper extends CommandWrapper { + private static final Logger LOGGER = Logger.getLogger(LibvirtCopyZipFilesCommandWrapper.class); + + public final Logger logger = null; + + + @Override + public Answer execute(CopyZipFilesCommand command, LibvirtComputingResource libvirtComputingResource) { + final String permKey = "/root/.ssh/id_rsa.cloud"; + String vmIp = command.getRouterIP(); + String filename = command.getZipFilesDir(); + boolean success = true; + String details = "Copying zip files: " + vmIp + ", file: " + filename; + LOGGER.info(details); + String cmdLine = String.format("/usr/bin/scp -P 3922 -o StrictHostKeyChecking=no -i %s root@%s:/root/%s .", permKey, vmIp, filename); + Script2 cmd = new Script2("/bin/bash", LOGGER); + cmd.add("-c"); + cmd.add(cmdLine); + String result = cmd.execute(); + return new Answer(command, true, result); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index 21bb0a1889e5..19c10e0ccf96 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -17,6 +17,8 @@ // under the License. package org.apache.cloudstack.diagnostics; + +import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -39,13 +41,21 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import org.apache.cloudstack.api.command.admin.diagnostics.GetDiagnosticsDataCmd; import org.apache.cloudstack.api.command.admin.diagnostics.RunDiagnosticsCmd; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.log4j.Logger; -public class DiagnosticsServiceImpl extends ManagerBase implements PluggableService, DiagnosticsService { + +public class DiagnosticsServiceImpl extends ManagerBase implements PluggableService, DiagnosticsService, Configurable { + private static final Logger LOGGER = Logger.getLogger(DiagnosticsServiceImpl.class); + protected static final int DefaultDomRSshPort = 3922; + final File permKey = new File("/root/.ssh/id_rsa.cloud"); + @Inject private AgentManager agentManager; @Inject @@ -55,6 +65,48 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ @Inject private NetworkOrchestrationService networkManager; + private static final ConfigKey EnableGarbageCollector = new ConfigKey<>("Advanced", Boolean.class, + "diagnostics.data.gc.enable", "true", "enable the diagnostics data files garbage collector", true); + + private static final ConfigKey TmpMgmtDataStoragePath = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.file.path", "/tmp", "set the temporary path in which to store data in the management server node", false); + + private static final ConfigKey DataRetrievalTimeout = new ConfigKey<>("Advanced", Integer.class, + "diagnostics.data.retrieval.timeout", "3600", "overall data retrieval timeout in seconds", false); + + private static final ConfigKey MaximumFileAgeforGarbageCollection = new ConfigKey<>("Advanced", Integer.class, + "diagnostics.data.max.file.age", "86400", "maximum file age for garbage collection in seconds", false); + + private static final ConfigKey GarbageCollectionInterval = new ConfigKey<>("Advanced", Integer.class, + "diagnostics.data.gc.interval", "86400", "garbage collection interval in seconds", false); + + private static final ConfigKey DiskQuotaPercentageThreshold = new ConfigKey<>("Advanced", Integer.class, + "diagnostics.data.disable.threshold", "0.95", "Minimum disk space percentage to initiate diagnostics file retrieval", false); + + private static final ConfigKey DefaultSupportedDataTypes = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.supported.types", "property, log, configuration, dns, dhcp, vpn, userdata,lb, [iptables], [ifconfig], [routes]", "List of supported diagnostics data type options", false); + + /** + * Global configs below are used to set the diagnostics + * data types applicable for each system vm. + * + * the names wrapped in square brackets are for data types that need to first execute a script + * in the system vm and grab output for retrieval, e.g. the output from iptables-save is written to a file + * which will then be retrieved. + */ + private static final ConfigKey SsvmDefaultSupportedLogFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.ssvm.supported.log.files", "agent.log, cloud.log", "List of supported diagnostics data log file options for ssvm", false); + + private static final ConfigKey SsvmDefaultSupportedNetworkFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.ssvm.supported.network.files", "[iptables], [ifconfig], [routes]", "List of supported diagnostics data network file options for ssvm", false); + + private static final ConfigKey CpvmDefaultSupportedLogFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.cpvm.supported.log.files", " agent.log", "List of supported diagnostics data log file options for cpvm", false); + + private static final ConfigKey CpvmDefaultSupportedNetworkFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.cpvm.supported.log.files", "[iptables], [ifconfig], [routes]", "List of supported diagnostics data network file options for cpvm", false); + + @Override @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "running diagnostics on system vm", async = true) public Map runDiagnosticsCommand(final RunDiagnosticsCmd cmd) { @@ -110,7 +162,6 @@ protected boolean hasValidChars(String optionalArgs) { final Pattern pattern = Pattern.compile(regex); return pattern.matcher(optionalArgs).find(); } - } protected String prepareShellCmd(String cmdType, String ipAddress, String optionalParams) { @@ -126,10 +177,89 @@ protected String prepareShellCmd(String cmdType, String ipAddress, String option } } + + //TODO Implement Retrieve diagnostics data files + @Override + @ActionEvent(eventType = "Retrieving diagnostics", eventDescription = "running diagnostics on system vm", async = true) + public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { + final Long vmId = cmd.getId(); + List dataType = cmd.getDataTypeList(); + List detail = cmd.getAdditionalFilesList(); + final VMInstanceVO vmInstance = instanceDao.findByIdTypes(vmId, VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.DomainRouter, VirtualMachine.Type.SecondaryStorageVm); + + + if (vmInstance == null) { + throw new InvalidParameterValueException("Unable to find a system vm with id " + vmId); + } + + final Map accessDetails = networkManager.getSystemVMAccessDetails(vmInstance); + + PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(dataType); + prepareZipFilesCommand.setAccessDetail(accessDetails); + final Answer zipFilesAnswer = agentManager.easySend(vmInstance.getHostId(), prepareZipFilesCommand); + Answer copyZipAnswer = null; + + if (zipFilesAnswer.getResult()){ + String zipFileDir = zipFilesAnswer.getDetails().replace("\n", ""); + CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), zipFileDir); + copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); + + } + +// DataStoreTO dataStoreTO +// CopyToSecondaryStorageCommand toSecondaryStorageCommand = new CopyToSecondaryStorageCommand(); +// return copyZipAnswer.getDetails(); + return null; + } + + + +// private String[] getDefaults(VirtualMachine vm) { +// VirtualMachine.Type vmType = vm.getType(); +// String[] supportedTypes = DefaultSupportedDataTypes.value().split(","); +// String[] defaults; +// List filesToRetrieve = new ArrayList<>(); +// switch (vmType){ +// case DomainRouter: +// +// break; +// case ConsoleProxy: +// break; +// case SecondaryStorageVm: +// break; +// +// default: +// throw new CloudRuntimeException("Unsupported VM type"); +// +// } +// +// return supportedTypes; +// } + +// private boolean isTypeSupported(String type) { +// String[] supportedTypes = DefaultSupportedDataTypes.value().split(","); +// return Arrays.stream(supportedTypes).anyMatch(type::equals); +// } + + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{EnableGarbageCollector, TmpMgmtDataStoragePath, DataRetrievalTimeout, + MaximumFileAgeforGarbageCollection, GarbageCollectionInterval, DiskQuotaPercentageThreshold, DefaultSupportedDataTypes, + SsvmDefaultSupportedLogFiles, SsvmDefaultSupportedNetworkFiles, CpvmDefaultSupportedLogFiles, CpvmDefaultSupportedNetworkFiles, + }; + } + @Override public List> getCommands() { List> cmdList = new ArrayList<>(); cmdList.add(RunDiagnosticsCmd.class); + cmdList.add(GetDiagnosticsDataCmd.class); return cmdList; } + + @Override + public String getConfigComponentName() { + return DiagnosticsService.class.getSimpleName(); + } } \ No newline at end of file diff --git a/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py b/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py new file mode 100755 index 000000000000..14da25d8d925 --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import zipfile +import time +import tarfile + + +def zip_files(file_name): + compression = zipfile.ZIP_DEFLATED + time_str = time.strftime("%Y%m%d-%H%M%S") + zf_name = 'zipfile_' + time_str + '.tar.gz' + # zf = zipfile.ZipFile(zf_name, mode='w') + zf = tarfile.open(zf_name, mode='w') + try: + for f in file_name: + # zf.write(f, compress_type=compression) + zf.add(f) + + except RuntimeError as e: + print "File not found" + finally: + zf.close() + print zf_name.strip() + + +if __name__ == '__main__': + file_names = sys.argv[1:] + zip_files(file_names) From eeb9be33151ab5ffcfc9e7d390ef274aa0e322ea Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Mon, 15 Oct 2018 12:23:37 +0200 Subject: [PATCH 02/11] Rebase against latest master and resolve conflicts --- .../api/command/admin/diagnostics/GetDiagnosticsDataCmd.java | 1 - .../apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java index 5d7a503b032f..157f2104422d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java @@ -121,7 +121,6 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE this.setResponseObject(response); } - //TODO rebase after Run Diagnostics get merged to get the return types @Override public String getEventType() { VirtualMachine.Type vmType = _entityMgr.findById(VirtualMachine.class, getId()).getType(); diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index 19c10e0ccf96..6d3f8af96430 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -178,9 +178,8 @@ protected String prepareShellCmd(String cmdType, String ipAddress, String option } - //TODO Implement Retrieve diagnostics data files @Override - @ActionEvent(eventType = "Retrieving diagnostics", eventDescription = "running diagnostics on system vm", async = true) + @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "running diagnostics on system vm", async = true) public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { final Long vmId = cmd.getId(); List dataType = cmd.getDataTypeList(); From f26bca18aa7ffa1adba2c749812be06ae2e9a022 Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Mon, 15 Oct 2018 18:29:13 +0200 Subject: [PATCH 03/11] Add volume ID param to Cmd --- .../diagnostics/GetDiagnosticsDataCmd.java | 14 +++++++++++ .../diagnostics/DiagnosticsServiceImpl.java | 25 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java index 157f2104422d..df51d38b63de 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java @@ -36,6 +36,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GetDiagnosticsDataResponse; import org.apache.cloudstack.api.response.SystemVmResponse; +import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.diagnostics.DiagnosticsService; import org.apache.log4j.Logger; @@ -78,6 +79,15 @@ public class GetDiagnosticsDataCmd extends BaseAsyncCmd { description = "The diagnostics data dataTypeList required, examples are dhcp, iptables, dns or log files. Defaults are taken from the database if none has been provided.") private List dataTypeList; + @Parameter( + name = ApiConstants.VOLUME_ID, + type = CommandType.UUID, + entityType = VolumeResponse.class, + validations = {ApiArgValidator.PositiveNumber}, + description = "The ID of the volume to be temporarily mounted on system VM to seed diagnostics data files", + required = true) + private Long volumeId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -94,6 +104,10 @@ public List getDataTypeList() { return dataTypeList; } + public Long getVolumeId() { + return volumeId; + } + ///////////////////////////////////////////////////// /////////////////// Implementation ////////////////// ///////////////////////////////////////////////////// diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index 6d3f8af96430..b4de1e45184e 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -36,6 +36,7 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; @@ -44,8 +45,13 @@ import org.apache.cloudstack.api.command.admin.diagnostics.GetDiagnosticsDataCmd; import org.apache.cloudstack.api.command.admin.diagnostics.RunDiagnosticsCmd; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; @@ -64,6 +70,12 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ private VirtualMachineManager vmManager; @Inject private NetworkOrchestrationService networkManager; + @Inject + VolumeOrchestrationService volumeOcherstartor; + @Inject + private VolumeDataFactory volFactory; + @Inject + private PrimaryDataStoreDao storagePoolDao; private static final ConfigKey EnableGarbageCollector = new ConfigKey<>("Advanced", Boolean.class, "diagnostics.data.gc.enable", "true", "enable the diagnostics data files garbage collector", true); @@ -178,6 +190,8 @@ protected String prepareShellCmd(String cmdType, String ipAddress, String option } + + @Override @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "running diagnostics on system vm", async = true) public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { @@ -205,6 +219,17 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { } + // Create temporary volume on Primary Storage to store files + + VolumeInfo volumeToAttach = volFactory.getVolume(cmd.getVolumeId()); + Hypervisor.HypervisorType rootDiskHyperType = vmInstance.getHypervisorType(); + StoragePoolVO destPrimaryStorage = storagePoolDao.findById(volumeToAttach.getPoolId()); + try { + VolumeInfo newVolumeOnPrimaryStore = volumeOcherstartor.createVolumeOnPrimaryStorage(vmInstance, volumeToAttach,rootDiskHyperType, destPrimaryStorage); + } catch (NoTransitionException e) { + e.printStackTrace(); + } + // DataStoreTO dataStoreTO // CopyToSecondaryStorageCommand toSecondaryStorageCommand = new CopyToSecondaryStorageCommand(); // return copyZipAnswer.getDetails(); From 98424659b714739898f9578ba14438b2b76048f1 Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Tue, 16 Oct 2018 18:09:08 +0200 Subject: [PATCH 04/11] Add global configs for supporteddiagnostics data types --- .../diagnostics/GetDiagnosticsDataCmd.java | 14 ----- .../LibvirtCopyZipFilesCommandWrapper.java | 3 +- .../diagnostics/DiagnosticsServiceImpl.java | 61 +++++++++---------- 3 files changed, 29 insertions(+), 49 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java index df51d38b63de..157f2104422d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java @@ -36,7 +36,6 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GetDiagnosticsDataResponse; import org.apache.cloudstack.api.response.SystemVmResponse; -import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.diagnostics.DiagnosticsService; import org.apache.log4j.Logger; @@ -79,15 +78,6 @@ public class GetDiagnosticsDataCmd extends BaseAsyncCmd { description = "The diagnostics data dataTypeList required, examples are dhcp, iptables, dns or log files. Defaults are taken from the database if none has been provided.") private List dataTypeList; - @Parameter( - name = ApiConstants.VOLUME_ID, - type = CommandType.UUID, - entityType = VolumeResponse.class, - validations = {ApiArgValidator.PositiveNumber}, - description = "The ID of the volume to be temporarily mounted on system VM to seed diagnostics data files", - required = true) - private Long volumeId; - ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -104,10 +94,6 @@ public List getDataTypeList() { return dataTypeList; } - public Long getVolumeId() { - return volumeId; - } - ///////////////////////////////////////////////////// /////////////////// Implementation ////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java index 540660a2f1c8..81d3f1c2b780 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java @@ -33,7 +33,6 @@ public class LibvirtCopyZipFilesCommandWrapper extends CommandWrapper DefaultSupportedDataTypes = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.supported.types", "property, log, configuration, dns, dhcp, vpn, userdata,lb, [iptables], [ifconfig], [routes]", "List of supported diagnostics data type options", false); + "diagnostics.data.supported.types", "config,log,userdata,script", "List of supported diagnostics data type options", false); /** * Global configs below are used to set the diagnostics @@ -107,16 +104,25 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ * which will then be retrieved. */ private static final ConfigKey SsvmDefaultSupportedLogFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.ssvm.supported.log.files", "agent.log, cloud.log", "List of supported diagnostics data log file options for ssvm", false); + "diagnostics.data.ssvm.supported.log.files", "agent.log, cloud.log", "List of supported diagnostics data log file options for the ssvm and cpvm", false); private static final ConfigKey SsvmDefaultSupportedNetworkFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.ssvm.supported.network.files", "[iptables], [ifconfig], [routes]", "List of supported diagnostics data network file options for ssvm", false); + "diagnostics.data.ssvm.supported.network.files", "iptables, ifconfig, route", "List of supported diagnostics data network file options for the ssvm and cpvm", false); - private static final ConfigKey CpvmDefaultSupportedLogFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.cpvm.supported.log.files", " agent.log", "List of supported diagnostics data log file options for cpvm", false); + private static final ConfigKey SsvmDefaultSupportedConfigFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.ssvm.supported.network.files", "agent.properties,consoleproxy.properties", "List of supported diagnostics data network file options for the ssvm and cpvm", false); - private static final ConfigKey CpvmDefaultSupportedNetworkFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.cpvm.supported.log.files", "[iptables], [ifconfig], [routes]", "List of supported diagnostics data network file options for cpvm", false); + private static final ConfigKey VrDefaultSupportedLogFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.vr.supported.log.files", "agent.log, cloud.log, userdata", "List of supported diagnostics data log file options for the VR", false); + + private static final ConfigKey VrDefaultSupportedNetworkFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.vr.supported.log.files", "iptables,ifconfig,route", "List of supported diagnostics data network file options for the VR", false); + + private static final ConfigKey VrDefaultSupportedConfigFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.vr.supported.log.files", "dnsmasq.conf,resolv.conf,haproxy.conf,hosts.conf,dnsmaq-resolv.conf", "List of supported diagnostics data network file options for the VR", false); + + private static final ConfigKey CustomSupportedScripts = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.vr.supported.log.files", "iptables.py, route.py, ifconfig.py", "List of supported diagnostics data network file options for the VR", false); @Override @@ -189,9 +195,6 @@ protected String prepareShellCmd(String cmdType, String ipAddress, String option } } - - - @Override @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "running diagnostics on system vm", async = true) public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { @@ -209,31 +212,23 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(dataType); prepareZipFilesCommand.setAccessDetail(accessDetails); - final Answer zipFilesAnswer = agentManager.easySend(vmInstance.getHostId(), prepareZipFilesCommand); - Answer copyZipAnswer = null; + //final Answer zipFilesAnswer = agentManager.easySend(vmInstance.getHostId(), prepareZipFilesCommand); - if (zipFilesAnswer.getResult()){ - String zipFileDir = zipFilesAnswer.getDetails().replace("\n", ""); - CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), zipFileDir); - copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); + CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), "copy.txt"); + Answer copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); - } - - // Create temporary volume on Primary Storage to store files +// if (zipFilesAnswer.getResult()){ +// String zipFileDir = zipFilesAnswer.getDetails().replace("\n", ""); +// CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), zipFileDir); +// copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); +// +// } - VolumeInfo volumeToAttach = volFactory.getVolume(cmd.getVolumeId()); - Hypervisor.HypervisorType rootDiskHyperType = vmInstance.getHypervisorType(); - StoragePoolVO destPrimaryStorage = storagePoolDao.findById(volumeToAttach.getPoolId()); - try { - VolumeInfo newVolumeOnPrimaryStore = volumeOcherstartor.createVolumeOnPrimaryStorage(vmInstance, volumeToAttach,rootDiskHyperType, destPrimaryStorage); - } catch (NoTransitionException e) { - e.printStackTrace(); - } // DataStoreTO dataStoreTO // CopyToSecondaryStorageCommand toSecondaryStorageCommand = new CopyToSecondaryStorageCommand(); // return copyZipAnswer.getDetails(); - return null; + return copyZipAnswer.getDetails(); } @@ -270,8 +265,8 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { public ConfigKey[] getConfigKeys() { return new ConfigKey[]{EnableGarbageCollector, TmpMgmtDataStoragePath, DataRetrievalTimeout, MaximumFileAgeforGarbageCollection, GarbageCollectionInterval, DiskQuotaPercentageThreshold, DefaultSupportedDataTypes, - SsvmDefaultSupportedLogFiles, SsvmDefaultSupportedNetworkFiles, CpvmDefaultSupportedLogFiles, CpvmDefaultSupportedNetworkFiles, - }; + SsvmDefaultSupportedLogFiles, SsvmDefaultSupportedNetworkFiles, SsvmDefaultSupportedConfigFiles, VrDefaultSupportedConfigFiles, VrDefaultSupportedLogFiles, + VrDefaultSupportedNetworkFiles, CustomSupportedScripts}; } @Override From c15f3061aa4ee21f15eac953b52f352c8e629085 Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Thu, 18 Oct 2018 14:33:43 +0200 Subject: [PATCH 05/11] Fix global settings and request parameters --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../diagnostics/GetDiagnosticsDataCmd.java | 16 ++-- .../diagnostics/DiagnosticsServiceImpl.java | 82 +++++++------------ 3 files changed, 39 insertions(+), 60 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index b7779cb2a8d9..d42e77dc3dcc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -713,6 +713,7 @@ public class ApiConstants { public static final String STDERR = "stderr"; public static final String EXITCODE = "exitcode"; public static final String TARGET_ID = "targetid"; + public static final String ADDITIONAL_FILES = "additionalfiles"; public enum HostDetails { all, capacity, events, stats, min; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java index 157f2104422d..da84b2870b53 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/GetDiagnosticsDataCmd.java @@ -66,17 +66,17 @@ public class GetDiagnosticsDataCmd extends BaseAsyncCmd { description = "The ID of the system VM instance to retrieve diagnostics data files from") private Long id; - @Parameter(name = ApiConstants.DETAILS, + @Parameter(name = ApiConstants.TYPE, type = BaseCmd.CommandType.LIST, collectionType = BaseCmd.CommandType.STRING, - description = "Optional list of additional files to retrieve and must correspond with the diagnostics file dataTypeList. Can be specified as file name or path.") - private List additionalFilesList; + description = "A comma separated list of diagnostics data files to be retrieved. Defaults are taken from global settings if none has been provided.") + private List dataTypeList; - @Parameter(name = ApiConstants.TYPE, + @Parameter(name = ApiConstants.ADDITIONAL_FILES, type = BaseCmd.CommandType.LIST, collectionType = BaseCmd.CommandType.STRING, - description = "The diagnostics data dataTypeList required, examples are dhcp, iptables, dns or log files. Defaults are taken from the database if none has been provided.") - private List dataTypeList; + description = "Optional list of additional files to can be retrieved. Can be specified as file name or path.") + private List additionalFileList; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -86,8 +86,8 @@ public Long getId() { return id; } - public List getAdditionalFilesList() { - return additionalFilesList; + public List getAdditionalFileList() { + return additionalFileList; } public List getDataTypeList() { diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index b93bf592e571..17bf3df441fc 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -92,9 +92,6 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ private static final ConfigKey DiskQuotaPercentageThreshold = new ConfigKey<>("Advanced", Integer.class, "diagnostics.data.disable.threshold", "0.95", "Minimum disk space percentage to initiate diagnostics file retrieval", false); - private static final ConfigKey DefaultSupportedDataTypes = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.supported.types", "config,log,userdata,script", "List of supported diagnostics data type options", false); - /** * Global configs below are used to set the diagnostics * data types applicable for each system vm. @@ -103,26 +100,21 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ * in the system vm and grab output for retrieval, e.g. the output from iptables-save is written to a file * which will then be retrieved. */ - private static final ConfigKey SsvmDefaultSupportedLogFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.ssvm.supported.log.files", "agent.log, cloud.log", "List of supported diagnostics data log file options for the ssvm and cpvm", false); - - private static final ConfigKey SsvmDefaultSupportedNetworkFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.ssvm.supported.network.files", "iptables, ifconfig, route", "List of supported diagnostics data network file options for the ssvm and cpvm", false); - - private static final ConfigKey SsvmDefaultSupportedConfigFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.ssvm.supported.network.files", "agent.properties,consoleproxy.properties", "List of supported diagnostics data network file options for the ssvm and cpvm", false); + private static final ConfigKey SsvmDefaultSupportedFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.ssvm.defaults", "[IPTABLES], [IFCONFIG], [ROUTE], /usr/local/cloud/systemvm/conf/agent.properties," + + " /usr/local/cloud/systemvm/conf/consoleproxy.properties, /var/log/cloud.log", + "List of supported diagnostics data file options for the ssvm", false); - private static final ConfigKey VrDefaultSupportedLogFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.vr.supported.log.files", "agent.log, cloud.log, userdata", "List of supported diagnostics data log file options for the VR", false); + private static final ConfigKey CpvmDefaultSupportedFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.cpvm.defaults", "[IPTABLES], [IFCONFIG], [ROUTE], /usr/local/cloud/systemvm/conf/agent.properties, " + + "/usr/local/cloud/systemvm/conf/consoleproxy.properties, /var/log/cloud.log", + "List of supported diagnostics data file options for the cpvm", false); - private static final ConfigKey VrDefaultSupportedNetworkFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.vr.supported.log.files", "iptables,ifconfig,route", "List of supported diagnostics data network file options for the VR", false); - - private static final ConfigKey VrDefaultSupportedConfigFiles = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.vr.supported.log.files", "dnsmasq.conf,resolv.conf,haproxy.conf,hosts.conf,dnsmaq-resolv.conf", "List of supported diagnostics data network file options for the VR", false); - - private static final ConfigKey CustomSupportedScripts = new ConfigKey<>("Advanced", String.class, - "diagnostics.data.vr.supported.log.files", "iptables.py, route.py, ifconfig.py", "List of supported diagnostics data network file options for the VR", false); + private static final ConfigKey VrDefaultSupportedFiles = new ConfigKey<>("Advanced", String.class, + "diagnostics.data.vr.defaults", "defaults: \"[IPTABLES], [IFCONFIG], [ROUTE], " + + "/etc/dnsmasq.conf, /etc/resolv.conf, /etc/haproxy.conf, /etc/hosts.conf, /etcdnsmaq-resolv.conf, /var/log/cloud.log, " + + "/var/log/routerServiceMonitor.log, /var/log/dnsmasq.log", + "List of supported diagnostics data file options for the VR", false); @Override @@ -200,7 +192,7 @@ protected String prepareShellCmd(String cmdType, String ipAddress, String option public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { final Long vmId = cmd.getId(); List dataType = cmd.getDataTypeList(); - List detail = cmd.getAdditionalFilesList(); + List filesList = cmd.getAdditionalFileList(); final VMInstanceVO vmInstance = instanceDao.findByIdTypes(vmId, VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.DomainRouter, VirtualMachine.Type.SecondaryStorageVm); @@ -210,9 +202,9 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { final Map accessDetails = networkManager.getSystemVMAccessDetails(vmInstance); - PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(dataType); + PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(filesList); prepareZipFilesCommand.setAccessDetail(accessDetails); - //final Answer zipFilesAnswer = agentManager.easySend(vmInstance.getHostId(), prepareZipFilesCommand); + Answer zipFilesAnswer = agentManager.easySend(vmInstance.getHostId(), prepareZipFilesCommand); CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), "copy.txt"); Answer copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); @@ -231,42 +223,28 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { return copyZipAnswer.getDetails(); } + private List prepareFiles(List dataTypeList, List detailFileList, VirtualMachine vm){ + List files = new ArrayList<>(); + VirtualMachine.Type vmType = vm.getType(); + switch (vmType){ + case DomainRouter: + break; + case SecondaryStorageVm: + case ConsoleProxy: + break; + } + return files; -// private String[] getDefaults(VirtualMachine vm) { -// VirtualMachine.Type vmType = vm.getType(); -// String[] supportedTypes = DefaultSupportedDataTypes.value().split(","); -// String[] defaults; -// List filesToRetrieve = new ArrayList<>(); -// switch (vmType){ -// case DomainRouter: -// -// break; -// case ConsoleProxy: -// break; -// case SecondaryStorageVm: -// break; -// -// default: -// throw new CloudRuntimeException("Unsupported VM type"); -// -// } -// -// return supportedTypes; -// } + } -// private boolean isTypeSupported(String type) { -// String[] supportedTypes = DefaultSupportedDataTypes.value().split(","); -// return Arrays.stream(supportedTypes).anyMatch(type::equals); -// } @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{EnableGarbageCollector, TmpMgmtDataStoragePath, DataRetrievalTimeout, - MaximumFileAgeforGarbageCollection, GarbageCollectionInterval, DiskQuotaPercentageThreshold, DefaultSupportedDataTypes, - SsvmDefaultSupportedLogFiles, SsvmDefaultSupportedNetworkFiles, SsvmDefaultSupportedConfigFiles, VrDefaultSupportedConfigFiles, VrDefaultSupportedLogFiles, - VrDefaultSupportedNetworkFiles, CustomSupportedScripts}; + MaximumFileAgeforGarbageCollection, GarbageCollectionInterval, DiskQuotaPercentageThreshold, + SsvmDefaultSupportedFiles, CpvmDefaultSupportedFiles, VrDefaultSupportedFiles}; } @Override From ae4a289e5d73327d3ea185620f4e367f54fc008d Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Fri, 19 Oct 2018 17:07:10 +0200 Subject: [PATCH 06/11] Add unit tests --- .../LibvirtCopyZipFilesCommandWrapper.java | 2 +- .../diagnostics/DiagnosticsServiceImpl.java | 82 ++++++++----- .../DiagnosticsServiceImplTest.java | 109 +++++++++++++----- .../cloud/bin/retrieve_diagnostics_files.py | 12 +- test/integration/smoke/test_diagnostics.py | 39 ++++++- 5 files changed, 177 insertions(+), 67 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java index 81d3f1c2b780..6c7d1f24df4d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyZipFilesCommandWrapper.java @@ -41,7 +41,7 @@ public Answer execute(CopyZipFilesCommand command, LibvirtComputingResource libv boolean success = true; String details = "Copying zip files: " + vmIp + ", file: " + filename; LOGGER.info(details); - String cmdLine = String.format("/usr/bin/scp -P 3922 -o StrictHostKeyChecking=no -i %s root@%s:/root/%s /root/%s", permKey, vmIp, filename, filename); + String cmdLine = String.format("/usr/bin/scp -P 3922 -o StrictHostKeyChecking=no -i %s root@%s:%s %s", permKey, vmIp, filename, filename); Script2 cmd = new Script2("/bin/bash", LOGGER); cmd.add("-c"); cmd.add(cmdLine); diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index 17bf3df441fc..fa6790e58899 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -18,7 +18,6 @@ package org.apache.cloudstack.diagnostics; -import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -49,6 +48,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cxf.common.util.CollectionUtils; import org.apache.log4j.Logger; @@ -56,9 +56,6 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ private static final Logger LOGGER = Logger.getLogger(DiagnosticsServiceImpl.class); - protected static final int DefaultDomRSshPort = 3922; - final File permKey = new File("/root/.ssh/id_rsa.cloud"); - @Inject private AgentManager agentManager; @Inject @@ -191,51 +188,78 @@ protected String prepareShellCmd(String cmdType, String ipAddress, String option @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "running diagnostics on system vm", async = true) public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { final Long vmId = cmd.getId(); - List dataType = cmd.getDataTypeList(); - List filesList = cmd.getAdditionalFileList(); final VMInstanceVO vmInstance = instanceDao.findByIdTypes(vmId, VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.DomainRouter, VirtualMachine.Type.SecondaryStorageVm); - if (vmInstance == null) { throw new InvalidParameterValueException("Unable to find a system vm with id " + vmId); } final Map accessDetails = networkManager.getSystemVMAccessDetails(vmInstance); - PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(filesList); + List fileList = prepareFiles(cmd.getDataTypeList(), cmd.getAdditionalFileList(), vmInstance); + + PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(fileList); prepareZipFilesCommand.setAccessDetail(accessDetails); Answer zipFilesAnswer = agentManager.easySend(vmInstance.getHostId(), prepareZipFilesCommand); - CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), "copy.txt"); - Answer copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); - -// if (zipFilesAnswer.getResult()){ -// String zipFileDir = zipFilesAnswer.getDetails().replace("\n", ""); -// CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), zipFileDir); -// copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); -// -// } + // Copy files to ssvm + String zipFileDir = ""; + if (zipFilesAnswer.getResult()){ + zipFileDir = zipFilesAnswer.getDetails().replace("\n", ""); + } + CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), zipFileDir); + Answer copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); -// DataStoreTO dataStoreTO -// CopyToSecondaryStorageCommand toSecondaryStorageCommand = new CopyToSecondaryStorageCommand(); -// return copyZipAnswer.getDetails(); return copyZipAnswer.getDetails(); } - private List prepareFiles(List dataTypeList, List detailFileList, VirtualMachine vm){ - List files = new ArrayList<>(); + // Prepare List of files to be retrieved from system vm or VR + protected List prepareFiles(List dataTypeList, List additionalFileList, VirtualMachine vm){ + List filesList = new ArrayList<>(); VirtualMachine.Type vmType = vm.getType(); - switch (vmType){ - case DomainRouter: - break; + String[] defaultFiles; + if (CollectionUtils.isEmpty(dataTypeList)){ + switch (vmType){ + case DomainRouter: + defaultFiles = VrDefaultSupportedFiles.value().split(","); + for (String file: defaultFiles) { + filesList.add(file); + } + break; + case SecondaryStorageVm: + defaultFiles = SsvmDefaultSupportedFiles.value().split(","); + for (String file: defaultFiles) { + filesList.add(file); + } + break; + case ConsoleProxy: + defaultFiles = CpvmDefaultSupportedFiles.value().split(","); + for (String file: defaultFiles) { + filesList.add(file); + } + break; + default: + throw new CloudRuntimeException("Unsupported vm type for retrieve diagnostics data: "); + } + } else { + for (String file: dataTypeList) { + filesList.add(file); + } + } + if (!CollectionUtils.isEmpty(additionalFileList)){ + for (String extraFile: additionalFileList) { + filesList.add(extraFile); - case SecondaryStorageVm: - case ConsoleProxy: - break; + } } - return files; + return filesList; + } + //TODO + protected String copyToSecondaryStorage(){ + String url = ""; + return url; } diff --git a/server/src/test/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImplTest.java index d85c5434d9a1..3cedaea6fee2 100644 --- a/server/src/test/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImplTest.java @@ -18,6 +18,13 @@ // package org.apache.cloudstack.diagnostics; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import com.cloud.agent.AgentManager; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.exception.InvalidParameterValueException; @@ -39,9 +46,6 @@ import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; -import java.util.HashMap; -import java.util.Map; - @RunWith(MockitoJUnitRunner.class) public class DiagnosticsServiceImplTest extends TestCase { @@ -54,21 +58,21 @@ public class DiagnosticsServiceImplTest extends TestCase { @Mock private DiagnosticsCommand command; @Mock - private VMInstanceVO instanceVO; + private VMInstanceVO vmInstance; @Mock private VirtualMachineManager vmManager; @Mock private NetworkOrchestrationService networkManager; @InjectMocks - private DiagnosticsServiceImpl diagnosticsService = new DiagnosticsServiceImpl(); + private DiagnosticsServiceImpl serviceImpl = new DiagnosticsServiceImpl(); @Before public void setUp() throws Exception { Mockito.when(diagnosticsCmd.getId()).thenReturn(1L); Mockito.when(diagnosticsCmd.getType()).thenReturn(DiagnosticsType.PING); Mockito.when(instanceDao.findByIdTypes(Mockito.anyLong(), Mockito.any(VirtualMachine.Type.class), - Mockito.any(VirtualMachine.Type.class), Mockito.any(VirtualMachine.Type.class))).thenReturn(instanceVO); + Mockito.any(VirtualMachine.Type.class), Mockito.any(VirtualMachine.Type.class))).thenReturn(vmInstance); } @@ -77,7 +81,7 @@ public void tearDown() throws Exception { Mockito.reset(diagnosticsCmd); Mockito.reset(agentManager); Mockito.reset(instanceDao); - Mockito.reset(instanceVO); + Mockito.reset(vmInstance); Mockito.reset(command); } @@ -102,7 +106,7 @@ public void testRunDiagnosticsCommandTrue() throws Exception { Mockito.when(agentManager.easySend(Mockito.anyLong(), Mockito.any(DiagnosticsCommand.class))).thenReturn(new DiagnosticsAnswer(command, true, details)); - Map detailsMap = diagnosticsService.runDiagnosticsCommand(diagnosticsCmd); + Map detailsMap = serviceImpl.runDiagnosticsCommand(diagnosticsCmd); String stdout = "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.\n" + "64 bytes from 8.8.8.8: icmp_seq=1 ttl=125 time=7.88 ms\n" + @@ -141,7 +145,7 @@ public void testRunDiagnosticsCommandFalse() throws Exception { "4 packets transmitted, 0 packets received, 100% packet loss"; Mockito.when(agentManager.easySend(Mockito.anyLong(), Mockito.any(DiagnosticsCommand.class))).thenReturn(new DiagnosticsAnswer(command, true, details)); - Map detailsMap = diagnosticsService.runDiagnosticsCommand(diagnosticsCmd); + Map detailsMap = serviceImpl.runDiagnosticsCommand(diagnosticsCmd); assertEquals(3, detailsMap.size()); assertEquals("Mismatch between actual and expected STDERR", "", detailsMap.get(ApiConstants.STDERR)); @@ -155,7 +159,7 @@ public void testRunDiagnosticsThrowsInvalidParamException() throws Exception { Mockito.when(instanceDao.findByIdTypes(Mockito.anyLong(), Mockito.any(VirtualMachine.Type.class), Mockito.any(VirtualMachine.Type.class), Mockito.any(VirtualMachine.Type.class))).thenReturn(null); - diagnosticsService.runDiagnosticsCommand(diagnosticsCmd); + serviceImpl.runDiagnosticsCommand(diagnosticsCmd); } @Test(expected = CloudRuntimeException.class) @@ -166,31 +170,80 @@ public void testVMControlIPisNull() throws Exception { accessDetailsMap.put(NetworkElementCommand.ROUTER_IP, null); Mockito.when(networkManager.getSystemVMAccessDetails(Mockito.any(VMInstanceVO.class))).thenReturn(accessDetailsMap); - diagnosticsService.runDiagnosticsCommand(diagnosticsCmd); + serviceImpl.runDiagnosticsCommand(diagnosticsCmd); } @Test public void testInvalidCharsInParams() throws Exception { - assertFalse(diagnosticsService.hasValidChars("'\\''")); - assertFalse(diagnosticsService.hasValidChars("-I eth0 &")); - assertFalse(diagnosticsService.hasValidChars("-I eth0 ;")); - assertFalse(diagnosticsService.hasValidChars(" &2 > ")); - assertFalse(diagnosticsService.hasValidChars(" &2 >> ")); - assertFalse(diagnosticsService.hasValidChars(" | ")); - assertFalse(diagnosticsService.hasValidChars("|")); - assertFalse(diagnosticsService.hasValidChars(",")); + assertFalse(serviceImpl.hasValidChars("'\\''")); + assertFalse(serviceImpl.hasValidChars("-I eth0 &")); + assertFalse(serviceImpl.hasValidChars("-I eth0 ;")); + assertFalse(serviceImpl.hasValidChars(" &2 > ")); + assertFalse(serviceImpl.hasValidChars(" &2 >> ")); + assertFalse(serviceImpl.hasValidChars(" | ")); + assertFalse(serviceImpl.hasValidChars("|")); + assertFalse(serviceImpl.hasValidChars(",")); } @Test public void testValidCharsInParams() throws Exception { - assertTrue(diagnosticsService.hasValidChars("")); - assertTrue(diagnosticsService.hasValidChars(".")); - assertTrue(diagnosticsService.hasValidChars(" ")); - assertTrue(diagnosticsService.hasValidChars("-I eth0 www.google.com")); - assertTrue(diagnosticsService.hasValidChars(" ")); - assertTrue(diagnosticsService.hasValidChars(" -I cloudbr0 --sport ")); - assertTrue(diagnosticsService.hasValidChars(" --back -m20 ")); - assertTrue(diagnosticsService.hasValidChars("-c 5 -4")); - assertTrue(diagnosticsService.hasValidChars("-c 5 -4 -AbDfhqUV")); + assertTrue(serviceImpl.hasValidChars("")); + assertTrue(serviceImpl.hasValidChars(".")); + assertTrue(serviceImpl.hasValidChars(" ")); + assertTrue(serviceImpl.hasValidChars("-I eth0 www.google.com")); + assertTrue(serviceImpl.hasValidChars(" ")); + assertTrue(serviceImpl.hasValidChars(" -I cloudbr0 --sport ")); + assertTrue(serviceImpl.hasValidChars(" --back -m20 ")); + assertTrue(serviceImpl.hasValidChars("-c 5 -4")); + assertTrue(serviceImpl.hasValidChars("-c 5 -4 -AbDfhqUV")); + } + + @Test + public void testPrepareFileListFromInput() throws Exception { + List dataTypeList = new ArrayList<>(); + dataTypeList.add("/var/log/auth.log"); + dataTypeList.add("/etc/dnsmasq.conf"); + dataTypeList.add("[IPTABLES]"); + dataTypeList.add("[IFCONFIG]"); + + Mockito.when(vmInstance.getType()).thenReturn(VirtualMachine.Type.ConsoleProxy); + + List additionalList = new ArrayList<>(); + + List files = serviceImpl.prepareFiles(dataTypeList, additionalList, vmInstance); + + assertEquals(dataTypeList, files); } + + @Test + public void testPrepareFilesWithAdditionalFiles() throws Exception { + List dataTypeList = new ArrayList<>(); + dataTypeList.add("/var/log/auth.log"); + dataTypeList.add("/etc/dnsmasq.conf"); + dataTypeList.add("[IPTABLES]"); + dataTypeList.add("[IFCONFIG]"); + + List additionalFiles = new ArrayList<>(); + additionalFiles.add("/etc/hosts"); + additionalFiles.add("/etc/resolv.conf"); + + List allFiles = Stream.concat(dataTypeList.stream(), additionalFiles.stream()) + .distinct() + .collect(Collectors.toList()); + + List files = serviceImpl.prepareFiles(dataTypeList, additionalFiles, vmInstance); + assertEquals(files, allFiles); + } + +// @Test +// public void testPrepareFilesFromDefaults() throws Exception { +// List dataTypeList = new ArrayList<>(); +// List addList = new ArrayList<>(); +// List files = serviceImpl.prepareFiles(dataTypeList, addList, vmInstance); +// +// Field cpvmDefaults = serviceImpl.getClass().getDeclaredField("CpvmDefaultSupportedFiles"); +// cpvmDefaults.setAccessible(true); +// String defaults = cpvmDefaults.getName(); +// cpvmDefaults. +// } } \ No newline at end of file diff --git a/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py b/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py index 14da25d8d925..d80c2fe71f88 100755 --- a/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py +++ b/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py @@ -19,20 +19,16 @@ import sys import zipfile import time -import tarfile - +# Create zip archive and append files for retrieval def zip_files(file_name): compression = zipfile.ZIP_DEFLATED time_str = time.strftime("%Y%m%d-%H%M%S") - zf_name = 'zipfile_' + time_str + '.tar.gz' - # zf = zipfile.ZipFile(zf_name, mode='w') - zf = tarfile.open(zf_name, mode='w') + zf_name = '/root/diagnostics_files_' + time_str + '.tar.gz' + zf = zipfile.ZipFile(zf_name, 'w', compression) try: for f in file_name: - # zf.write(f, compress_type=compression) - zf.add(f) - + zf.write(f, f[f.rfind('/')+1:]) except RuntimeError as e: print "File not found" finally: diff --git a/test/integration/smoke/test_diagnostics.py b/test/integration/smoke/test_diagnostics.py index 6364d83eeee8..26ffda4d549c 100644 --- a/test/integration/smoke/test_diagnostics.py +++ b/test/integration/smoke/test_diagnostics.py @@ -19,7 +19,7 @@ # Import Local Modules from marvin.codes import FAILED from marvin.cloudstackTestCase import cloudstackTestCase -from marvin.cloudstackAPI import runDiagnostics +from marvin.cloudstackAPI import runDiagnostics, getDiagnosticsData from marvin.lib.utils import (cleanup_resources) from marvin.lib.base import (Account, ServiceOffering, @@ -537,3 +537,40 @@ def test_12_traceroute_in_cpvm(self): cmd_response.exitcode, 'Failed to run remote Traceroute in CPVM' ) + + + ''' + Add Get Diagnostics data BVT + ''' + @attr(tags=["advanced", "advancedns", "ssh", "smoke","retrieve"], required_hardware="true") + def test_13_retrieve_vr_default_files(self): + list_router_response = list_routers( + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) + self.assertEqual( + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) + router = list_router_response[0] + self.debug('Setting up SSVM with ID %s' % router.id) + + cmd = getDiagnosticsData.getDiagnosticsDataCmd() + cmd.targetid = router.id + response = self.apiclient.getDiagnosticsData(cmd) + pass + + @attr(tags=["advanced", "advancedns", "ssh", "smoke", "retrieve"], required_hardware="true") + def test_14_retrieve_ssvm_default_files(self): + pass + + @attr(tags=["advanced", "advancedns", "ssh", "smoke", "retrieve"], required_hardware="true") + def test_15_retrieve_cpvm_default_files(self): + pass + + + + + From 6abee814f85699825a92af225fbb387570c9b1b1 Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Mon, 29 Oct 2018 19:27:14 +0200 Subject: [PATCH 07/11] Add Java BitTorrent library to use for distributed file sharing between system VMs and secondary storage https://github.com/atomashpolskiy/bt --- .../diagnostics/DiagnosticsServiceImpl.java | 105 +++++++++++------- 1 file changed, 64 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index fa6790e58899..f4ce27a6e706 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -19,8 +19,10 @@ import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.regex.Pattern; import javax.inject.Inject; @@ -28,10 +30,12 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.routing.NetworkElementCommand; +import com.cloud.configuration.Config; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.EncryptionUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; @@ -40,16 +44,19 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.apache.cloudstack.api.command.admin.diagnostics.GetDiagnosticsDataCmd; import org.apache.cloudstack.api.command.admin.diagnostics.RunDiagnosticsCmd; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; import org.apache.cxf.common.util.CollectionUtils; import org.apache.log4j.Logger; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; public class DiagnosticsServiceImpl extends ManagerBase implements PluggableService, DiagnosticsService, Configurable { @@ -65,11 +72,8 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ @Inject private NetworkOrchestrationService networkManager; @Inject - VolumeOrchestrationService volumeOcherstartor; - @Inject - private VolumeDataFactory volFactory; - @Inject - private PrimaryDataStoreDao storagePoolDao; + private ConfigurationDao configDao; + private static final ConfigKey EnableGarbageCollector = new ConfigKey<>("Advanced", Boolean.class, "diagnostics.data.gc.enable", "true", "enable the diagnostics data files garbage collector", true); @@ -114,6 +118,30 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ "List of supported diagnostics data file options for the VR", false); + + protected boolean hasValidChars(String optionalArgs) { + if (Strings.isNullOrEmpty(optionalArgs)) { + return true; + } else { + final String regex = "^[\\w\\-\\s.]+$"; + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(optionalArgs).find(); + } + } + + protected String prepareShellCmd(String cmdType, String ipAddress, String optionalParams) { + final String CMD_TEMPLATE = String.format("%s %s", cmdType, ipAddress); + if (Strings.isNullOrEmpty(optionalParams)) { + return CMD_TEMPLATE; + } else { + if (hasValidChars(optionalParams)) { + return String.format("%s %s", CMD_TEMPLATE, optionalParams); + } else { + return null; + } + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "running diagnostics on system vm", async = true) public Map runDiagnosticsCommand(final RunDiagnosticsCmd cmd) { @@ -161,27 +189,26 @@ public Map runDiagnosticsCommand(final RunDiagnosticsCmd cmd) { } } - protected boolean hasValidChars(String optionalArgs) { - if (Strings.isNullOrEmpty(optionalArgs)) { - return true; - } else { - final String regex = "^[\\w\\-\\s.]+$"; - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(optionalArgs).find(); - } - } + private String getSecondaryStoragePostUploadParams(String b64encodedData){ + Map uploadParams = new HashMap<>(); + String ssvmUrlDomain = configDao.getValue(Config.SecStorageSecureCopyCert.key()); + String uuid = UUID.randomUUID().toString(); + List vm = instanceDao.listByTypes(VirtualMachine.Type.SecondaryStorageVm); + String url = ImageStoreUtil.generatePostUploadUrl(ssvmUrlDomain,"172.20.20.11", uuid ); - protected String prepareShellCmd(String cmdType, String ipAddress, String optionalParams) { - final String CMD_TEMPLATE = String.format("%s %s", cmdType, ipAddress); - if (Strings.isNullOrEmpty(optionalParams)) { - return CMD_TEMPLATE; - } else { - if (hasValidChars(optionalParams)) { - return String.format("%s %s", CMD_TEMPLATE, optionalParams); - } else { - return null; - } - } + DateTime currentTime = new DateTime(DateTimeZone.UTC); + String expires = currentTime.plusMinutes(3).toString(); + + // Get Key + String key = configDao.getValue(Config.SSVMPSK.key()); + + // encode metadata using the post upload config ssh key + Gson gson = new GsonBuilder().create(); + String metadata = EncryptionUtil.encodeData(gson.toJson(b64encodedData), key); + + // Compute signature on url, expiry and metadata + String signature = EncryptionUtil.generateSignature(metadata + url + expires, key); + return url; } @Override @@ -198,20 +225,22 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { List fileList = prepareFiles(cmd.getDataTypeList(), cmd.getAdditionalFileList(), vmInstance); - PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(fileList); + //PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(fileList, secondaryStorageUrl); + String dummyUrl = "http://172.20.20.11"; + PrepareFilesCommand prepareZipFilesCommand = new PrepareFilesCommand(fileList, dummyUrl); prepareZipFilesCommand.setAccessDetail(accessDetails); Answer zipFilesAnswer = agentManager.easySend(vmInstance.getHostId(), prepareZipFilesCommand); - // Copy files to ssvm - String zipFileDir = ""; + // Retrieve zip file is b64 encoded payload + String b64EncodedPayload = ""; if (zipFilesAnswer.getResult()){ - zipFileDir = zipFilesAnswer.getDetails().replace("\n", ""); + b64EncodedPayload = zipFilesAnswer.getDetails(); } - CopyZipFilesCommand copyZipCommand = new CopyZipFilesCommand(accessDetails.get("Control"), zipFileDir); - Answer copyZipAnswer = agentManager.easySend(vmInstance.getHostId(), copyZipCommand); + String secondaryStorageUrl = getSecondaryStoragePostUploadParams(b64EncodedPayload); + - return copyZipAnswer.getDetails(); + return secondaryStorageUrl; } // Prepare List of files to be retrieved from system vm or VR @@ -256,12 +285,6 @@ protected List prepareFiles(List dataTypeList, List addi return filesList; } - //TODO - protected String copyToSecondaryStorage(){ - String url = ""; - return url; - } - @Override From ef470fd770589d55651f0667e72606ef5c8f79ae Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Mon, 29 Oct 2018 19:28:19 +0200 Subject: [PATCH 08/11] Add Java BitTorrent library to use for distributed file sharing between system VMs and secondary storage. --- .../VirtualRoutingResource.java | 4 ++- .../CopyToSecondaryStorageCommand.java | 4 +-- .../diagnostics/PrepareFilesCommand.java | 8 ++++- server/pom.xml | 5 +++ .../cloud/bin/retrieve_diagnostics_files.py | 31 ++++++++++++++----- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 10a06cb9126e..2df0e59240a4 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -312,7 +312,9 @@ private Answer execute(DiagnosticsCommand cmd) { private Answer execute(PrepareFilesCommand cmd) { String fileList = String.join(" ", cmd.getFilesToRetrieveList()); - final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.RETRIEVE_DIAGNOSTICS, fileList); + String url = cmd.getUrl(); + String cmdLine = String.format("-f %s -u %s", fileList, url); + final ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), VRScripts.RETRIEVE_DIAGNOSTICS, cmdLine); if (result.isSuccess()){ return new PrepareFilesAnswer(cmd, true, result.getDetails()); } diff --git a/core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java b/core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java index 550065fa274e..8f3daea28962 100644 --- a/core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java +++ b/core/src/main/java/org/apache/cloudstack/diagnostics/CopyToSecondaryStorageCommand.java @@ -19,10 +19,10 @@ package org.apache.cloudstack.diagnostics; -import com.cloud.agent.api.storage.StorageNfsVersionCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.to.DataStoreTO; -public class CopyToSecondaryStorageCommand extends StorageNfsVersionCommand { +public class CopyToSecondaryStorageCommand extends NetworkElementCommand { private DataStoreTO destStore; diff --git a/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java b/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java index e8f8fd74b9ed..4439f6888c87 100644 --- a/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java +++ b/core/src/main/java/org/apache/cloudstack/diagnostics/PrepareFilesCommand.java @@ -25,9 +25,15 @@ public class PrepareFilesCommand extends NetworkElementCommand { List filesToRetrieveList; + String url; - public PrepareFilesCommand(List filesToRetrieve) { + public PrepareFilesCommand(List filesToRetrieve, String url) { this.filesToRetrieveList = filesToRetrieve; + this.url = url; + } + + public String getUrl() { + return url; } public List getFilesToRetrieveList() { diff --git a/server/pom.xml b/server/pom.xml index e24615089329..7325c4085439 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -27,6 +27,11 @@ 4.12.0.0-SNAPSHOT + + com.github.atomashpolskiy + bt-core + 1.7 + commons-io commons-io diff --git a/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py b/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py index d80c2fe71f88..f7b29e072c27 100755 --- a/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py +++ b/systemvm/debian/opt/cloud/bin/retrieve_diagnostics_files.py @@ -16,26 +16,43 @@ # specific language governing permissions and limitations # under the License. -import sys +import argparse +import base64 import zipfile import time +parser = argparse.ArgumentParser(description="List of files to retrive and secondary storage url for zip file upload") +parser.add_argument("-f", "--files", help="List of diagnostic files to be retrieved") +parser.add_argument("-u", "--url", help="Secondary storage url") + +args = parser.parse_args() +file_list = args.files +url = args.url + + # Create zip archive and append files for retrieval -def zip_files(file_name): +def zip_files(files): compression = zipfile.ZIP_DEFLATED time_str = time.strftime("%Y%m%d-%H%M%S") zf_name = '/root/diagnostics_files_' + time_str + '.tar.gz' zf = zipfile.ZipFile(zf_name, 'w', compression) try: - for f in file_name: - zf.write(f, f[f.rfind('/')+1:]) + for f in files: + zf.write(f, f[f.rfind('/') + 1:]) except RuntimeError as e: print "File not found" finally: zf.close() - print zf_name.strip() + print b64encode_file(zf_name) + + +# B64 encode zip file to string +def b64encode_file(file): + with open(file, 'rb') as fin, open("output.zip.b64", 'w') as fout: + base64.b64decode(bytes) + return repr(open("output.zip.b64").read()) if __name__ == '__main__': - file_names = sys.argv[1:] - zip_files(file_names) + zip_files(file_list) + From 021731d2358c2f63bcab85f8ec065cdf5527bde9 Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Thu, 1 Nov 2018 12:33:19 +0200 Subject: [PATCH 09/11] Build system vm with python-pip install --- .../systemvmtemplate/scripts/install_systemvm_packages.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index 585b38d4eb00..da4ca845e6c2 100644 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -70,7 +70,8 @@ function install_packages() { radvd \ sharutils genisoimage aria2 \ strongswan libcharon-extra-plugins libstrongswan-extra-plugins \ - virt-what open-vm-tools qemu-guest-agent hyperv-daemons + virt-what open-vm-tools qemu-guest-agent hyperv-daemons \ + python-pip apt-get -q -y -t stretch-backports install nftables From 3ccc14a4159dae62cf7c4e759efa7c6d8e88a46e Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Mon, 5 Nov 2018 10:56:42 +0200 Subject: [PATCH 10/11] Remove duplicated filed epSelector --- .../cloud/template/TemplateManagerImpl.java | 8 ++--- .../diagnostics/DiagnosticsServiceImpl.java | 29 +++++++++++++++---- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index dc57a8a9ef36..271f44f961f7 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -261,7 +261,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject private SnapshotDataFactory _snapshotFactory; @Inject - StorageStrategyFactory _storageStrategyFactory; + private StorageStrategyFactory _storageStrategyFactory; @Inject private TemplateService _tmpltSvr; @Inject @@ -269,7 +269,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject private VolumeOrchestrationService _volumeMgr; @Inject - private EndPointSelector _epSelector; + private EndPointSelector epSelector; @Inject private UserVmJoinDao _userVmJoinDao; @Inject @@ -288,8 +288,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject private StorageCacheManager cacheMgr; - @Inject - private EndPointSelector selector; private TemplateAdapter getAdapter(HypervisorType type) { @@ -684,7 +682,7 @@ public VMTemplateStoragePoolVO prepareTemplateForCreate(VMTemplateVO templ, Stor @Override public String getChecksum(DataStore store, String templatePath, String algorithm) { - EndPoint ep = _epSelector.select(store); + EndPoint ep = epSelector.select(store); ComputeChecksumCommand cmd = new ComputeChecksumCommand(store.getTO(), templatePath, algorithm); Answer answer = null; if (ep == null) { diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index f4ce27a6e706..e47fee49579c 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -31,10 +31,13 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.configuration.Config; +import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; import com.cloud.hypervisor.Hypervisor; +import com.cloud.server.ManagementServer; +import com.cloud.user.AccountManager; import com.cloud.utils.EncryptionUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; @@ -73,6 +76,12 @@ public class DiagnosticsServiceImpl extends ManagerBase implements PluggableServ private NetworkOrchestrationService networkManager; @Inject private ConfigurationDao configDao; + @Inject + private DataCenterDao centerDao; + @Inject + private AccountManager accountMgr; + @Inject + private ManagementServer msServer; private static final ConfigKey EnableGarbageCollector = new ConfigKey<>("Advanced", Boolean.class, @@ -193,7 +202,7 @@ private String getSecondaryStoragePostUploadParams(String b64encodedData){ Map uploadParams = new HashMap<>(); String ssvmUrlDomain = configDao.getValue(Config.SecStorageSecureCopyCert.key()); String uuid = UUID.randomUUID().toString(); - List vm = instanceDao.listByTypes(VirtualMachine.Type.SecondaryStorageVm); + String url = ImageStoreUtil.generatePostUploadUrl(ssvmUrlDomain,"172.20.20.11", uuid ); DateTime currentTime = new DateTime(DateTimeZone.UTC); @@ -211,6 +220,18 @@ private String getSecondaryStoragePostUploadParams(String b64encodedData){ return url; } + /** + * Assumption is that there should be at least 1 ssvm running within a zone + * @return + */ + private VMInstanceVO getSsvmInZone(){ + List vmList = instanceDao.listByTypes(VirtualMachine.Type.SecondaryStorageVm); + if(org.apache.commons.collections.CollectionUtils.isEmpty(vmList)){ + throw new CloudRuntimeException("No ssvm found in Zone"); + } + return vmList.get(0); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "running diagnostics on system vm", async = true) public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { @@ -238,11 +259,11 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { } String secondaryStorageUrl = getSecondaryStoragePostUploadParams(b64EncodedPayload); - - return secondaryStorageUrl; } + + // Prepare List of files to be retrieved from system vm or VR protected List prepareFiles(List dataTypeList, List additionalFileList, VirtualMachine vm){ List filesList = new ArrayList<>(); @@ -285,8 +306,6 @@ protected List prepareFiles(List dataTypeList, List addi return filesList; } - - @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{EnableGarbageCollector, TmpMgmtDataStoragePath, DataRetrievalTimeout, From 5126c6815476ba62f5cda27dfc135a2c210bf2fc Mon Sep 17 00:00:00 2001 From: Dingane Hlaluku Date: Mon, 5 Nov 2018 11:18:32 +0200 Subject: [PATCH 11/11] Remove torrent frameworks --- server/pom.xml | 5 ----- .../cloudstack/diagnostics/DiagnosticsServiceImpl.java | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/server/pom.xml b/server/pom.xml index 7325c4085439..e24615089329 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -27,11 +27,6 @@ 4.12.0.0-SNAPSHOT - - com.github.atomashpolskiy - bt-core - 1.7 - commons-io commons-io diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index e47fee49579c..bd584f4e7496 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -202,7 +202,7 @@ private String getSecondaryStoragePostUploadParams(String b64encodedData){ Map uploadParams = new HashMap<>(); String ssvmUrlDomain = configDao.getValue(Config.SecStorageSecureCopyCert.key()); String uuid = UUID.randomUUID().toString(); - + List vm = instanceDao.listByTypes(VirtualMachine.Type.SecondaryStorageVm); String url = ImageStoreUtil.generatePostUploadUrl(ssvmUrlDomain,"172.20.20.11", uuid ); DateTime currentTime = new DateTime(DateTimeZone.UTC); @@ -262,8 +262,6 @@ public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { return secondaryStorageUrl; } - - // Prepare List of files to be retrieved from system vm or VR protected List prepareFiles(List dataTypeList, List additionalFileList, VirtualMachine vm){ List filesList = new ArrayList<>();