diff --git a/core/src/main/java/com/cloud/agent/api/RebootCommand.java b/core/src/main/java/com/cloud/agent/api/RebootCommand.java index eecf7f635d38..74ed76283514 100644 --- a/core/src/main/java/com/cloud/agent/api/RebootCommand.java +++ b/core/src/main/java/com/cloud/agent/api/RebootCommand.java @@ -19,8 +19,11 @@ package com.cloud.agent.api; +import com.cloud.agent.api.to.VirtualMachineTO; + public class RebootCommand extends Command { String vmName; + VirtualMachineTO vm; protected boolean executeInSequence = false; protected RebootCommand() { @@ -35,6 +38,14 @@ public String getVmName() { return this.vmName; } + public void setVirtualMachine(VirtualMachineTO vm) { + this.vm = vm; + } + + public VirtualMachineTO getVirtualMachine() { + return vm; + } + @Override public boolean executeInSequence() { return this.executeInSequence; diff --git a/core/src/main/java/com/cloud/agent/api/SecurityGroupRulesCmd.java b/core/src/main/java/com/cloud/agent/api/SecurityGroupRulesCmd.java index 1d3d15439927..ea4ab96c5a7c 100644 --- a/core/src/main/java/com/cloud/agent/api/SecurityGroupRulesCmd.java +++ b/core/src/main/java/com/cloud/agent/api/SecurityGroupRulesCmd.java @@ -30,12 +30,13 @@ import org.apache.log4j.Logger; import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.utils.net.NetUtils; public class SecurityGroupRulesCmd extends Command { private static final String CIDR_LENGTH_SEPARATOR = "/"; - private static final char RULE_TARGET_SEPARATOR = ','; - private static final char RULE_COMMAND_SEPARATOR = ';'; + public static final char RULE_TARGET_SEPARATOR = ','; + public static final char RULE_COMMAND_SEPARATOR = ';'; protected static final String EGRESS_RULE = "E:"; protected static final String INGRESS_RULE = "I:"; private static final Logger LOGGER = Logger.getLogger(SecurityGroupRulesCmd.class); @@ -51,6 +52,7 @@ public class SecurityGroupRulesCmd extends Command { private List ingressRuleSet; private List egressRuleSet; private final List secIps; + private VirtualMachineTO vmTO; public static class IpPortAndProto { private final String proto; @@ -252,6 +254,14 @@ public Long getVmId() { return vmId; } + public void setVmTO(VirtualMachineTO vmTO) { + this.vmTO = vmTO; + } + + public VirtualMachineTO getVmTO() { + return vmTO; + } + /** * used for logging * @return the number of Cidrs in the in and egress rule sets for this security group rules command. diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManager.java b/engine/components-api/src/main/java/com/cloud/network/security/SecurityGroupManager.java similarity index 95% rename from server/src/main/java/com/cloud/network/security/SecurityGroupManager.java rename to engine/components-api/src/main/java/com/cloud/network/security/SecurityGroupManager.java index 16d8ba617bca..ffca4bb013b6 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/security/SecurityGroupManager.java @@ -53,4 +53,6 @@ public interface SecurityGroupManager { SecurityGroup getSecurityGroup(String name, long accountId); boolean isVmMappedToDefaultSecurityGroup(long vmId); + + void scheduleRulesetUpdateToHosts(List affectedVms, boolean updateSeqno, Long delayMs); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 2c4079cd3fde..8e52c38902cc 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; @@ -166,6 +167,7 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.router.VirtualRouter; +import com.cloud.network.security.SecurityGroupManager; import com.cloud.offering.DiskOffering; import com.cloud.offering.DiskOfferingInfo; import com.cloud.offering.NetworkOffering; @@ -333,6 +335,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private NetworkOfferingDetailsDao networkOfferingDetailsDao; @Inject private NetworkDetailsDao networkDetailsDao; + @Inject + private SecurityGroupManager _securityGroupManager; VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); @@ -3140,11 +3144,18 @@ private void orchestrateReboot(final String vmUuid, final Map affectedVms = new ArrayList(); + affectedVms.add(vm.getId()); + _securityGroupManager.scheduleRulesetUpdateToHosts(affectedVms, true, null); + } return; } s_logger.info("Unable to reboot VM " + vm + " on " + dest.getHost() + " due to " + (rebootAnswer == null ? " no reboot answer" : rebootAnswer.getDetails())); @@ -3154,6 +3165,29 @@ private void orchestrateReboot(final String vmUuid, final Map nics = _nicsDao.listByVmId(profile.getId()); + Collections.sort(nics, new Comparator() { + @Override + public int compare(NicVO nic1, NicVO nic2) { + Long nicId1 = Long.valueOf(nic1.getDeviceId()); + Long nicId2 = Long.valueOf(nic2.getDeviceId()); + return nicId1.compareTo(nicId2); + } + }); + for (final NicVO nic : nics) { + final Network network = _networkModel.getNetwork(nic.getNetworkId()); + final NicProfile nicProfile = + new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), null, _networkModel.isSecurityGroupSupportedInNetwork(network), + _networkModel.getNetworkTag(profile.getHypervisorType(), network)); + profile.addNic(nicProfile); + } + final VirtualMachineTO to = toVmTO(profile); + return to; + } + public Command cleanup(final VirtualMachine vm, Map dpdkInterfaceMapping) { StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false); cmd.setControlIp(getControlNicIpForVM(vm)); @@ -3670,7 +3704,7 @@ private boolean orchestrateRemoveNicFromVm(final VirtualMachine vm, final Nic ni //3) Remove the nic _networkMgr.removeNic(vmProfile, nic); - _nicsDao.expunge(nic.getId()); + _nicsDao.remove(nic.getId()); return true; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index df4fb06325f9..316c2dd53d72 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -50,6 +50,8 @@ public interface NicDao extends GenericDao { NicVO findDefaultNicForVM(long instanceId); + NicVO findFirstNicForVM(long instanceId); + /** * @param networkId * @param instanceId diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index c125d80c5342..6314ca0391ba 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -69,6 +69,7 @@ protected void init() { AllFieldsSearch.and("strategy", AllFieldsSearch.entity().getReservationStrategy(), Op.EQ); AllFieldsSearch.and("reserverName",AllFieldsSearch.entity().getReserver(),Op.EQ); AllFieldsSearch.and("macAddress", AllFieldsSearch.entity().getMacAddress(), Op.EQ); + AllFieldsSearch.and("deviceid", AllFieldsSearch.entity().getDeviceId(), Op.EQ); AllFieldsSearch.done(); IpSearch = createSearchBuilder(String.class); @@ -222,6 +223,14 @@ public NicVO findDefaultNicForVM(long instanceId) { return findOneBy(sc); } + @Override + public NicVO findFirstNicForVM(long instanceId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("instance", instanceId); + sc.setParameters("deviceid", 0); + return findOneBy(sc); + } + @Override public NicVO getControlNicForVM(long vmId){ SearchCriteria sc = AllFieldsSearch.create(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 34439fc6e702..e2c284cd0bc0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -106,6 +106,7 @@ import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer; import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource; +import com.cloud.agent.api.SecurityGroupRulesCmd; import com.cloud.dc.Vlan; import com.cloud.exception.InternalErrorException; import com.cloud.host.Host.Type; @@ -146,6 +147,7 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.hypervisor.kvm.storage.KVMStorageProcessor; import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.IsolationType; import com.cloud.network.Networks.RouterPrivateIpStrategy; import com.cloud.network.Networks.TrafficType; import com.cloud.resource.RequestWrapper; @@ -3548,7 +3550,117 @@ public boolean destroyNetworkRulesForVM(final Connect conn, final String vmName) return true; } - public boolean defaultNetworkRules(final Connect conn, final String vmName, final NicTO nic, final Long vmId, final String secIpStr) { + /** + * Function to destroy the security group rules applied to the nic's + * @param conn + * @param vmName + * @param nic + * @return + * true : If success + * false : If failure + */ + public boolean destroyNetworkRulesForNic(final Connect conn, final String vmName, final NicTO nic) { + if (!_canBridgeFirewall) { + return false; + } + final List nicSecIps = nic.getNicSecIps(); + String secIpsStr; + final StringBuilder sb = new StringBuilder(); + if (nicSecIps != null) { + for (final String ip : nicSecIps) { + sb.append(ip).append(SecurityGroupRulesCmd.RULE_COMMAND_SEPARATOR); + } + secIpsStr = sb.toString(); + } else { + secIpsStr = "0" + SecurityGroupRulesCmd.RULE_COMMAND_SEPARATOR; + } + final List intfs = getInterfaces(conn, vmName); + if (intfs.size() == 0 || intfs.size() < nic.getDeviceId()) { + return false; + } + + final InterfaceDef intf = intfs.get(nic.getDeviceId()); + final String brname = intf.getBrName(); + final String vif = intf.getDevName(); + + final Script cmd = new Script(_securityGroupPath, _timeout, s_logger); + cmd.add("destroy_network_rules_for_vm"); + cmd.add("--vmname", vmName); + if (nic.getIp() != null) { + cmd.add("--vmip", nic.getIp()); + } + cmd.add("--vmmac", nic.getMac()); + cmd.add("--vif", vif); + cmd.add("--nicsecips", secIpsStr); + + final String result = cmd.execute(); + if (result != null) { + return false; + } + return true; + } + + /** + * Function to apply default network rules for a VM + * @param conn + * @param vm + * @param checkBeforeApply + * @return + */ + public boolean applyDefaultNetworkRules(final Connect conn, final VirtualMachineTO vm, final boolean checkBeforeApply) { + NicTO[] nicTOs = new NicTO[] {}; + if (vm != null && vm.getNics() != null) { + s_logger.debug("Checking default network rules for vm " + vm.getName()); + nicTOs = vm.getNics(); + } + for (NicTO nic : nicTOs) { + if (vm.getType() != VirtualMachine.Type.User) { + nic.setPxeDisable(true); + } + } + boolean isFirstNic = true; + for (final NicTO nic : nicTOs) { + if (nic.isSecurityGroupEnabled() || nic.getIsolationUri() != null && nic.getIsolationUri().getScheme().equalsIgnoreCase(IsolationType.Ec2.toString())) { + if (vm.getType() != VirtualMachine.Type.User) { + configureDefaultNetworkRulesForSystemVm(conn, vm.getName()); + break; + } + if (!applyDefaultNetworkRulesOnNic(conn, vm.getName(), vm.getId(), nic, isFirstNic, checkBeforeApply)) { + s_logger.error("Unable to apply default network rule for nic " + nic.getName() + " for VM " + vm.getName()); + return false; + } + isFirstNic = false; + } + } + return true; + } + + /** + * Function to apply default network rules for a NIC + * @param conn + * @param vmName + * @param vmId + * @param nic + * @param isFirstNic + * @param checkBeforeApply + * @return + */ + public boolean applyDefaultNetworkRulesOnNic(final Connect conn, final String vmName, final Long vmId, final NicTO nic, boolean isFirstNic, boolean checkBeforeApply) { + final List nicSecIps = nic.getNicSecIps(); + String secIpsStr; + final StringBuilder sb = new StringBuilder(); + if (nicSecIps != null) { + for (final String ip : nicSecIps) { + sb.append(ip).append(SecurityGroupRulesCmd.RULE_COMMAND_SEPARATOR); + } + secIpsStr = sb.toString(); + } else { + secIpsStr = "0" + SecurityGroupRulesCmd.RULE_COMMAND_SEPARATOR; + } + return defaultNetworkRules(conn, vmName, nic, vmId, secIpsStr, isFirstNic, checkBeforeApply); + } + + public boolean defaultNetworkRules(final Connect conn, final String vmName, final NicTO nic, final Long vmId, final String secIpStr, final boolean isFirstNic, final boolean checkBeforeApply) { if (!_canBridgeFirewall) { return false; } @@ -3576,6 +3688,12 @@ public boolean defaultNetworkRules(final Connect conn, final String vmName, fina cmd.add("--vif", vif); cmd.add("--brname", brname); cmd.add("--nicsecips", secIpStr); + if (isFirstNic) { + cmd.add("--isFirstNic"); + } + if (checkBeforeApply) { + cmd.add("--check"); + } final String result = cmd.execute(); if (result != null) { return false; @@ -3665,7 +3783,7 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str return true; } - public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String secIp, final String action) { + public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action) { if (!_canBridgeFirewall) { return false; @@ -3674,6 +3792,7 @@ public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final Stri final Script cmd = new Script(_securityGroupPath, _timeout, s_logger); cmd.add("network_rules_vmSecondaryIp"); cmd.add("--vmname", vmName); + cmd.add("--vmmac", vmMac); cmd.add("--nicsecips", secIp); cmd.add("--action=" + action); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java index 1ad5cb459cfb..07c091ee0eee 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java @@ -41,11 +41,11 @@ public Answer execute(final NetworkRulesVmSecondaryIpCommand command, final Libv final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(command.getVmName()); - result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmSecIp(), command.getAction()); + result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction()); } catch (final LibvirtException e) { s_logger.debug("Could not configure VM secondary IP! => " + e.getLocalizedMessage()); } return new Answer(command, result, ""); } -} \ No newline at end of file +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java index 1ef32afccbfc..553a71a30dfe 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java @@ -29,6 +29,7 @@ import com.cloud.hypervisor.kvm.resource.VifDriver; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.vm.VirtualMachine; import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.Domain; @@ -45,6 +46,7 @@ public final class LibvirtPlugNicCommandWrapper extends CommandWrapper nicSecIps = nic.getNicSecIps(); - String secIpsStr; - final StringBuilder sb = new StringBuilder(); - if (nicSecIps != null) { - for (final String ip : nicSecIps) { - sb.append(ip).append(";"); - } - secIpsStr = sb.toString(); - } else { - secIpsStr = "0;"; - } - libvirtComputingResource.defaultNetworkRules(conn, vmName, nic, vmSpec.getId(), secIpsStr); - } - } - } + libvirtComputingResource.applyDefaultNetworkRules(conn, vmSpec, false); // pass cmdline info to system vms if (vmSpec.getType() != VirtualMachine.Type.User) { String controlIp = null; - for (final NicTO nic : nics) { + for (final NicTO nic : vmSpec.getNics()) { if (nic.getType() == TrafficType.Control) { controlIp = nic.getIp(); break; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnPlugNicCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnPlugNicCommandWrapper.java index 57f4083c96ff..071352c5c9a0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnPlugNicCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnPlugNicCommandWrapper.java @@ -55,6 +55,9 @@ public Answer execute(final UnPlugNicCommand command, final LibvirtComputingReso for (final InterfaceDef pluggedNic : pluggedNics) { if (pluggedNic.getMacAddress().equalsIgnoreCase(nic.getMac())) { + if (nic.isSecurityGroupEnabled()) { + libvirtComputingResource.destroyNetworkRulesForNic(conn, vmName, nic); + } vm.detachDevice(pluggedNic.toString()); // We don't know which "traffic type" is associated with // each interface at this point, so inform all vif drivers @@ -79,4 +82,4 @@ public Answer execute(final UnPlugNicCommand command, final LibvirtComputingReso } } } -} \ No newline at end of file +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 13f8df9f2dd2..1bf27d06ef68 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -2413,7 +2413,7 @@ public void testNetworkRulesVmSecondaryIpCommand() { } catch (final LibvirtException e) { fail(e.getMessage()); } - when(libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmSecIp(), command.getAction())).thenReturn(true); + when(libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction())).thenReturn(true); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -2427,7 +2427,7 @@ public void testNetworkRulesVmSecondaryIpCommand() { fail(e.getMessage()); } verify(libvirtComputingResource, times(1)).getLibvirtUtilitiesHelper(); - verify(libvirtComputingResource, times(1)).configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmSecIp(), command.getAction()); + verify(libvirtComputingResource, times(1)).configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction()); } @SuppressWarnings("unchecked") @@ -3021,6 +3021,8 @@ public void testSecurityGroupRulesCmdTrue() { cidrs.add("0.0.0.0/0"); final SecurityGroupRulesCmd command = new SecurityGroupRulesCmd(guestIp, guestIp6, guestMac, vmName, vmId, signature, seqNum, ingressRuleSet, egressRuleSet, secIps); + final VirtualMachineTO vm = Mockito.mock(VirtualMachineTO.class); + command.setVmTO(vm); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); final Connect conn = Mockito.mock(Connect.class); @@ -3053,6 +3055,7 @@ public void testSecurityGroupRulesCmdTrue() { when(egressRuleSet[0].getEndPort()).thenReturn(22); when(egressRuleSet[0].getAllowedCidrs()).thenReturn(cidrs); + when(libvirtComputingResource.applyDefaultNetworkRules(conn, vm, true)).thenReturn(true); when(libvirtComputingResource.addNetworkRules(command.getVmName(), Long.toString(command.getVmId()), command.getGuestIp(), command.getGuestIp6(), command.getSignature(), Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString())).thenReturn(true); diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index 2ca7ba82a4e7..680177e69549 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -145,6 +145,44 @@ def split_ips_by_family(ips): ip6s.append(ip) return ip4s, ip6s +def destroy_network_rules_for_nic(vm_name, vm_ip, vm_mac, vif, sec_ips): + try: + rules = execute("""iptables-save -t filter | awk '/ %s / { sub(/-A/, "-D", $1) ; print }'""" % vif ).split("\n") + for rule in filter(None, rules): + try: + execute("iptables " + rule) + except: + logging.debug("Ignoring failure to delete rule: " + rule) + except: + pass + + try: + dnats = execute("""iptables-save -t nat | awk '/ %s / { sub(/-A/, "-D", $1) ; print }'""" % vif ).split("\n") + for dnat in filter(None, dnats): + try: + execute("iptables -t nat " + dnat) + except: + logging.debug("Ignoring failure to delete dnat: " + dnat) + except: + pass + + ips = sec_ips.split(';') + ips.pop() + ips.append(vm_ip) + add_to_ipset(vm_name, ips, "-D") + ebtables_rules_vmip(vm_name, vm_mac, ips, "-D") + + vmchain_in = vm_name + "-in" + vmchain_out = vm_name + "-out" + vmchain_in_src = vm_name + "-in-src" + vmchain_out_dst = vm_name + "-out-dst" + try: + execute("ebtables -t nat -D " + vmchain_in_src + " -s " + vm_mac + " -j RETURN") + execute("ebtables -t nat -D " + vmchain_out_dst + " -p ARP --arp-op Reply --arp-mac-dst " + vm_mac + " -j RETURN") + execute("ebtables -t nat -D PREROUTING -i " + vif + " -j " + vmchain_in) + execute("ebtables -t nat -D POSTROUTING -o " + vif + " -j " + vmchain_out) + except: + logging.debug("Ignoring failure to delete ebtable rules for vm: " + vm_name) def get_bridge_physdev(brname): physdev = execute("bridge -o link show | awk '/master %s / && !/^[0-9]+: vnet/ {print $2}' | head -1" % brname) @@ -158,6 +196,9 @@ def destroy_network_rules_for_vm(vm_name, vif=None): vm_ipsetname=ipset_chain_name(vm_name) delete_rules_for_vm_in_bridge_firewall_chain(vm_name) + if 1 in [vm_name.startswith(c) for c in ['r-', 's-', 'v-']]: + return True + if vm_name.startswith('i-'): vmchain_default = '-'.join(vm_name.split('-')[:-1]) + "-def" @@ -198,9 +239,6 @@ def destroy_network_rules_for_vm(vm_name, vif=None): remove_rule_log_for_vm(vm_name) remove_secip_log_for_vm(vm_name) - if 1 in [vm_name.startswith(c) for c in ['r-', 's-', 'v-']]: - return True - return True @@ -228,7 +266,7 @@ def destroy_ebtables_rules(vm_name, vif): execute("ebtables -t nat " + cmd) except: logging.debug("Ignoring failure to delete ebtables rules for vm " + vm_name) - chains = [eb_vm_chain+"-in", eb_vm_chain+"-out", eb_vm_chain+"-in-ips", eb_vm_chain+"-out-ips"] + chains = [eb_vm_chain+"-in", eb_vm_chain+"-out", eb_vm_chain+"-in-ips", eb_vm_chain+"-out-ips", eb_vm_chain+"-in-src", eb_vm_chain+"-out-dst"] for chain in chains: try: execute("ebtables -t nat -F " + chain) @@ -237,14 +275,33 @@ def destroy_ebtables_rules(vm_name, vif): logging.debug("Ignoring failure to delete ebtables chain for vm " + vm_name) -def default_ebtables_rules(vm_name, vm_ip, vm_mac, vif): +def default_ebtables_rules(vm_name, vm_ip, vm_mac, vif, is_first_nic=False): eb_vm_chain=ebtables_chain_name(vm_name) vmchain_in = eb_vm_chain + "-in" vmchain_out = eb_vm_chain + "-out" vmchain_in_ips = eb_vm_chain + "-in-ips" vmchain_out_ips = eb_vm_chain + "-out-ips" + vmchain_in_src = eb_vm_chain + "-in-src" + vmchain_out_dst = eb_vm_chain + "-out-dst" - for chain in [vmchain_in, vmchain_out, vmchain_in_ips, vmchain_out_ips]: + if not is_first_nic: + try: + execute("ebtables -t nat -A PREROUTING -i " + vif + " -j " + vmchain_in) + execute("ebtables -t nat -A POSTROUTING -o " + vif + " -j " + vmchain_out) + execute("ebtables -t nat -I " + vmchain_in_src + " -s " + vm_mac + " -j RETURN") + if vm_ip: + execute("ebtables -t nat -I " + vmchain_in_ips + " -p ARP -s " + vm_mac + " --arp-mac-src " + vm_mac + " --arp-ip-src " + vm_ip + " -j RETURN") + execute("ebtables -t nat -I " + vmchain_out_dst + " -p ARP --arp-op Reply --arp-mac-dst " + vm_mac + " -j RETURN") + if vm_ip: + execute("ebtables -t nat -I " + vmchain_out_ips + " -p ARP --arp-ip-dst " + vm_ip + " -j RETURN") + except: + logging.debug("Failed to program rules for additional nic " + vif) + return False + return + + destroy_ebtables_rules(vm_name, vif) + + for chain in [vmchain_in, vmchain_out, vmchain_in_ips, vmchain_out_ips, vmchain_in_src, vmchain_out_dst]: try: execute("ebtables -t nat -N " + chain) except: @@ -256,17 +313,19 @@ def default_ebtables_rules(vm_name, vm_ip, vm_mac, vif): execute("ebtables -t nat -A POSTROUTING -o " + vif + " -j " + vmchain_out) execute("ebtables -t nat -A " + vmchain_in_ips + " -j DROP") execute("ebtables -t nat -A " + vmchain_out_ips + " -j DROP") + execute("ebtables -t nat -A " + vmchain_in_src + " -j DROP") + execute("ebtables -t nat -A " + vmchain_out_dst + " -p ARP --arp-op Reply -j DROP") + except: logging.debug("Failed to program default rules") return False try: - execute("ebtables -t nat -A " + vmchain_in + " -s ! " + vm_mac + " -j DROP") - execute("ebtables -t nat -A " + vmchain_in + " -p ARP -s ! " + vm_mac + " -j DROP") - execute("ebtables -t nat -A " + vmchain_in + " -p ARP --arp-mac-src ! " + vm_mac + " -j DROP") + execute("ebtables -t nat -A " + vmchain_in + " -j " + vmchain_in_src) + execute("ebtables -t nat -I " + vmchain_in_src + " -s " + vm_mac + " -j RETURN") + execute("ebtables -t nat -A " + vmchain_in + " -p ARP -j " + vmchain_in_ips) if vm_ip: - execute("ebtables -t nat -A " + vmchain_in + " -p ARP -j " + vmchain_in_ips) - execute("ebtables -t nat -I " + vmchain_in_ips + " -p ARP --arp-ip-src " + vm_ip + " -j RETURN") + execute("ebtables -t nat -I " + vmchain_in_ips + " -p ARP -s " + vm_mac + " --arp-mac-src " + vm_mac + " --arp-ip-src " + vm_ip + " -j RETURN") execute("ebtables -t nat -A " + vmchain_in + " -p ARP --arp-op Request -j ACCEPT") execute("ebtables -t nat -A " + vmchain_in + " -p ARP --arp-op Reply -j ACCEPT") execute("ebtables -t nat -A " + vmchain_in + " -p ARP -j DROP") @@ -275,9 +334,10 @@ def default_ebtables_rules(vm_name, vm_ip, vm_mac, vif): return False try: - execute("ebtables -t nat -A " + vmchain_out + " -p ARP --arp-op Reply --arp-mac-dst ! " + vm_mac + " -j DROP") + execute("ebtables -t nat -A " + vmchain_out + " -p ARP --arp-op Reply -j " + vmchain_out_dst) + execute("ebtables -t nat -I " + vmchain_out_dst + " -p ARP --arp-op Reply --arp-mac-dst " + vm_mac + " -j RETURN") + execute("ebtables -t nat -A " + vmchain_out + " -p ARP -j " + vmchain_out_ips ) if vm_ip: - execute("ebtables -t nat -A " + vmchain_out + " -p ARP -j " + vmchain_out_ips ) execute("ebtables -t nat -I " + vmchain_out_ips + " -p ARP --arp-ip-dst " + vm_ip + " -j RETURN") execute("ebtables -t nat -A " + vmchain_out + " -p ARP --arp-op Request -j ACCEPT") execute("ebtables -t nat -A " + vmchain_out + " -p ARP --arp-op Reply -j ACCEPT") @@ -286,6 +346,28 @@ def default_ebtables_rules(vm_name, vm_ip, vm_mac, vif): logging.debug("Failed to program default ebtables OUT rules") return False +def refactor_ebtable_rules(vm_name): + vmchain_in = vm_name + "-in" + vmchain_in_ips = vm_name + "-in-ips" + vmchain_in_src = vm_name + "-in-src" + + try: + execute("ebtables -t nat -L " + vmchain_in_src) + logging.debug("Chain '" + vmchain_in_src + "' exists, ebtables rules have newer version, skip refactoring") + return True + except: + logging.debug("Chain '" + vmchain_in_src + "' does not exist, ebtables rules have old version, start refactoring") + + vif = execute("ebtables -t nat -L PREROUTING | grep " + vmchain_in + " | awk '{print $2}'").strip() + vm_mac = execute("ebtables -t nat -L " + vmchain_in + " | grep arp-mac-src | awk '{print $5}'").strip() + vm_ips = execute("ebtables -t nat -L " + vmchain_in_ips + " | grep arp-ip-src | awk '{print $4}'").split('\n') + + destroy_ebtables_rules(vm_name, vif) + default_ebtables_rules(vm_name, None, vm_mac, vif, True) + ebtables_rules_vmip(vm_name, vm_mac, vm_ips, "-A") + + logging.debug("Refactoring ebtables rules for vm " + vm_name + " is done") + return True def default_network_rules_systemvm(vm_name, localbrname): bridges = get_bridges(vm_name) @@ -382,8 +464,9 @@ def add_to_ipset(ipsetname, ips, action): return result -def network_rules_vmSecondaryIp(vm_name, ip_secondary, action): +def network_rules_vmSecondaryIp(vm_name, vm_mac, ip_secondary, action): logging.debug("vmName = "+ vm_name) + logging.debug("vmMac = " + vm_mac) logging.debug("action = "+ action) vmchain = vm_name @@ -393,16 +476,16 @@ def network_rules_vmSecondaryIp(vm_name, ip_secondary, action): add_to_ipset(vmchain, ip4s, action) - #add ebtables rules for the secondary ips - ebtables_rules_vmip(vm_name, ip4s, action) - #add ipv6 addresses to ipv6 ipset add_to_ipset(vmchain6, ip6s, action) - return True + #add ebtables rules for the secondary ip + refactor_ebtable_rules(vm_name) + ebtables_rules_vmip(vm_name, vm_mac, [ip_secondary], action) + return True -def ebtables_rules_vmip (vmname, ips, action): +def ebtables_rules_vmip (vmname, vmmac, ips, action): eb_vm_chain=ebtables_chain_name(vmname) vmchain_inips = eb_vm_chain + "-in-ips" vmchain_outips = eb_vm_chain + "-out-ips" @@ -415,46 +498,75 @@ def ebtables_rules_vmip (vmname, ips, action): if ip == 0 or ip == "0": continue try: - execute("ebtables -t nat " + action + " " + vmchain_inips + " -p ARP --arp-ip-src " + ip + " -j RETURN") + execute("ebtables -t nat " + action + " " + vmchain_inips + " -p ARP -s " + vmmac + " --arp-mac-src " + vmmac + " --arp-ip-src " + ip + " -j RETURN") execute("ebtables -t nat " + action + " " + vmchain_outips + " -p ARP --arp-ip-dst " + ip + " -j RETURN") except: logging.debug("Failed to program ebtables rules for secondary ip %s for vm %s with action %s" % (ip, vmname, action)) +def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False): + brfw = get_br_fw(brname) + vmchain_default = '-'.join(vm_name.split('-')[:-1]) + "-def" + try: + rules = execute("iptables-save |grep -w %s |grep -w %s |grep -w %s" % (brfw, vif, vmchain_default)) + except: + rules = None + if rules is None or rules is "": + logging.debug("iptables rules do not exist, programming default rules for %s %s" % (vm_name,vif)) + default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic) + else: + vmchain_in = vm_name + "-in" + try: + rules = execute("ebtables -t nat -L PREROUTING | grep %s |grep -w %s" % (vmchain_in, vif)) + except: + rules = None + if rules is None or rules is "": + logging.debug("ebtables rules do not exist, programming default ebtables rules for %s %s" % (vm_name,vif)) + default_ebtables_rules(vm_name, vm_ip, vm_mac, vif, is_first_nic) + ips = sec_ips.split(';') + ips.pop() + ebtables_rules_vmip(vm_name, vm_mac, ips, "-I") + return True -def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips): +def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False): if not add_fw_framework(brname): return False vmName = vm_name brfw = get_br_fw(brname) domID = get_vm_id(vm_name) - delete_rules_for_vm_in_bridge_firewall_chain(vmName) + vmchain = iptables_chain_name(vm_name) vmchain_egress = egress_chain_name(vm_name) vmchain_default = '-'.join(vmchain.split('-')[:-1]) + "-def" ipv6_link_local = ipv6_link_local_addr(vm_mac) - destroy_ebtables_rules(vm_name, vif) - - for chain in [vmchain, vmchain_egress, vmchain_default]: - try: - execute('iptables -N ' + chain) - except: - execute('iptables -F ' + chain) - - try: - execute('ip6tables -N ' + chain) - except: - execute('ip6tables -F ' + chain) - action = "-A" vmipsetName = ipset_chain_name(vm_name) vmipsetName6 = vmipsetName + '-6' - #create ipset and add vm ips to that ip set - if not create_ipset_forvm(vmipsetName): - logging.debug("failed to create ipset for rule %s", vmipsetName) - return False + if is_first_nic: + delete_rules_for_vm_in_bridge_firewall_chain(vmName) + destroy_ebtables_rules(vmName, vif) + + for chain in [vmchain, vmchain_egress, vmchain_default]: + try: + execute('iptables -N ' + chain) + except: + execute('iptables -F ' + chain) + + try: + execute('ip6tables -N ' + chain) + except: + execute('ip6tables -F ' + chain) + + #create ipset and add vm ips to that ip set + if not create_ipset_forvm(vmipsetName): + logging.debug("failed to create ipset for rule %s", vmipsetName) + return False + + if not create_ipset_forvm(vmipsetName6, family='inet6', type='hash:net'): + logging.debug("failed to create ivp6 ipset for rule %s", vmipsetName6) + return False #add primary nic ip to ipset if not add_to_ipset(vmipsetName, [vm_ip], action ): @@ -487,31 +599,30 @@ def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, se #allow dhcp execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -p udp --dport 67 --sport 68 -j ACCEPT") execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -p udp --dport 68 --sport 67 -j ACCEPT") + execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -p udp --sport 67 -j DROP") #don't let vm spoof its ip address if vm_ip: execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set ! --match-set " + vmipsetName + " src -j DROP") + execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -m set ! --match-set " + vmipsetName + " dst -j DROP") execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -p udp --dport 53 -j RETURN ") execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -p tcp --dport 53 -j RETURN ") execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -j " + vmchain_egress) + execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -j " + vmchain) execute("iptables -A " + vmchain + " -j DROP") except: logging.debug("Failed to program default rules for vm " + vm_name) return False - default_ebtables_rules(vm_name, vm_ip, vm_mac, vif) + default_ebtables_rules(vmchain, vm_ip, vm_mac, vif, is_first_nic) #default ebtables rules for vm secondary ips - ebtables_rules_vmip(vm_name, ip4s, "-I") + ebtables_rules_vmip(vm_name, vm_mac, ip4s, "-I") - if vm_ip: + if vm_ip and is_first_nic: if not write_rule_log_for_vm(vmName, vm_id, vm_ip, domID, '_initial_', '-1'): logging.debug("Failed to log default network rules, ignoring") - if not create_ipset_forvm(vmipsetName6, family='inet6', type='hash:net'): - logging.debug(" failed to create ivp6 ipset for rule " + str(tokens)) - return False - vm_ip6_addr = [ipv6_link_local] try: ip6 = ipaddress.ip_address(vm_ip6) @@ -763,7 +874,8 @@ def get_rule_logs_for_vms(): name = name.rstrip() if 1 not in [name.startswith(c) for c in ['r-', 's-', 'v-', 'i-'] ]: continue - network_rules_for_rebooted_vm(name) + # Move actions on rebooted vm to java code + # network_rules_for_rebooted_vm(name) if name.startswith('i-'): log = get_rule_log_for_vm(name) result.append(log) @@ -992,8 +1104,10 @@ def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, ru logging.debug("Rules already programmed for vm " + vm_name) return True - if changes[0] or changes[1] or changes[2] or changes[3]: - default_network_rules(vmName, vm_id, vm_ip, vm_ip6, vmMac, vif, brname, sec_ips) + if rules == "" or rules == None: + lines = [] + else: + lines = rules.split(';')[:-1] logging.debug("programming network rules for IP: " + vm_ip + " vmname=%s", vm_name) @@ -1007,7 +1121,7 @@ def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, ru execute('ip6tables -F ' + chain) except: logging.debug("Error flushing iptables rules for " + vm_name + ". Presuming firewall rules deleted, re-initializing." ) - default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vmMac, vif, brname, sec_ips) + default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vmMac, vif, brname, sec_ips, True) egressrule_v4 = 0 egressrule_v6 = 0 @@ -1162,6 +1276,11 @@ def get_br_fw(brname): def add_fw_framework(brname): + try: + execute("modprobe br_netfilter") + except: + logging.debug("failed to load kernel module br_netfilter") + try: execute("sysctl -w net.bridge.bridge-nf-call-arptables=1") execute("sysctl -w net.bridge.bridge-nf-call-iptables=1") @@ -1239,6 +1358,236 @@ def add_fw_framework(brname): return False return False +def verify_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips): + if vm_name is None or vm_ip is None or vm_mac is None: + print("vm_name, vm_ip and vm_mac must be specifed") + sys.exit(1) + + if vm_id is None: + vm_id = vm_name.split("-")[-2] + + if brname is None: + brname = execute("virsh domiflist %s |grep -w '%s' |tr -s ' '|cut -d ' ' -f3" % (vm_name, vm_mac)).strip() + if brname is None or brname == "": + print("Cannot find bridge") + sys.exit(1) + + if vif is None: + vif = execute("virsh domiflist %s |grep -w '%s' |tr -s ' '|cut -d ' ' -f1" % (vm_name, vm_mac)).strip() + if vif is None or vif == "": + print("Cannot find vif") + sys.exit(1) + + #vm_name = "i-2-55-VM" + #vm_id = 55 + #vm_ip = "10.11.118.128" + #vm_ip6 = "fe80::1c00:b4ff:fe00:5" + #vm_mac = "1e:00:b4:00:00:05" + #vif = "vnet11" + #brname = "cloudbr0" + #sec_ips = "10.11.118.133;10.11.118.135;10.11.118.138;" # end with ";" and seperated by ";" + + vm_ips = [] + if sec_ips is not None: + vm_ips = sec_ips.split(';') + vm_ips.pop() + vm_ips.reverse() + vm_ips.append(vm_ip) + + if not verify_ipset_for_vm(vm_name, vm_id, vm_ips, vm_ip6): + sys.exit(2) + if not verify_iptables_rules_for_bridge(brname): + sys.exit(3) + if not verify_default_iptables_rules_for_vm(vm_name, vm_id, vm_ips, vm_ip6, vm_mac, vif, brname): + sys.exit(4) + if not verify_ebtables_rules_for_vm(vm_name, vm_id, vm_ips, vm_ip6, vm_mac, vif, brname): + sys.exit(5) + + sys.exit(0) + +def verify_ipset_for_vm(vm_name, vm_id, vm_ips, vm_ip6): + vmipsetName = ipset_chain_name(vm_name) + vmipsetName6 = vmipsetName + '-6' + + rules = [] + for rule in execute("ipset list %s" % vmipsetName).split('\n'): + rules.append(rule) + + # Check if all vm ips and ip6 exist + for vm_ip in vm_ips: + found = False + for rule in rules: + if rule == vm_ip: + found = True + break + if not found: + print("vm ip %s is not found" % vm_ip) + return False + + rules = [] + for rule in execute("ipset list %s" % vmipsetName6).split('\n'): + rules.append(rule) + + if vm_ip6 is not None: + found = False + for rule in rules: + if rule == vm_ip6: + found = True + break + if not found: + print("vm ipv6 %s is not found" % vm_ip6) + return False + + return True + +def verify_iptables_rules_for_bridge(brname): + brfw = get_br_fw(brname) + brfwin = brfw + "-IN" + brfwout = brfw + "-OUT" + + expected_rules = [] + expected_rules.append("-A FORWARD -o %s -m physdev --physdev-is-bridged -j %s" % (brname, brfw)) + expected_rules.append("-A FORWARD -i %s -m physdev --physdev-is-bridged -j %s" % (brname, brfw)) + expected_rules.append("-A %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % (brfw)) + expected_rules.append("-A %s -m physdev --physdev-is-in --physdev-is-bridged -j %s" % (brfw, brfwin)) + expected_rules.append("-A %s -m physdev --physdev-is-out --physdev-is-bridged -j %s" % (brfw, brfwout)) + phydev = execute("brctl show | awk '/^%s[ \t]/ {print $4}'" % brname ).strip() + expected_rules.append("-A %s -m physdev --physdev-out %s --physdev-is-bridged -j ACCEPT" % (brfw, phydev)) + + rules = execute("iptables-save |grep -w %s |grep -v \"^:\"" % brfw).split('\n') + + return verify_expected_rules_exist(expected_rules, rules) + +def verify_default_iptables_rules_for_vm(vm_name, vm_id, vm_ips, vm_ip6, vm_mac, vif, brname): + brfw = get_br_fw(brname) + brfwin = brfw + "-IN" + brfwout = brfw + "-OUT" + vmchain = iptables_chain_name(vm_name) + vmchain_egress = egress_chain_name(vm_name) + vm_def = '-'.join(vm_name.split('-')[:-1]) + "-def" + + expected_rules = [] + expected_rules.append("-A %s -m physdev --physdev-in %s --physdev-is-bridged -j %s" % (brfwin, vif, vm_def)) + expected_rules.append("-A %s -m physdev --physdev-out %s --physdev-is-bridged -j %s" % (brfwout, vif, vm_def)) + expected_rules.append("-A %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % (vm_def)) + expected_rules.append("-A %s -p udp -m physdev --physdev-in %s --physdev-is-bridged -m udp --sport 68 --dport 67 -j ACCEPT" % (vm_def, vif)) + expected_rules.append("-A %s -p udp -m physdev --physdev-out %s --physdev-is-bridged -m udp --sport 67 --dport 68 -j ACCEPT" % (vm_def, vif)) + expected_rules.append("-A %s -p udp -m physdev --physdev-in %s --physdev-is-bridged -m udp --sport 67 -j DROP" % (vm_def, vif)) + expected_rules.append("-A %s -m physdev --physdev-in %s --physdev-is-bridged -m set ! --match-set %s src -j DROP" % (vm_def, vif, vm_name)) + expected_rules.append("-A %s -m physdev --physdev-out %s --physdev-is-bridged -m set ! --match-set %s dst -j DROP" % (vm_def, vif, vm_name)) + expected_rules.append("-A %s -p udp -m physdev --physdev-in %s --physdev-is-bridged -m set --match-set %s src -m udp --dport 53 -j RETURN" % (vm_def, vif, vm_name)) + expected_rules.append("-A %s -p tcp -m physdev --physdev-in %s --physdev-is-bridged -m set --match-set %s src -m tcp --dport 53 -j RETURN" % (vm_def, vif, vm_name)) + expected_rules.append("-A %s -m physdev --physdev-in %s --physdev-is-bridged -m set --match-set %s src -j %s" % (vm_def, vif, vm_name, vmchain_egress)) + expected_rules.append("-A %s -m physdev --physdev-out %s --physdev-is-bridged -j %s" % (vm_def, vif, vmchain)) + + rules = execute("iptables-save |grep -E \"%s|%s\" |grep -v \"^:\"" % (vm_name, vm_def)).split('\n') + + return verify_expected_rules_in_order(expected_rules, rules) + +def verify_ebtables_rules_for_vm(vm_name, vm_id, vm_ips, vm_ip6, vm_mac, vif, brname): + vmchain_in = vm_name + "-in" + vmchain_out = vm_name + "-out" + vmchain_in_ips = vm_name + "-in-ips" + vmchain_out_ips = vm_name + "-out-ips" + vmchain_in_src = vm_name + "-in-src" + vmchain_out_dst = vm_name + "-out-dst" + + new_mac = trim_mac_address(vm_mac) + + # PREROUTING/POSTROUTING + expected_rules = [] + expected_rules.append("-A PREROUTING -i %s -j %s" % (vif, vmchain_in)) + expected_rules.append("-A POSTROUTING -o %s -j %s" % (vif, vmchain_out)) + rules = execute("ebtables-save |grep -E \"PREROUTING|POSTROUTING\" | grep %s" % vm_name).split('\n') + if not verify_expected_rules_exist(expected_rules, rules): + return False + + rules = execute("ebtables-save | grep %s" % vm_name).split('\n') + + # vmchain_in + expected_rules = [] + expected_rules.append("-A %s -j %s" % (vmchain_in, vmchain_in_src)) + expected_rules.append("-A %s -p ARP -j %s" % (vmchain_in, vmchain_in_ips)) + expected_rules.append("-A %s -p ARP --arp-op Request -j ACCEPT" % (vmchain_in)) + expected_rules.append("-A %s -p ARP --arp-op Reply -j ACCEPT" % (vmchain_in)) + expected_rules.append("-A %s -p ARP -j DROP" % (vmchain_in)) + if not verify_expected_rules_in_order(expected_rules, rules): + return False + + # vmchain_in_src + expected_rules = [] + expected_rules.append("-A %s -s %s -j RETURN" % (vmchain_in_src, new_mac)) + expected_rules.append("-A %s -j DROP" % (vmchain_in_src)) + if not verify_expected_rules_in_order(expected_rules, rules): + return False + + # vmchain_in_ips + expected_rules = [] + for vm_ip in vm_ips: + expected_rules.append("-A %s -p ARP -s %s --arp-ip-src %s --arp-mac-src %s -j RETURN" % (vmchain_in_ips, new_mac, vm_ip, new_mac)) + expected_rules.append("-A %s -j DROP" % (vmchain_in_ips)) + if not verify_expected_rules_in_order(expected_rules, rules): + return False + + # vmchain_out + expected_rules = [] + expected_rules.append("-A %s -p ARP --arp-op Reply -j %s" % (vmchain_out, vmchain_out_dst)) + expected_rules.append("-A %s -p ARP -j %s" % (vmchain_out, vmchain_out_ips)) + expected_rules.append("-A %s -p ARP --arp-op Request -j ACCEPT" % (vmchain_out)) + expected_rules.append("-A %s -p ARP --arp-op Reply -j ACCEPT" % (vmchain_out)) + expected_rules.append("-A %s -p ARP -j DROP" % (vmchain_out)) + if not verify_expected_rules_in_order(expected_rules, rules): + return False + + # vmchain_out_dst + expected_rules = [] + expected_rules.append("-A %s -p ARP --arp-op Reply --arp-mac-dst %s -j RETURN" % (vmchain_out_dst, new_mac)) + expected_rules.append("-A %s -p ARP --arp-op Reply -j DROP" % (vmchain_out_dst)) + if not verify_expected_rules_in_order(expected_rules, rules): + return False + + # vmchain_out_ips + expected_rules = [] + for vm_ip in vm_ips: + expected_rules.append("-A %s -p ARP --arp-ip-dst %s -j RETURN" % (vmchain_out_ips, vm_ip)) + expected_rules.append("-A %s -j DROP" % (vmchain_out_ips)) + if not verify_expected_rules_in_order(expected_rules, rules): + return False + + return True + +def trim_mac_address(vm_mac): + new_mac = "" + for mac in vm_mac.split(":"): + if mac.startswith("0"): + new_mac += ":" + mac[1:] + else: + new_mac += ":" + mac + return new_mac[1:] + +def verify_expected_rules_exist(expected_rules, rules): + # Check if expected rules exist + for expected_rule in expected_rules: + found = False + for rule in rules: + if rule == expected_rule: + found = True + break + if not found: + print("Rule '%s' is not found" % expected_rule) + return False + return True + +def verify_expected_rules_in_order(expected_rules, rules): + # Check if expected rules exist in order (!!!) + i = 0 + for rule in rules: + if i < len(expected_rules) and rule == expected_rules[i]: + i += 1 + if i != len(expected_rules): + print("Cannot find rule '%s'" % expected_rules[i]) + return False + return True if __name__ == '__main__': logging.basicConfig(filename="/var/log/cloudstack/agent/security_group.log", format="%(asctime)s - %(message)s", level=logging.DEBUG) @@ -1261,6 +1610,8 @@ def add_fw_framework(brname): parser.add_argument("--nicsecips", dest="nicSecIps") parser.add_argument("--action", dest="action") parser.add_argument("--privnic", dest="privnic") + parser.add_argument("--isFirstNic", action="store_true", dest="isFirstNic") + parser.add_argument("--check", action="store_true", dest="check") args = parser.parse_args() cmd = args.command logging.debug("Executing command: %s", cmd) @@ -1274,10 +1625,15 @@ def add_fw_framework(brname): if cmd == "can_bridge_firewall": can_bridge_firewall(args.privnic) - elif cmd == "default_network_rules": - default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps) + elif cmd == "default_network_rules" and args.check: + check_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic) + elif cmd == "default_network_rules" and not args.check: + default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic) elif cmd == "destroy_network_rules_for_vm": - destroy_network_rules_for_vm(args.vmName, args.vif) + if args.vmIP is None: + destroy_network_rules_for_vm(args.vmName, args.vif) + else: + destroy_network_rules_for_nic(args.vmName, args.vmIP, args.vmMAC, args.vif, args.nicSecIps) elif cmd == "default_network_rules_systemvm": default_network_rules_systemvm(args.vmName, args.localbrname) elif cmd == "get_rule_logs_for_vms": @@ -1285,11 +1641,13 @@ def add_fw_framework(brname): elif cmd == "add_network_rules": add_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.sig, args.seq, args.vmMAC, args.rules, args.vif, args.brname, args.nicSecIps) elif cmd == "network_rules_vmSecondaryIp": - network_rules_vmSecondaryIp(args.vmName, args.nicSecIps, args.action) + network_rules_vmSecondaryIp(args.vmName, args.vmMAC, args.nicSecIps, args.action) elif cmd == "cleanup_rules": cleanup_rules() elif cmd == "post_default_network_rules": post_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmMAC, args.vif, args.brname, args.dhcpSvr, args.hostIp, args.hostMacAddr) + elif cmd == "verify_network_rules": + verify_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps) else: logging.debug("Unknown command: " + cmd) sys.exit(1) diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java index a0c828588c00..4ce7df853f59 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -58,6 +58,7 @@ import com.cloud.agent.api.NetworkRulesVmSecondaryIpCommand; import com.cloud.agent.api.SecurityGroupRulesCmd; import com.cloud.agent.api.SecurityGroupRulesCmd.IpPortAndProto; +import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.manager.Commands; import com.cloud.api.query.dao.SecurityGroupJoinDao; import com.cloud.configuration.Config; @@ -113,6 +114,8 @@ import com.cloud.vm.VirtualMachine.Event; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.UserVmDao; @@ -377,6 +380,7 @@ public void handleVmStarted(VMInstanceVO vm) { } @DB + @Override public void scheduleRulesetUpdateToHosts(final List affectedVms, final boolean updateSeqno, Long delayMs) { if (affectedVms.size() == 0) { return; @@ -513,8 +517,35 @@ protected SecurityGroupRulesCmd generateRulesetCmd(String vmName, String guestIp egressResult.add(ipPortAndProto); } } - return new SecurityGroupRulesCmd(guestIp, guestIp6, guestMac, vmName, vmId, signature, seqnum, ingressResult.toArray(new IpPortAndProto[ingressResult.size()]), + SecurityGroupRulesCmd cmd = new SecurityGroupRulesCmd(guestIp, guestIp6, guestMac, vmName, vmId, signature, seqnum, ingressResult.toArray(new IpPortAndProto[ingressResult.size()]), egressResult.toArray(new IpPortAndProto[egressResult.size()]), secIps); + + final VirtualMachineTO to = getVmTO(vmId); + cmd.setVmTO(to); + return cmd; + } + + protected VirtualMachineTO getVmTO(Long vmId) { + final VMInstanceVO vm = _vmDao.findById(vmId); + final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); + final List nics = _nicDao.listByVmId(profile.getId()); + Collections.sort(nics, new Comparator() { + @Override + public int compare(NicVO nic1, NicVO nic2) { + Long nicId1 = Long.valueOf(nic1.getDeviceId()); + Long nicId2 = Long.valueOf(nic2.getDeviceId()); + return nicId1.compareTo(nicId2); + } + }); + for (final NicVO nic : nics) { + final Network network = _networkModel.getNetwork(nic.getNetworkId()); + final NicProfile nicProfile = + new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), null, _networkModel.isSecurityGroupSupportedInNetwork(network), + _networkModel.getNetworkTag(profile.getHypervisorType(), network)); + profile.addNic(nicProfile); + } + final VirtualMachineTO to = _itMgr.toVmTO(profile); + return to; } protected void handleVmStopped(VMInstanceVO vm) { @@ -1014,16 +1045,17 @@ public void doInTransactionWithoutResult(TransactionStatus status) { agentId = vm.getHostId(); if (agentId != null) { // get nic secondary ip address - String privateIp = vm.getPrivateIpAddress(); - NicVO nic = _nicDao.findByIp4AddressAndVmId(privateIp, vm.getId()); + NicVO nic = _nicDao.findFirstNicForVM(vm.getId()); List nicSecIps = null; if (nic != null) { if (nic.getSecondaryIp()) { //get secondary ips of the vm nicSecIps = _nicSecIpDao.getSecondaryIpAddressesForNic(nic.getId()); } + } else { + return; } - SecurityGroupRulesCmd cmd = generateRulesetCmd(vm.getInstanceName(), nic.getIPv6Address(), vm.getPrivateIpAddress(), vm.getPrivateMacAddress(), vm.getId(), + SecurityGroupRulesCmd cmd = generateRulesetCmd(vm.getInstanceName(), nic.getIPv4Address(), nic.getIPv6Address(), vm.getPrivateMacAddress(), vm.getId(), generateRulesetSignature(ingressRules, egressRules), seqnum, ingressRules, egressRules, nicSecIps); Commands cmds = new Commands(cmd); try { @@ -1357,7 +1389,7 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool return true; } - String vmMac = vm.getPrivateMacAddress(); + String vmMac = nic.getMacAddress(); String vmName = vm.getInstanceName(); if (vmMac == null || vmName == null) { throw new InvalidParameterValueException("vm name or vm mac can't be null"); diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java index 2d0ec61d09c2..5b4b85fc70c2 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java @@ -177,16 +177,17 @@ public void sendRulesetUpdates(SecurityGroupWork work) { Map> egressRules = generateRulesForVM(userVmId, SecurityRuleType.EgressRule); Long agentId = vm.getHostId(); if (agentId != null) { - String privateIp = vm.getPrivateIpAddress(); - NicVO nic = _nicDao.findByIp4AddressAndVmId(privateIp, vm.getId()); + NicVO nic = _nicDao.findFirstNicForVM(vm.getId()); List nicSecIps = null; if (nic != null) { if (nic.getSecondaryIp()) { nicSecIps = _nicSecIpDao.getSecondaryIpAddressesForNic(nic.getId()); } + } else { + return; } SecurityGroupRulesCmd cmd = - generateRulesetCmd(vm.getInstanceName(), vm.getPrivateIpAddress(), nic.getIPv6Address(), vm.getPrivateMacAddress(), vm.getId(), null, work.getLogsequenceNumber(), + generateRulesetCmd(vm.getInstanceName(), nic.getIPv4Address(), nic.getIPv6Address(), vm.getPrivateMacAddress(), vm.getId(), null, work.getLogsequenceNumber(), ingressRules, egressRules, nicSecIps); cmd.setMsId(_serverId); if (s_logger.isDebugEnabled()) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ad2e453080ea..3f28f1283d8a 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -3225,21 +3225,23 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service throw new InvalidParameterValueException("Security group feature is not supported for vmWare hypervisor"); } // Only one network can be specified, and it should be security group enabled - if (networkIdList.size() > 1) { + if (networkIdList.size() > 1 && template.getHypervisorType() != HypervisorType.KVM && hypervisor != HypervisorType.KVM) { throw new InvalidParameterValueException("Only support one network per VM if security group enabled"); } - NetworkVO network = _networkDao.findById(networkIdList.get(0)); + for (Long networkId : networkIdList) { + NetworkVO network = _networkDao.findById(networkId); - if (network == null) { - throw new InvalidParameterValueException("Unable to find network by id " + networkIdList.get(0).longValue()); - } + if (network == null) { + throw new InvalidParameterValueException("Unable to find network by id " + networkId); + } - if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { - throw new InvalidParameterValueException("Network is not security group enabled: " + network.getId()); - } + if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { + throw new InvalidParameterValueException("Network is not security group enabled: " + network.getId()); + } - networkList.add(network); + networkList.add(network); + } isSecurityGroupEnabledNetworkUsed = true; } else { @@ -3253,10 +3255,6 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service boolean isSecurityGroupEnabled = _networkModel.isSecurityGroupSupportedInNetwork(network); if (isSecurityGroupEnabled) { - if (networkIdList.size() > 1) { - throw new InvalidParameterValueException("Can't create a vm with multiple networks one of" + " which is Security Group enabled"); - } - isSecurityGroupEnabledNetworkUsed = true; } diff --git a/test/integration/component/test_multiple_nic_support.py b/test/integration/component/test_multiple_nic_support.py new file mode 100644 index 000000000000..cf3a23364957 --- /dev/null +++ b/test/integration/component/test_multiple_nic_support.py @@ -0,0 +1,629 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" tests for supporting multiple NIC's in advanced zone with security groups in cloudstack 4.14.0.0 + +""" +# Import Local Modules +from nose.plugins.attrib import attr +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.sshClient import SshClient +from marvin.lib.utils import (validateList, + cleanup_resources, + get_host_credentials, + get_process_status, + execute_command_in_host, + random_gen) +from marvin.lib.base import (PhysicalNetwork, + Account, + Host, + TrafficType, + Domain, + Network, + NetworkOffering, + VirtualMachine, + ServiceOffering, + Zone, + NIC, + SecurityGroup) +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_virtual_machines, + list_routers, + list_hosts, + get_free_vlan) +from marvin.codes import (PASS, FAILED) +import logging +import random +import time + +class TestMulipleNicSupport(cloudstackTestCase): + @classmethod + def setUpClass(cls): + cls.testClient = super( + TestMulipleNicSupport, + cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + cls.testdata = cls.testClient.getParsedTestDataConfig() + cls.services = cls.testClient.getParsedTestDataConfig() + zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.zone = Zone(zone.__dict__) + cls._cleanup = [] + + if str(cls.zone.securitygroupsenabled) != "True": + sys.exit(1) + + cls.logger = logging.getLogger("TestMulipleNicSupport") + cls.stream_handler = logging.StreamHandler() + cls.logger.setLevel(logging.DEBUG) + cls.logger.addHandler(cls.stream_handler) + + # Get Domain and templates + cls.domain = get_domain(cls.apiclient) + cls.services['mode'] = cls.zone.networktype + + cls.template = get_template(cls.apiclient, cls.zone.id, hypervisor="KVM") + if cls.template == FAILED: + sys.exit(1) + + # Create new domain, account, network and VM + cls.user_domain = Domain.create( + cls.apiclient, + services=cls.testdata["acl"]["domain2"], + parentdomainid=cls.domain.id) + + # Create account + cls.account1 = Account.create( + cls.apiclient, + cls.testdata["acl"]["accountD2"], + admin=True, + domainid=cls.user_domain.id + ) + + # Create small service offering + cls.service_offering = ServiceOffering.create( + cls.apiclient, + cls.testdata["service_offerings"]["small"] + ) + + cls._cleanup.append(cls.service_offering) + cls.services["network"]["zoneid"] = cls.zone.id + cls.network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["network_offering"], + ) + # Enable Network offering + cls.network_offering.update(cls.apiclient, state='Enabled') + + cls._cleanup.append(cls.network_offering) + cls.testdata["virtual_machine"]["zoneid"] = cls.zone.id + cls.testdata["virtual_machine"]["template"] = cls.template.id + + if cls.zone.securitygroupsenabled: + # Enable networking for reaching to VM thorugh SSH + security_group = SecurityGroup.create( + cls.apiclient, + cls.testdata["security_group"], + account=cls.account1.name, + domainid=cls.account1.domainid + ) + + # Authorize Security group to SSH to VM + ingress_rule = security_group.authorize( + cls.apiclient, + cls.testdata["ingress_rule"], + account=cls.account1.name, + domainid=cls.account1.domainid + ) + + # Authorize Security group to SSH to VM + ingress_rule2 = security_group.authorize( + cls.apiclient, + cls.testdata["ingress_rule_ICMP"], + account=cls.account1.name, + domainid=cls.account1.domainid + ) + + cls.testdata["shared_network_offering_sg"]["specifyVlan"] = 'True' + cls.testdata["shared_network_offering_sg"]["specifyIpRanges"] = 'True' + cls.shared_network_offering = NetworkOffering.create( + cls.apiclient, + cls.testdata["shared_network_offering_sg"], + conservemode=False + ) + + NetworkOffering.update( + cls.shared_network_offering, + cls.apiclient, + id=cls.shared_network_offering.id, + state="enabled" + ) + + physical_network, vlan = get_free_vlan(cls.apiclient, cls.zone.id) + cls.testdata["shared_network_sg"]["physicalnetworkid"] = physical_network.id + + random_subnet_number = random.randrange(90, 99) + cls.testdata["shared_network_sg"]["name"] = "Shared-Network-SG-Test-vlan" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["displaytext"] = "Shared-Network-SG-Test-vlan" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["vlan"] = "vlan://" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["startip"] = "192.168." + str(random_subnet_number) + ".240" + cls.testdata["shared_network_sg"]["endip"] = "192.168." + str(random_subnet_number) + ".250" + cls.testdata["shared_network_sg"]["gateway"] = "192.168." + str(random_subnet_number) + ".254" + cls.network1 = Network.create( + cls.apiclient, + cls.testdata["shared_network_sg"], + networkofferingid=cls.shared_network_offering.id, + zoneid=cls.zone.id, + accountid=cls.account1.name, + domainid=cls.account1.domainid + ) + + random_subnet_number = random.randrange(100, 110) + cls.testdata["shared_network_sg"]["name"] = "Shared-Network-SG-Test-vlan" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["displaytext"] = "Shared-Network-SG-Test-vlan" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["vlan"] = "vlan://" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["startip"] = "192.168." + str(random_subnet_number) + ".240" + cls.testdata["shared_network_sg"]["endip"] = "192.168." + str(random_subnet_number) + ".250" + cls.testdata["shared_network_sg"]["gateway"] = "192.168." + str(random_subnet_number) + ".254" + cls.network2 = Network.create( + cls.apiclient, + cls.testdata["shared_network_sg"], + networkofferingid=cls.shared_network_offering.id, + zoneid=cls.zone.id, + accountid=cls.account1.name, + domainid=cls.account1.domainid + ) + + random_subnet_number = random.randrange(111, 120) + cls.testdata["shared_network_sg"]["name"] = "Shared-Network-SG-Test-vlan" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["displaytext"] = "Shared-Network-SG-Test-vlan" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["vlan"] = "vlan://" + str(random_subnet_number) + cls.testdata["shared_network_sg"]["startip"] = "192.168." + str(random_subnet_number) + ".240" + cls.testdata["shared_network_sg"]["endip"] = "192.168." + str(random_subnet_number) + ".250" + cls.testdata["shared_network_sg"]["gateway"] = "192.168." + str(random_subnet_number) + ".254" + cls.network3 = Network.create( + cls.apiclient, + cls.testdata["shared_network_sg"], + networkofferingid=cls.shared_network_offering.id, + zoneid=cls.zone.id, + accountid=cls.account1.name, + domainid=cls.account1.domainid + ) + + try: + cls.virtual_machine1 = VirtualMachine.create( + cls.apiclient, + cls.testdata["virtual_machine"], + accountid=cls.account1.name, + domainid=cls.account1.domainid, + serviceofferingid=cls.service_offering.id, + templateid=cls.template.id, + securitygroupids=[security_group.id], + networkids=cls.network1.id + ) + for nic in cls.virtual_machine1.nic: + if nic.isdefault: + cls.virtual_machine1.ssh_ip = nic.ipaddress + cls.virtual_machine1.default_network_id = nic.networkid + break + except Exception as e: + cls.fail("Exception while deploying virtual machine: %s" % e) + + try: + cls.virtual_machine2 = VirtualMachine.create( + cls.apiclient, + cls.testdata["virtual_machine"], + accountid=cls.account1.name, + domainid=cls.account1.domainid, + serviceofferingid=cls.service_offering.id, + templateid=cls.template.id, + securitygroupids=[security_group.id], + networkids=[str(cls.network1.id), str(cls.network2.id)] + ) + for nic in cls.virtual_machine2.nic: + if nic.isdefault: + cls.virtual_machine2.ssh_ip = nic.ipaddress + cls.virtual_machine2.default_network_id = nic.networkid + break + except Exception as e: + cls.fail("Exception while deploying virtual machine: %s" % e) + + cls._cleanup.append(cls.virtual_machine1) + cls._cleanup.append(cls.virtual_machine2) + cls._cleanup.append(cls.network1) + cls._cleanup.append(cls.network2) + cls._cleanup.append(cls.network3) + cls._cleanup.append(cls.shared_network_offering) + if cls.zone.securitygroupsenabled: + cls._cleanup.append(security_group) + cls._cleanup.append(cls.account1) + cls._cleanup.append(cls.user_domain) + + @classmethod + def tearDownClass(self): + try: + cleanup_resources(self.apiclient, self._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.cleanup = [] + return + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def verify_network_rules(self, vm_id): + virtual_machine = VirtualMachine.list( + self.apiclient, + id=vm_id + ) + vm = virtual_machine[0] + hosts = list_hosts( + self.apiclient, + id=vm.hostid + ) + host = hosts[0] + if host.hypervisor.lower() not in "kvm": + return + host.user, host.password = get_host_credentials(self.config, host.ipaddress) + for nic in vm.nic: + secips = "" + if len(nic.secondaryip) > 0: + for secip in nic.secondaryip: + secips += secip.ipaddress + ";" + command="/usr/share/cloudstack-common/scripts/vm/network/security_group.py verify_network_rules --vmname %s --vmip %s --vmmac %s --nicsecips '%s'" % (vm.instancename, nic.ipaddress, nic.macaddress, secips) + self.logger.debug("Executing command '%s' in host %s" % (command, host.ipaddress)) + result=execute_command_in_host(host.ipaddress, 22, + host.user, + host.password, + command) + if len(result) > 0: + self.fail("The iptables/ebtables rules for nic %s on vm %s on host %s are not correct" %(nic.ipaddress, vm.instancename, host.name)) + + @attr(tags=["adeancedsg"], required_hardware="false") + def test_01_create_vm_with_multiple_nics(self): + """Create Vm with multiple NIC's + + Steps: + # 1. Create more than 1 isolated or shared network + # 2. Create a vm and select more than 1 network while deploying + # 3. Vm is deployed successfully with 1 nic from each network + # 4. All the vm's should be pingable + :return: + """ + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine2.id + ) + self.assertEqual( + len(virtual_machine), 1, + "Virtual Machine create with 2 NIC's failed") + + nicIdInVm = virtual_machine[0].nic[0] + self.assertIsNotNone(nicIdInVm, "NIC 1 not found in Virtual Machine") + + nicIdInVm = virtual_machine[0].nic[1] + self.assertIsNotNone(nicIdInVm, "NIC 2 not found in Virtual Machine") + + self.verify_network_rules(self.virtual_machine2.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_02_add_nic_to_vm(self): + """Create VM with single NIC and then add additional NIC + + Steps: + # 1. Create a VM by selecting one default NIC + # 2. Create few more isolated or shared networks + # 3. Add extra NIC's to the vm from the newly created networks + # 4. The deployed VM should have extra nic's added in the above + # step without any fail + # 5. The IP's of the extra NIC's should be pingable + :return: + """ + self.virtual_machine1.add_nic(self.apiclient, self.network2.id) + + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine1.id + ) + + nicIdInVm = virtual_machine[0].nic[1] + self.assertIsNotNone(nicIdInVm, "Second NIC not found") + + self.verify_network_rules(self.virtual_machine1.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_03_add_ip_to_default_nic(self): + """ Add secondary IP's to the VM + + Steps: + # 1. Create a VM with more than 1 NIC + # 2) Navigate to Instances->NIC->Edit Secondary IP's + # ->Aquire new Secondary IP" + # 3) Add as many secondary Ip as possible to the VM + # 4) Configure the secondary IP's by referring to "Configure + # the secondary IP's" in the "Action Item" section + :return: + """ + ipaddress = NIC.addIp( + self.apiclient, + id=self.virtual_machine2.nic[0].id + ) + + self.assertIsNotNone( + ipaddress, + "Unable to add secondary IP to the default NIC") + + self.verify_network_rules(self.virtual_machine2.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_04_add_ip_to_remaining_nics(self): + """ Add secondary IP's to remaining NIC's + + Steps: + # 1) Create a VM with more than 1 NIC + # 2)Navigate to Instances-NIC's->Edit Secondary IP's + # ->Acquire new Secondary IP + # 3) Add secondary IP to all the NIC's of the VM + # 4) Confiugre the secondary IP's by referring to "Configure the + # secondary IP's" in the "Action Item" section + :return: + """ + + self.virtual_machine1.add_nic(self.apiclient, self.network3.id) + + vms = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine1.id + ) + + self.assertIsNotNone( + vms[0].nic[2], + "Third NIC is not added successfully to the VM") + + vms1_nic1_id = vms[0].nic[1]['id'] + vms1_nic2_id = vms[0].nic[2]['id'] + + ipaddress21 = NIC.addIp( + self.apiclient, + id=vms1_nic1_id + ) + + ipaddress22 = NIC.addIp( + self.apiclient, + id=vms1_nic1_id + ) + + self.assertIsNotNone( + ipaddress21, + "Unable to add first secondary IP to the second nic") + self.assertIsNotNone( + ipaddress22, + "Unable to add second secondary IP to second NIC") + + ipaddress31 = NIC.addIp( + self.apiclient, + id=vms1_nic2_id + ) + + ipaddress32 = NIC.addIp( + self.apiclient, + id=vms1_nic2_id + ) + + self.assertIsNotNone( + ipaddress31, + "Unable to add first secondary IP to third NIC") + self.assertIsNotNone( + ipaddress32, + "Unable to add second secondary IP to third NIC") + + self.verify_network_rules(self.virtual_machine1.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_05_stop_start_vm_with_multiple_nic(self): + """ Stop and Start a VM with Multple NIC + + Steps: + # 1) Create a Vm with multiple NIC's + # 2) Configure secondary IP's on the VM + # 3) Try to stop/start the VM + # 4) Ping the IP's of the vm + # 5) Remove Secondary IP from one of the NIC + :return: + """ + ipaddress1 = NIC.addIp( + self.apiclient, + id=self.virtual_machine2.nic[0].id + ) + + ipaddress2 = NIC.addIp( + self.apiclient, + id=self.virtual_machine2.nic[1].id + ) + # Stop the VM with multiple NIC's + self.virtual_machine2.stop(self.apiclient) + + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine2.id + ) + + self.assertEqual( + virtual_machine[0]['state'], 'Stopped', + "Could not stop the VM with multiple NIC's") + + if virtual_machine[0]['state'] == 'Stopped': + # If stopped then try to start the VM + self.virtual_machine2.start(self.apiclient) + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine2.id + ) + self.assertEqual( + virtual_machine[0]['state'], 'Running', + "Could not start the VM with multiple NIC's") + + self.verify_network_rules(self.virtual_machine2.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_06_migrate_vm_with_multiple_nic(self): + """ Migrate a VM with Multple NIC + + Steps: + # 1) Create a Vm with multiple NIC's + # 2) Configure secondary IP's on the VM + # 3) Try to stop/start the VM + # 4) Ping the IP's of the vm + :return: + """ + # Skipping adding Secondary IP to NIC since its already + # done in the previous test cases + + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine1.id + ) + old_host_id = virtual_machine[0]['hostid'] + + try: + hosts = Host.list( + self.apiclient, + virtualmachineid=self.virtual_machine1.id, + listall=True) + self.assertEqual( + validateList(hosts)[0], + PASS, + "hosts list validation failed") + + # Get a host which is not already assigned to VM + for host in hosts: + if host.id == old_host_id: + continue + else: + host_id = host.id + break + + self.virtual_machine1.migrate(self.apiclient, host_id) + except Exception as e: + self.fail("Exception occured: %s" % e) + + # List the vm again + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine1.id) + + new_host_id = virtual_machine[0]['hostid'] + + self.assertNotEqual( + old_host_id, new_host_id, + "Migration of VM to new host failed" + ) + + self.verify_network_rules(self.virtual_machine1.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_07_remove_secondary_ip_from_nic(self): + """ Remove secondary IP from any NIC + Steps: + # 1) Navigate to Instances + # 2) Select any vm + # 3) NIC's ->Edit secondary IP's->Release IP + # 4) The secondary IP should be successfully removed + """ + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine2.id) + + # Check which NIC is having secondary IP + secondary_ips = virtual_machine[0].nic[1].secondaryip + + for secondary_ip in secondary_ips: + NIC.removeIp(self.apiclient, ipaddressid=secondary_ip['id']) + + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine2.id + ) + + self.assertFalse( + virtual_machine[0].nic[1].secondaryip, + 'Failed to remove secondary IP') + + self.verify_network_rules(self.virtual_machine2.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_08_remove_nic_from_vm(self): + """ Remove NIC from VM + Steps: + # 1) Navigate to Instances->select any vm->NIC's->NIC 2 + # ->Click on "X" button to remove the second NIC + # 2) Remove other NIC's as well from the VM + # 3) All the NIC's should be successfully removed from the VM + :return: + """ + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine2.id) + + for nic in virtual_machine[0].nic: + if nic.isdefault: + continue + self.virtual_machine2.remove_nic(self.apiclient, nic.id) + + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine2.id) + + self.assertEqual( + len(virtual_machine[0].nic), 1, + "Failed to remove all the nics from the virtual machine") + + self.verify_network_rules(self.virtual_machine2.id) + + @attr(tags=["advancedsg"], required_hardware="false") + def test_09_reboot_vm_with_multiple_nic(self): + """ Reboot a VM with Multple NIC + + Steps: + # 1) Create a Vm with multiple NIC's + # 2) Configure secondary IP's on the VM + # 3) Try to reboot the VM + # 4) Ping the IP's of the vm + :return: + """ + # Skipping adding Secondary IP to NIC since its already + # done in the previous test cases + + virtual_machine = VirtualMachine.list( + self.apiclient, + id=self.virtual_machine1.id + ) + try: + self.virtual_machine1.reboot(self.apiclient) + except Exception as e: + self.fail("Exception occured: %s" % e) + + self.verify_network_rules(self.virtual_machine1.id) + diff --git a/tools/marvin/marvin/lib/utils.py b/tools/marvin/marvin/lib/utils.py index 1ad457ccf5b9..f170e0dc3bf6 100644 --- a/tools/marvin/marvin/lib/utils.py +++ b/tools/marvin/marvin/lib/utils.py @@ -62,13 +62,15 @@ def _configure_timeout(hypervisor): return timeout -def _execute_ssh_command(hostip, port, username, password, ssh_command): +def _execute_ssh_command(hostip, port, username, password, ssh_command, timeout=5): #SSH to the machine ssh = SshClient(hostip, port, username, password) # Ensure the SSH login is successful while True: res = ssh.execute(ssh_command) - if "Connection refused".lower() in res[0].lower(): + if len(res) == 0: + return res + elif "Connection refused".lower() in res[0].lower(): pass elif res[0] != "Host key verification failed.": break @@ -228,6 +230,10 @@ def get_host_credentials(config, hostip): raise Exception("Unresolvable host %s error is %s" % (hostip, e)) raise KeyError("Please provide the marvin configuration file with credentials to your hosts") +def execute_command_in_host(hostip, port, username, password, command, hypervisor=None): + timeout = _configure_timeout(hypervisor) + result = _execute_ssh_command(hostip, port, username, password, command) + return result def get_process_status(hostip, port, username, password, linklocalip, command, hypervisor=None): """Double hop and returns a command execution result""" diff --git a/ui/scripts/instanceWizard.js b/ui/scripts/instanceWizard.js index a6fdfbb3b952..04cceaa9d06c 100644 --- a/ui/scripts/instanceWizard.js +++ b/ui/scripts/instanceWizard.js @@ -168,11 +168,16 @@ }); } - return $.grep(selectedNetworks, function(network) { + var total = $.grep(selectedNetworks, function(network) { return $.grep(network.service, function(service) { return service.name == 'SecurityGroup'; }).length; }).length; //return total number of selected sg networks + + if (total > 0 && selectedHypervisor == "KVM") { + return -1; // vm with multiple IPs is supported in KVM + } + return total; }, // Data providers for each wizard step @@ -1284,8 +1289,10 @@ if (selectedZoneObj.networktype == "Advanced" && selectedZoneObj.securitygroupsenabled == true) { // Advanced SG-enabled zone var array2 = []; + var array3 = []; var myNetworks = $('.multi-wizard:visible form').data('my-networks'); //widget limitation: If using an advanced security group zone, get the guest networks like this - var defaultNetworkId = $('.multi-wizard:visible form').find('input[name=defaultNetwork]:checked').val(); + var defaultNetworkId = $('.multi-wizard:visible form').data('defaultNetwork'); + //var defaultNetworkId = $('.multi-wizard:visible form').find('input[name=defaultNetwork]:checked').val(); var checkedNetworkIdArray; if (typeof(myNetworks) == "object" && myNetworks.length != null) { //myNetworks is an array of string, e.g. ["203", "202"], @@ -1302,17 +1309,43 @@ array2.push(defaultNetworkId); } - //then, add other checked networks + var myNetworkIps = $('.multi-wizard:visible form').data('my-network-ips'); if (checkedNetworkIdArray.length > 0) { for (var i = 0; i < checkedNetworkIdArray.length; i++) { - if (checkedNetworkIdArray[i] != defaultNetworkId) //exclude defaultNetworkId that has been added to array2 + if (checkedNetworkIdArray[i] == defaultNetworkId) { + array2.unshift(defaultNetworkId); + + var ipToNetwork = { + networkid: defaultNetworkId + }; + if (myNetworkIps[i] != null && myNetworkIps[i].length > 0) { + $.extend(ipToNetwork, { + ip: myNetworkIps[i] + }); + } + array3.unshift(ipToNetwork); + } else { array2.push(checkedNetworkIdArray[i]); + + var ipToNetwork = { + networkid: checkedNetworkIdArray[i] + }; + if (myNetworkIps[i] != null && myNetworkIps[i].length > 0) { + $.extend(ipToNetwork, { + ip: myNetworkIps[i] + }); + } + array3.push(ipToNetwork); + } } } - $.extend(deployVmData, { - networkids : array2.join(",") - }); + for (var k = 0; k < array3.length; k++) { + deployVmData["iptonetworklist[" + k + "].networkid"] = array3[k].networkid; + if (array3[k].ip != undefined && array3[k].ip.length > 0) { + deployVmData["iptonetworklist[" + k + "].ip"] = array3[k].ip; + } + } } } else if (step6ContainerType == 'nothing-to-select') { if ("vpc" in args.context) { //from VPC tier diff --git a/ui/scripts/instances.js b/ui/scripts/instances.js index 545f7428faa8..c6fb671e1845 100644 --- a/ui/scripts/instances.js +++ b/ui/scripts/instances.js @@ -3354,6 +3354,16 @@ label: 'label.description' } }], + viewAll: { + path: 'network.securityGroups', + attachTo: 'id', + label: 'label.security.groups', + title: function(args) { + var title = _l('label.security.groups'); + + return title; + } + }, dataProvider: function(args) { // args.response.success({data: args.context.instances[0].securitygroup}); $.ajax({ diff --git a/ui/scripts/network.js b/ui/scripts/network.js index c7ca7a0b9ebc..5b52ced137e6 100644 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -4503,6 +4503,14 @@ var data = {}; listViewDataProvider(args, data); + if (args.context != null) { + if ("securityGroups" in args.context) { + $.extend(data, { + id: args.context.securityGroups[0].id + }); + } + } + $.ajax({ url: createURL('listSecurityGroups'), data: data, diff --git a/ui/scripts/ui-custom/instanceWizard.js b/ui/scripts/ui-custom/instanceWizard.js index b732b9ccf4b4..2450ed1b4a88 100644 --- a/ui/scripts/ui-custom/instanceWizard.js +++ b/ui/scripts/ui-custom/instanceWizard.js @@ -1516,14 +1516,29 @@ if (advSGFilter == 0) { //when total number of selected sg networks is 0, then 'Select Security Group' is skipped, go to step 6 directly showStep(6); - } else { //when total number of selected sg networks > 0 + } else if (advSGFilter > 0) { //when total number of selected sg networks > 0 if ($activeStep.find('input[type=checkbox]:checked').length > 1) { //when total number of selected networks > 1 cloudStack.dialog.notice({ message: "Can't create a vm with multiple networks one of which is Security Group enabled" }); return false; } + } else if (advSGFilter == -1) { // vm with multiple IPs is supported in KVM + var $selectNetwork = $activeStep.find('input[type=checkbox]:checked'); + var myNetworkIps = []; + $selectNetwork.each(function() { + var $specifyIp = $(this).parent().find('.specify-ip input[type=text]'); + myNetworkIps.push($specifyIp.val() == -1 ? null : $specifyIp.val()); + }); + $activeStep.closest('form').data('my-network-ips', myNetworkIps); + $selectNetwork.each(function() { + if ($(this).parent().find('input[type=radio]').is(':checked')) { + $activeStep.closest('form').data('defaultNetwork', $(this).val()); + return; + } + }) } + } }