NSX: Cleanup NSX resources during k8s cluster cleanup (#8528)

Pearl1594 · web-flow · commit 080f171c6d70 · 2024-01-19T12:48:08.000-03:00
diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
@@ -22,4 +22,5 @@
 public interface KubernetesClusterHelper extends Adapter {
 
     ControlledEntity findByUuid(String uuid);
+    ControlledEntity findByVmId(long vmId);
 }
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
@@ -17,25 +17,38 @@
 package com.cloud.kubernetes.cluster;
 
 import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
 import com.cloud.utils.component.AdapterBase;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.springframework.stereotype.Component;
 
 import javax.inject.Inject;
+import java.util.Objects;
 
 @Component
 public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable {
 
     @Inject
     private KubernetesClusterDao kubernetesClusterDao;
+    @Inject
+    private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
 
     @Override
     public ControlledEntity findByUuid(String uuid) {
         return kubernetesClusterDao.findByUuid(uuid);
     }
 
+    @Override
+    public ControlledEntity findByVmId(long vmId) {
+        KubernetesClusterVmMapVO clusterVmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId);
+        if (Objects.isNull(clusterVmMapVO)) {
+            return null;
+        }
+        return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId());
+    }
+
     @Override
     public String getConfigComponentName() {
         return KubernetesClusterHelper.class.getSimpleName();
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
@@ -23,6 +23,8 @@
 
 public interface KubernetesClusterVmMapDao extends GenericDao<KubernetesClusterVmMapVO, Long> {
     public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId);
+
+    public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId);
     public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
 
     int removeByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
@@ -31,12 +31,17 @@
 public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClusterVmMapVO, Long> implements KubernetesClusterVmMapDao {
 
     private final SearchBuilder<KubernetesClusterVmMapVO> clusterIdSearch;
+    private final SearchBuilder<KubernetesClusterVmMapVO> vmIdSearch;
 
     public KubernetesClusterVmMapDaoImpl() {
         clusterIdSearch = createSearchBuilder();
         clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
         clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(), SearchCriteria.Op.IN);
         clusterIdSearch.done();
+
+        vmIdSearch = createSearchBuilder();
+        vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
+        vmIdSearch.done();
     }
 
     @Override
@@ -47,6 +52,13 @@ public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId) {
         return listBy(sc, filter);
     }
 
+    @Override
+    public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) {
+        SearchCriteria<KubernetesClusterVmMapVO> sc = vmIdSearch.create();
+        sc.setParameters("vmId", vmId);
+        return findOneBy(sc);
+    }
+
     @Override
     public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds) {
         SearchCriteria<KubernetesClusterVmMapVO> sc = clusterIdSearch.create();
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -505,10 +505,12 @@ public boolean applyPFRules(Network network, List<PortForwardingRule> rules) thr
         if (!canHandle(network, Network.Service.PortForwarding)) {
             return false;
         }
+        boolean result = true;
         for (PortForwardingRule rule : rules) {
             IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId());
             UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId());
-            if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) {
+            if ((vm == null && (rule.getState() != FirewallRule.State.Revoke)) ||
+                    (vm != null && networkModel.getNicInNetwork(vm.getId(), network.getId()) == null)) {
                 continue;
             }
             NsxOpObject nsxObject = getNsxOpObject(network);
@@ -523,21 +525,21 @@ public boolean applyPFRules(Network network, List<PortForwardingRule> rules) thr
                     .setNetworkResourceId(nsxObject.getNetworkResourceId())
                     .setNetworkResourceName(nsxObject.getNetworkResourceName())
                     .setVpcResource(nsxObject.isVpcResource())
-                    .setVmId(vm.getId())
-                    .setVmIp(vm.getPrivateIpAddress())
+                    .setVmId(Objects.nonNull(vm) ? vm.getId() : 0)
+                    .setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() : null)
                     .setPublicIp(publicIp.getAddress().addr())
                     .setPrivatePort(privatePort)
                     .setPublicPort(publicPort)
                     .setRuleId(rule.getId())
                     .setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT))
                     .build();
             if (rule.getState() == FirewallRule.State.Add) {
-                return nsxService.createPortForwardRule(networkRule);
+                result &= nsxService.createPortForwardRule(networkRule);
             } else if (rule.getState() == FirewallRule.State.Revoke) {
-                return nsxService.deletePortForwardRule(networkRule);
+                result &= nsxService.deletePortForwardRule(networkRule);
             }
         }
-        return true;
+        return result;
     }
 
     public Pair<VpcVO, NetworkVO> getVpcOrNetwork(Long vpcId, long networkId) {
@@ -613,6 +615,7 @@ private NsxOpObject getNsxOpObject(Network network) {
 
     @Override
     public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) throws ResourceUnavailableException {
+        boolean result = true;
         for (LoadBalancingRule loadBalancingRule : rules) {
             if (loadBalancingRule.getState() == FirewallRule.State.Active) {
                 continue;
@@ -638,12 +641,12 @@ public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) thro
                     .setAlgorithm(loadBalancingRule.getAlgorithm())
                     .build();
             if (loadBalancingRule.getState() == FirewallRule.State.Add) {
-                return nsxService.createLbRule(networkRule);
+                result &= nsxService.createLbRule(networkRule);
             } else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) {
-                return nsxService.deleteLbRule(networkRule);
+                result &= nsxService.deleteLbRule(networkRule);
             }
         }
-        return true;
+        return result;
     }
 
     @Override
diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
@@ -51,6 +51,9 @@
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.ParserConfigurationException;
 
+import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
+import com.cloud.network.dao.NsxProviderDao;
+import com.cloud.network.element.NsxProviderVO;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -589,6 +592,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
     @Inject
     VMScheduleManager vmScheduleManager;
+    @Inject
+    NsxProviderDao nsxProviderDao;
 
     private ScheduledExecutorService _executor = null;
     private ScheduledExecutorService _vmIpFetchExecutor = null;
@@ -597,6 +602,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private boolean _dailyOrHourly = false;
     private int capacityReleaseInterval;
     private ExecutorService _vmIpFetchThreadExecutor;
+    private List<KubernetesClusterHelper> kubernetesClusterHelpers;
 
 
     private String _instance;
@@ -610,6 +616,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private static final int NUM_OF_2K_BLOCKS = 512;
     private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES;
 
+    public List<KubernetesClusterHelper> getKubernetesClusterHelpers() {
+        return kubernetesClusterHelpers;
+    }
+
+    public void setKubernetesClusterHelpers(final List<KubernetesClusterHelper> kubernetesClusterHelpers) {
+        this.kubernetesClusterHelpers = kubernetesClusterHelpers;
+    }
+
     @Inject
     private OrchestrationService _orchSrvc;
 
@@ -2528,11 +2542,15 @@ private boolean cleanupVmResources(long vmId) {
         }
 
         // cleanup port forwarding rules
-        if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
-            s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
-        } else {
-            success = false;
-            s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+        VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
+        NsxProviderVO nsx = nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId());
+        if (Objects.isNull(nsx) || Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) {
+            if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
+                s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
+            } else {
+                success = false;
+                s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+            }
         }
 
         // cleanup load balancer rules
diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
@@ -106,8 +106,9 @@
 
     <bean id="configurationServerImpl" class="com.cloud.server.ConfigurationServerImpl" />
 
-
-    <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl" />
+    <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl">
+        <property name="kubernetesClusterHelpers" value="#{kubernetesClusterHelperRegistry.registered}" />
+    </bean>
 
     <bean id="consoleProxyManagerImpl" class="com.cloud.consoleproxy.ConsoleProxyManagerImpl">
         <property name="consoleProxyAllocators"

Original file line number	Diff line number	Diff line change
`@@ -22,4 +22,5 @@`
`22`	`22`	`public interface KubernetesClusterHelper extends Adapter {`
`23`	`23`
`24`	`24`	`ControlledEntity findByUuid(String uuid);`
	`25`	`+ ControlledEntity findByVmId(long vmId);`
`25`	`26`	`}`