Externalize retries and inverval for NSX segment deletion (#67)

Author: nvazquez

api/src/main/java/com/cloud/network/nsx/NsxService.java

@@ -18,9 +18,19 @@
 
 import com.cloud.network.IpAddress;
 import com.cloud.network.vpc.Vpc;
+import org.apache.cloudstack.framework.config.ConfigKey;
 
 public interface NsxService {
 
+    ConfigKey<Integer> NSX_API_FAILURE_RETRIES = new ConfigKey<>("Advanced", Integer.class,
+            "nsx.api.failure.retries", "30",
+            "Number of retries for NSX API operations in case of failures",
+            true, ConfigKey.Scope.Zone);
+    ConfigKey<Integer> NSX_API_FAILURE_INTERVAL = new ConfigKey<>("Advanced", Integer.class,
+            "nsx.api.failure.interval", "60",
+            "Waiting time (in seconds) before retrying an NSX API operation in case of failure",
+            true, ConfigKey.Scope.Zone);
+
     boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled);
     boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
 }

plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java

@@ -17,6 +17,7 @@
 package org.apache.cloudstack.service;
 
 import com.cloud.network.Network;
+import com.cloud.network.nsx.NsxService;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.vmware.nsx.cluster.Status;
 import com.vmware.nsx.model.ClusterStatus;
@@ -460,7 +461,7 @@ public void deleteSegment(long zoneId, long domainId, long accountId, Long vpcId
                 String t1GatewayName = getTier1GatewayName(domainId, accountId, zoneId, networkId, false);
                 deleteLoadBalancer(getLoadBalancerName(t1GatewayName));
             }
-            removeSegment(segmentName);
+            removeSegment(segmentName, zoneId);
             DhcpRelayConfigs dhcpRelayConfig = (DhcpRelayConfigs) nsxService.apply(DhcpRelayConfigs.class);
             String dhcpRelayConfigId = NsxControllerUtils.getNsxDhcpRelayConfigId(zoneId, domainId, accountId, vpcId, networkId);
             logger.debug(String.format("Removing the DHCP relay config with ID %s", dhcpRelayConfigId));
@@ -473,7 +474,8 @@ public void deleteSegment(long zoneId, long domainId, long accountId, Long vpcId
         }
     }
 
-    protected void removeSegment(String segmentName) {
+
+    protected void removeSegment(String segmentName, long zoneId) {
         logger.debug(String.format("Removing the segment with ID %s", segmentName));
         Segments segmentService = (Segments) nsxService.apply(Segments.class);
         String errMsg = String.format("The segment with ID %s is not found, skipping removal", segmentName);
@@ -492,15 +494,16 @@ protected void removeSegment(String segmentName) {
         SegmentPorts segmentPortsService = (SegmentPorts) nsxService.apply(SegmentPorts.class);
         PolicyGroupMembersListResult segmentPortsList = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath);
         Long portCount = segmentPortsList.getResultCount();
-        portCount = retrySegmentDeletion(segmentPortsService, portCount, segmentName, enforcementPointPath);
-        logger.info("Port count: " + portCount);
+        if (portCount > 0L) {
+            portCount = retrySegmentDeletion(segmentPortsService, segmentName, enforcementPointPath, zoneId);
+        }
         if (portCount == 0L) {
             logger.debug(String.format("Removing the segment with ID %s", segmentName));
             removeGroupForSegment(segmentName);
             segmentService.delete(segmentName);
         } else {
             String msg = String.format("Cannot remove the NSX segment %s because there are still %s port group(s) attached to it", segmentName, portCount);
-            logger.debug(msg);
+            logger.error(msg);
             throw new CloudRuntimeException(msg);
         }
     }
@@ -510,13 +513,16 @@ private PolicyGroupMembersListResult getSegmentPortList(SegmentPorts segmentPort
                 false, null, 50L, false, null);
     }
 
-    private Long retrySegmentDeletion(SegmentPorts segmentPortsService, Long portCount, String segmentName, String enforcementPointPath) {
-        int retries = 20;
+    private Long retrySegmentDeletion(SegmentPorts segmentPortsService, String segmentName, String enforcementPointPath, long zoneId) {
+        int retries = NsxService.NSX_API_FAILURE_RETRIES.valueIn(zoneId);
+        int waitingSecs = NsxService.NSX_API_FAILURE_INTERVAL.valueIn(zoneId);
         int count = 1;
+        Long portCount;
         do {
             try {
-                logger.info("Waiting for all port groups to be unlinked from the segment - Attempt: " + count++ + " Waiting for 5 secs");
-                Thread.sleep(5000);
+                logger.info(String.format("Waiting for all port groups to be unlinked from the segment %s - " +
+                        "Attempt: %s. Waiting for %s secs", segmentName, count++, waitingSecs));
+                Thread.sleep(waitingSecs * 1000L);
                 portCount = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath).getResultCount();
                 retries--;
             } catch (InterruptedException e) {

plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java

@@ -19,7 +19,6 @@
 import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
 import com.cloud.network.nsx.NsxService;
-import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkVO;
 import com.cloud.network.vpc.Vpc;
 import com.cloud.network.vpc.VpcVO;
@@ -37,6 +36,8 @@
 import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand;
 import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand;
 import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.resource.NsxNetworkRule;
 import org.apache.cloudstack.utils.NsxControllerUtils;
 import org.apache.cloudstack.utils.NsxHelper;
@@ -47,13 +48,11 @@
 import java.util.List;
 import java.util.Objects;
 
-public class NsxServiceImpl implements NsxService {
+public class NsxServiceImpl implements NsxService, Configurable {
     @Inject
     NsxControllerUtils nsxControllerUtils;
     @Inject
     VpcDao vpcDao;
-    @Inject
-    NetworkDao networkDao;
 
     protected Logger logger = LogManager.getLogger(getClass());
 
@@ -189,4 +188,16 @@ public boolean deleteFirewallRules(Network network, List<NsxNetworkRule> netRule
         NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, network.getDataCenterId());
         return result.getResult();
     }
+
+    @Override
+    public String getConfigComponentName() {
+        return NsxApiClient.class.getSimpleName();
+    }
+
+    @Override
+    public ConfigKey<?>[] getConfigKeys() {
+        return new ConfigKey<?>[] {
+            NSX_API_FAILURE_RETRIES, NSX_API_FAILURE_INTERVAL
+        };
+    }
 }