From 6f86193f336de75f5c4a400ecb9aa5e645a5702d Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 23 Jan 2024 08:46:13 -0500 Subject: [PATCH 1/2] NSX: Add retry logic with sleep to delete segments --- .../cloudstack/service/NsxApiClient.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java index d4ac0e7a7e2e..1e2c17c44724 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java @@ -453,8 +453,9 @@ protected void removeSegment(String segmentName) { String siteId = getDefaultSiteId(); String enforcementPointPath = getDefaultEnforcementPointPath(siteId); SegmentPorts segmentPortsService = (SegmentPorts) nsxService.apply(SegmentPorts.class); - PolicyGroupMembersListResult segmentPortsList = segmentPortsService.list(DEFAULT_DOMAIN, segmentName, null, enforcementPointPath, - false, null, 50L, false, null); + PolicyGroupMembersListResult segmentPortsList = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath); + Long portCount = segmentPortsList.getResultCount(); + portCount = retrySegmentDeletion(segmentPortsService, portCount, segmentName, enforcementPointPath); if (segmentPortsList.getResultCount() == 0L) { LOGGER.debug(String.format("Removing the segment with ID %s", segmentName)); removeGroupForSegment(segmentName); @@ -466,6 +467,25 @@ protected void removeSegment(String segmentName) { } } + private PolicyGroupMembersListResult getSegmentPortList(SegmentPorts segmentPortsService, String segmentName, String enforcementPointPath) { + return segmentPortsService.list(DEFAULT_DOMAIN, segmentName, null, enforcementPointPath, + false, null, 50L, false, null); + } + + private Long retrySegmentDeletion(SegmentPorts segmentPortsService, Long portCount, String segmentName, String enforcementPointPath) { + int retries = 5; + do { + try { + Thread.sleep(5000); + portCount = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath).getResultCount(); + retries--; + } catch (InterruptedException e) { + throw new CloudRuntimeException(String.format("Unable to delete segment %s due to: %s", segmentName, e.getLocalizedMessage())); + } + } while (retries > 0 && portCount > 0); + return portCount; + } + public void createStaticNatRule(String vpcName, String tier1GatewayName, String ruleName, String publicIp, String vmIp) { try { From db06d5a5fd8b6b0dd19961fede5b67536de10ce2 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 23 Jan 2024 09:24:06 -0500 Subject: [PATCH 2/2] add logs --- .../java/org/apache/cloudstack/service/NsxApiClient.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java index 1e2c17c44724..cdcb83ec572a 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java @@ -456,12 +456,13 @@ protected void removeSegment(String segmentName) { PolicyGroupMembersListResult segmentPortsList = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath); Long portCount = segmentPortsList.getResultCount(); portCount = retrySegmentDeletion(segmentPortsService, portCount, segmentName, enforcementPointPath); - if (segmentPortsList.getResultCount() == 0L) { + LOGGER.info("Port count: " + portCount); + if (portCount == 0L) { LOGGER.debug(String.format("Removing the segment with ID %s", segmentName)); removeGroupForSegment(segmentName); segmentService.delete(segmentName); } else { - String msg = String.format("Cannot remove the NSX segment %s because there are still %s port group(s) attached to it", segmentName, segmentPortsList.getResultCount()); + String msg = String.format("Cannot remove the NSX segment %s because there are still %s port group(s) attached to it", segmentName, portCount); LOGGER.debug(msg); throw new CloudRuntimeException(msg); } @@ -473,9 +474,11 @@ private PolicyGroupMembersListResult getSegmentPortList(SegmentPorts segmentPort } private Long retrySegmentDeletion(SegmentPorts segmentPortsService, Long portCount, String segmentName, String enforcementPointPath) { - int retries = 5; + int retries = 20; + int count = 1; do { try { + LOGGER.info("Waiting for all port groups to be unlinked from the segment - Attempt: " + count++ + " Waiting for 5 secs"); Thread.sleep(5000); portCount = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath).getResultCount(); retries--;