From e34ea1f4e659a33bf287546ef58fcd68a70630dd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 13 Oct 2023 15:07:05 +0530 Subject: [PATCH 1/3] kvm: fix direct download HTTP template size Fixes #8037 Signed-off-by: Abhishek Kumar --- utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index 32a54722aaeb..4867344bed1d 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -114,7 +114,8 @@ private static long getVirtualSizeFromInputStream(InputStream inputStream) throw public static long getVirtualSize(String urlStr) { try { URL url = new URL(urlStr); - return getVirtualSizeFromInputStream(url.openStream()); + boolean isCompressed = !urlStr.endsWith("qcow2"); + return getVirtualSize(url.openStream(), isCompressed); } catch (MalformedURLException e) { LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); throw new IllegalArgumentException("Invalid URL: " + urlStr); From dd8d3584b12b0d84d7467f0f908a29d9dffa901f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 13 Oct 2023 16:03:17 +0530 Subject: [PATCH 2/3] compressed only for particular formats Signed-off-by: Abhishek Kumar --- .../HttpsDirectTemplateDownloader.java | 51 ++++++++++--------- .../com/cloud/utils/storage/QCOW2Utils.java | 3 +- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java index 1bee45c477d2..28c7a7949197 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -19,29 +19,6 @@ package org.apache.cloudstack.direct.download; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; -import com.cloud.utils.storage.QCOW2Utils; -import org.apache.cloudstack.utils.security.SSLUtils; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.io.IOUtils; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.commons.collections.MapUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -60,6 +37,32 @@ import java.util.List; import java.util.Map; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; + +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.utils.storage.QCOW2Utils; + public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected CloseableHttpClient httpsClient; @@ -183,7 +186,7 @@ public Long getRemoteFileSize(String url, String format) { SSLContext context = getSSLContext(); urlConnection.setSSLSocketFactory(context.getSocketFactory()); urlConnection.connect(); - boolean isCompressed = !url.endsWith("qcow2"); + boolean isCompressed = UriUtils.COMMPRESSION_FORMATS.stream().anyMatch(f -> !url.toLowerCase().endsWith(f)); return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed); } catch (IOException e) { throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index 4867344bed1d..deea179a7613 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -32,6 +32,7 @@ import org.apache.log4j.Logger; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.UriUtils; public final class QCOW2Utils { public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName()); @@ -114,7 +115,7 @@ private static long getVirtualSizeFromInputStream(InputStream inputStream) throw public static long getVirtualSize(String urlStr) { try { URL url = new URL(urlStr); - boolean isCompressed = !urlStr.endsWith("qcow2"); + boolean isCompressed = UriUtils.COMMPRESSION_FORMATS.stream().anyMatch(f -> !urlStr.toLowerCase().endsWith(f)); return getVirtualSize(url.openStream(), isCompressed); } catch (MalformedURLException e) { LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); From 9d09b24fedba3a54b3d60160b125d35b34d1e9d1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 14 Oct 2023 21:52:01 +0530 Subject: [PATCH 3/3] refactor Signed-off-by: Abhishek Kumar --- .../download/HttpsDirectTemplateDownloader.java | 3 +-- utils/src/main/java/com/cloud/utils/UriUtils.java | 8 ++++++-- .../java/com/cloud/utils/storage/QCOW2Utils.java | 3 +-- .../utils/imagestore/ImageStoreUtil.java | 2 +- .../test/java/com/cloud/utils/UriUtilsTest.java | 14 +++++++++++--- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java index 28c7a7949197..70a3eb29bc7a 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -186,8 +186,7 @@ public Long getRemoteFileSize(String url, String format) { SSLContext context = getSSLContext(); urlConnection.setSSLSocketFactory(context.getSocketFactory()); urlConnection.connect(); - boolean isCompressed = UriUtils.COMMPRESSION_FORMATS.stream().anyMatch(f -> !url.toLowerCase().endsWith(f)); - return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed); + return QCOW2Utils.getVirtualSize(urlObj.openStream(), UriUtils.isUrlForCompressedFile(url)); } catch (IOException e) { throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); } diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index dffc0106e8a8..a2bfa9eaa6fa 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -408,14 +408,14 @@ public static List getMetalinkUrls(String metalinkUrl) { return urls; } - public static final Set COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); + public static final Set COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); public static final Set buildExtensionSet(boolean metalink, String... baseExtensions) { final ImmutableSet.Builder builder = ImmutableSet.builder(); for (String baseExtension : baseExtensions) { builder.add("." + baseExtension); - for (String format : COMMPRESSION_FORMATS) { + for (String format : COMPRESSION_FORMATS) { builder.add("." + baseExtension + "." + format); } } @@ -647,4 +647,8 @@ private static UriInfo getRbdUrlInfo(String url) { throw new CloudRuntimeException(url + " is not a valid uri for RBD"); } } + + public static boolean isUrlForCompressedFile(String url) { + return UriUtils.COMPRESSION_FORMATS.stream().anyMatch(f -> url.toLowerCase().endsWith(f)); + } } diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index deea179a7613..4daf138bb8c8 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -115,8 +115,7 @@ private static long getVirtualSizeFromInputStream(InputStream inputStream) throw public static long getVirtualSize(String urlStr) { try { URL url = new URL(urlStr); - boolean isCompressed = UriUtils.COMMPRESSION_FORMATS.stream().anyMatch(f -> !urlStr.toLowerCase().endsWith(f)); - return getVirtualSize(url.openStream(), isCompressed); + return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr)); } catch (MalformedURLException e) { LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); throw new IllegalArgumentException("Invalid URL: " + urlStr); diff --git a/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java index 02878cf29606..2b37ce5fb72f 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java @@ -101,7 +101,7 @@ public static boolean isCorrectExtension(String path, String format) { public static boolean isCompressedExtension(String path) { final String lowerCasePath = path.toLowerCase(); - return UriUtils.COMMPRESSION_FORMATS + return UriUtils.COMPRESSION_FORMATS .stream() .map(extension -> "." + extension) .anyMatch(lowerCasePath::endsWith); diff --git a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java index 1a3ae17f584e..4ec1f9a9bd92 100644 --- a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java @@ -19,13 +19,13 @@ package com.cloud.utils; -import org.junit.Assert; -import org.junit.Test; - import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.junit.Assert; +import org.junit.Test; + public class UriUtilsTest { @Test public void encodeURIComponent() { @@ -265,4 +265,12 @@ public void testGetUriInfoIpv6() { testGetUriInfoInternal(url11, host); testGetUriInfoInternal(url12, host); } + + @Test + public void testIsUrlForCompressedFile() { + Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2")); + Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip")); + Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz")); + Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2")); + } }