From dbd2768ca16b3e41f12e7218b1a45aee9310af11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karolis=20Vy=C4=8Dius?= Date: Tue, 2 Jul 2024 12:11:38 +0300 Subject: [PATCH 1/5] Increase ZIP_THRESHOLD_SIZE to 100GB (#943) --- .../main/java/com/onthegomap/planetiler/util/FileUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java index 431cc3ddea..ed9286298a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java @@ -38,7 +38,7 @@ public class FileUtils { private static final Format FORMAT = Format.defaultInstance(); // Prevent zip-bomb attack, see https://rules.sonarsource.com/java/RSPEC-5042 private static final int ZIP_THRESHOLD_ENTRIES = 10_000; - private static final int ZIP_THRESHOLD_SIZE = 1_000_000_000; + private static final long ZIP_THRESHOLD_SIZE = 100_000_000_000L; private static final double ZIP_THRESHOLD_RATIO = 1_000; private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); @@ -271,7 +271,7 @@ public static void unzipResource(String resource, Path dest) { */ public static void safeCopy(InputStream inputStream, Path destPath) { try (var outputStream = Files.newOutputStream(destPath, StandardOpenOption.CREATE, WRITE)) { - int totalSize = 0; + long totalSize = 0; int nBytes; byte[] buffer = new byte[2048]; @@ -295,7 +295,7 @@ public static void safeCopy(InputStream inputStream, Path destPath) { * @throws UncheckedIOException if an IO exception occurs */ public static void unzip(InputStream input, Path destDir) { - int totalSizeArchive = 0; + long totalSizeArchive = 0; int totalEntryArchive = 0; try (var zip = new ZipInputStream(input)) { ZipEntry entry; From bbea298da615a6fca8a06b8a108b4a56d9417e1e Mon Sep 17 00:00:00 2001 From: Michael Barry Date: Tue, 2 Jul 2024 07:33:20 -0400 Subject: [PATCH 2/5] Handle no content length in downloader (#944) --- .../planetiler/util/Downloader.java | 39 +++++++++------ .../planetiler/util/DownloaderTest.java | 47 ++++++++++++------- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java index 43e10961e3..7bad09b2d4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -165,7 +166,7 @@ public void run() { for (var toDownload : toDownloadList) { try { - long size = toDownload.metadata.get(10, TimeUnit.SECONDS).size; + long size = toDownload.metadata.get(10, TimeUnit.SECONDS).size.orElse(0); loggers.addStorageRatePercentCounter(toDownload.id, size, toDownload::bytesDownloaded, true); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -183,14 +184,19 @@ CompletableFuture downloadIfNecessary(ResourceToDownload resourceToDownloa return CompletableFuture.runAsync(RunnableThatThrows.wrap(() -> { LogUtil.setStage("download", resourceToDownload.id); long existingSize = FileUtils.size(resourceToDownload.output); - var metadata = httpHeadFollowRedirects(resourceToDownload.url, 0); - Path tmpPath = resourceToDownload.tmpPath(); - resourceToDownload.metadata.complete(metadata); - if (metadata.size == existingSize) { - LOGGER.info("Skipping {}: {} already up-to-date", resourceToDownload.id, resourceToDownload.output); - return; + try { + resourceToDownload.metadata.complete(httpHeadFollowRedirects(resourceToDownload.url, 0)); + } catch (Exception e) { + resourceToDownload.metadata.completeExceptionally(e); + throw e; } + Path tmpPath = resourceToDownload.tmpPath(); try { + var metadata = resourceToDownload.metadata.get(); + if (metadata.size.orElse(-1) == existingSize) { + LOGGER.info("Skipping {}: {} already up-to-date", resourceToDownload.id, resourceToDownload.output); + return; + } String redirectInfo = metadata.canonicalUrl.equals(resourceToDownload.url) ? "" : " (redirected to " + metadata.canonicalUrl + ")"; LOGGER.info("Downloading {}{} to {}", resourceToDownload.url, redirectInfo, resourceToDownload.output); @@ -198,7 +204,9 @@ CompletableFuture downloadIfNecessary(ResourceToDownload resourceToDownloa FileUtils.createParentDirectories(resourceToDownload.output); FileUtils.delete(tmpPath); FileUtils.deleteOnExit(tmpPath); - diskSpaceCheck.addDisk(tmpPath, metadata.size, resourceToDownload.id); + if (metadata.size.isPresent()) { + diskSpaceCheck.addDisk(tmpPath, metadata.size.getAsLong(), resourceToDownload.id); + } diskSpaceCheck.checkAgainstLimits(config.force(), false); httpDownload(resourceToDownload, tmpPath); Files.move(tmpPath, resourceToDownload.output); @@ -225,7 +233,7 @@ ResourceMetadata httpHead(String url) throws IOException, InterruptedException { responseInfo -> { int status = responseInfo.statusCode(); Optional location = Optional.empty(); - long contentLength = 0; + OptionalLong contentLength = OptionalLong.empty(); HttpHeaders headers = responseInfo.headers(); if (status >= 300 && status < 400) { location = responseInfo.headers().firstValue(LOCATION); @@ -235,7 +243,7 @@ ResourceMetadata httpHead(String url) throws IOException, InterruptedException { } else if (responseInfo.statusCode() != 200) { throw new IllegalStateException("Bad response: " + responseInfo.statusCode()); } else { - contentLength = headers.firstValueAsLong(CONTENT_LENGTH).orElseThrow(); + contentLength = headers.firstValueAsLong(CONTENT_LENGTH); } boolean supportsRangeRequest = headers.allValues(ACCEPT_RANGES).contains("bytes"); ResourceMetadata metadata = new ResourceMetadata(location, url, contentLength, supportsRangeRequest); @@ -250,12 +258,13 @@ private void httpDownload(ResourceToDownload resource, Path tmpPath) record Range(long start, long end) {} List chunks = new ArrayList<>(); boolean ranges = metadata.acceptRange && config.downloadThreads() > 1; - long chunkSize = ranges ? chunkSizeBytes : metadata.size; - for (long start = 0; start < metadata.size; start += chunkSize) { - long end = Math.min(start + chunkSize, metadata.size); + long fileSize = metadata.size.orElse(Long.MAX_VALUE); + long chunkSize = ranges ? chunkSizeBytes : fileSize; + for (long start = 0; start < fileSize; start += chunkSize) { + long end = Math.min(start + chunkSize, fileSize); chunks.add(new Range(start, end)); } - FileUtils.setLength(tmpPath, metadata.size); + FileUtils.setLength(tmpPath, metadata.size.orElse(1)); Semaphore perFileLimiter = new Semaphore(config.downloadThreads()); Worker.joinFutures(chunks.stream().map(range -> CompletableFuture.runAsync(RunnableThatThrows.wrap(() -> { LogUtil.setStage("download", resource.id); @@ -299,7 +308,7 @@ private HttpRequest.Builder newHttpRequest(String url) { .header(USER_AGENT, config.httpUserAgent()); } - record ResourceMetadata(Optional redirect, String canonicalUrl, long size, boolean acceptRange) {} + record ResourceMetadata(Optional redirect, String canonicalUrl, OptionalLong size, boolean acceptRange) {} record ResourceToDownload( String id, String url, Path output, CompletableFuture metadata, diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java index f5b285b265..37ff10b7e0 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java @@ -10,6 +10,7 @@ import java.nio.file.Path; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; @@ -25,7 +26,8 @@ class DownloaderTest { private final PlanetilerConfig config = PlanetilerConfig.defaults(); private AtomicLong downloads = new AtomicLong(0); - private Downloader mockDownloader(Map resources, boolean supportsRange) { + private Downloader mockDownloader(Map resources, boolean supportsRange, + boolean supportsContentLength) { return new Downloader(config, 2L) { @Override @@ -55,23 +57,30 @@ ResourceMetadata httpHead(String url) { if (parts.length > 1) { int redirectNum = Integer.parseInt(parts[1]); String next = redirectNum <= 1 ? parts[0] : (parts[0] + "#" + (redirectNum - 1)); - return new ResourceMetadata(Optional.of(next), url, 0, supportsRange); + return new ResourceMetadata(Optional.of(next), url, + supportsContentLength ? OptionalLong.of(0) : OptionalLong.empty(), supportsRange); } byte[] bytes = resources.get(url); - return new ResourceMetadata(Optional.empty(), url, bytes.length, supportsRange); + return new ResourceMetadata(Optional.empty(), url, + supportsContentLength ? OptionalLong.of(bytes.length) : OptionalLong.empty(), supportsRange); } }; } @ParameterizedTest @CsvSource({ - "false,0", - "true,0", - "false,1", - "false,2", - "true,4", + "false,0,true", + "true,0,true", + "false,1,true", + "false,2,true", + "true,4,true", + + "false,0,false", + "true,0,false", + "false,1,false", + "true,1,false", }) - void testDownload(boolean range, int redirects) throws Exception { + void testDownload(boolean range, int redirects, boolean supportsContentLength) throws Exception { Path dest = path.resolve("out"); String string = "0123456789"; String url = "http://url"; @@ -79,7 +88,7 @@ void testDownload(boolean range, int redirects) throws Exception { Map resources = new ConcurrentHashMap<>(); byte[] bytes = string.getBytes(StandardCharsets.UTF_8); - Downloader downloader = mockDownloader(resources, range); + Downloader downloader = mockDownloader(resources, range, supportsContentLength); // fails if no data var resource1 = new Downloader.ResourceToDownload("resource", initialUrl, dest); @@ -96,13 +105,15 @@ void testDownload(boolean range, int redirects) throws Exception { assertEquals(10, resource2.bytesDownloaded()); // does not re-request if size is the same - downloads.set(0); - var resource3 = new Downloader.ResourceToDownload("resource", initialUrl, dest); - downloader.downloadIfNecessary(resource3).get(); - assertEquals(0, downloads.get()); - assertEquals(string, Files.readString(dest)); - assertEquals(FileUtils.size(path), FileUtils.size(dest)); - assertEquals(0, resource3.bytesDownloaded()); + if (supportsContentLength) { + downloads.set(0); + var resource3 = new Downloader.ResourceToDownload("resource", initialUrl, dest); + downloader.downloadIfNecessary(resource3).get(); + assertEquals(0, downloads.get()); + assertEquals(string, Files.readString(dest)); + assertEquals(FileUtils.size(path), FileUtils.size(dest)); + assertEquals(0, resource3.bytesDownloaded()); + } // does re-download if size changes var resource4 = new Downloader.ResourceToDownload("resource", initialUrl, dest); @@ -131,7 +142,7 @@ InputStream openStreamRange(String url, long start, long end) { @Override ResourceMetadata httpHead(String url) { - return new ResourceMetadata(Optional.empty(), url, Long.MAX_VALUE, true); + return new ResourceMetadata(Optional.empty(), url, OptionalLong.of(Long.MAX_VALUE), true); } }; From a74f274c3162cd547c9b19a9a5e8705982a61755 Mon Sep 17 00:00:00 2001 From: Michael Barry Date: Tue, 2 Jul 2024 07:35:53 -0400 Subject: [PATCH 3/5] Allow centroid for other geometry types in yaml configs (#945) --- .../planetiler/geo/GeometryType.java | 7 ++- .../planetiler/geo/GeometryTypeTest.java | 7 +-- planetiler-custommap/README.md | 3 + planetiler-custommap/planetiler.schema.json | 5 +- .../configschema/FeatureGeometry.java | 8 ++- .../custommap/ConfiguredFeatureTest.java | 56 +++++++++++++++++++ 6 files changed, 79 insertions(+), 7 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java index 76df3f40ae..942b82ad3e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java @@ -13,7 +13,12 @@ import vector_tile.VectorTileProto; public enum GeometryType { - UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0, "unknown"), + UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0, "unknown") { + @Override + public Expression featureTest() { + return Expression.TRUE; + } + }, @JsonProperty("point") POINT(VectorTileProto.Tile.GeomType.POINT, 1, "point"), @JsonProperty("line") diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java index 374f221ca8..372a15b30f 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java @@ -3,7 +3,6 @@ import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.planetiler.TestUtils; @@ -42,9 +41,9 @@ void testGeometryFactory() { assertFalse(GeometryType.POLYGON.featureTest().evaluate(point)); assertTrue(GeometryType.POLYGON.featureTest().evaluate(poly)); - assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(point)); - assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(line)); - assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(poly)); + assertTrue(GeometryType.UNKNOWN.featureTest().evaluate(point)); + assertTrue(GeometryType.UNKNOWN.featureTest().evaluate(line)); + assertTrue(GeometryType.UNKNOWN.featureTest().evaluate(poly)); } @ParameterizedTest diff --git a/planetiler-custommap/README.md b/planetiler-custommap/README.md index 8515752be6..331533d2de 100644 --- a/planetiler-custommap/README.md +++ b/planetiler-custommap/README.md @@ -219,7 +219,10 @@ A feature is a defined set of objects that meet a specified filter criteria. of: - `point` `line` or `polygon` to pass the original feature through - `polygon_centroid` to match on polygons, and emit a point at the center + - `line_centroid` to match on lines, and emit a point at the center + - `centroid` to match any geometry, and emit a point at the center - `polygon_point_on_surface` to match on polygons, and emit an interior point + - `point_on_line` to match on lines, and emit a point somewhere along the line - `polygon_centroid_if_convex` to match on polygons, and if the polygon is convex emit the centroid, otherwise emit an interior point - `min_tile_cover_size` - Include objects of a certain geometry size, where 1.0 means "is diff --git a/planetiler-custommap/planetiler.schema.json b/planetiler-custommap/planetiler.schema.json index 027fe110f0..0fa3b028ae 100644 --- a/planetiler-custommap/planetiler.schema.json +++ b/planetiler-custommap/planetiler.schema.json @@ -354,8 +354,11 @@ "line", "polygon", "polygon_centroid", + "line_centroid", + "centroid", "polygon_centroid_if_convex", - "polygon_point_on_surface" + "polygon_point_on_surface", + "point_on_line" ] }, "source": { diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java index b0dc9ea9bf..96f31b66ce 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java @@ -16,10 +16,16 @@ public enum FeatureGeometry { POLYGON(GeometryType.POLYGON, FeatureCollector::polygon), @JsonProperty("polygon_centroid") POLYGON_CENTROID(GeometryType.POLYGON, FeatureCollector::centroid), + @JsonProperty("line_centroid") + LINE_CENTROID(GeometryType.LINE, FeatureCollector::centroid), + @JsonProperty("centroid") + CENTROID(GeometryType.UNKNOWN, FeatureCollector::centroid), @JsonProperty("polygon_centroid_if_convex") POLYGON_CENTROID_IF_CONVEX(GeometryType.POLYGON, FeatureCollector::centroidIfConvex), @JsonProperty("polygon_point_on_surface") - POLYGON_POINT_ON_SURFACE(GeometryType.POLYGON, FeatureCollector::pointOnSurface); + POLYGON_POINT_ON_SURFACE(GeometryType.POLYGON, FeatureCollector::pointOnSurface), + @JsonProperty("point_on_line") + POINT_ON_LINE(GeometryType.LINE, FeatureCollector::pointOnSurface); public final GeometryType geometryType; public final BiFunction geometryFactory; diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java index 5cabed4330..fc743a8925 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import org.locationtech.jts.geom.Puntal; class ConfiguredFeatureTest { private PlanetilerConfig planetilerConfig = PlanetilerConfig.defaults(); @@ -1222,4 +1223,59 @@ void testSchemaPostProcessWithMergePolygons() { ) ), loadConfig(config).findFeatureLayer("testLayer").postProcess()); } + + @Test + void testCentroid() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf + layers: + - id: testLayer + features: + - source: osm + geometry: centroid + """; + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of())); + testPolygon(config, Map.of( + "natural", "water" + ), feature -> { + assertInstanceOf(Puntal.class, feature.getGeometry()); + }, 1); + testLinestring(config, Map.of( + "natural", "water" + ), feature -> { + assertInstanceOf(Puntal.class, feature.getGeometry()); + }, 1); + testPoint(config, Map.of( + "natural", "water" + ), feature -> { + assertInstanceOf(Puntal.class, feature.getGeometry()); + }, 1); + } + + @ParameterizedTest + @ValueSource(strings = {"line_centroid", "point_on_line"}) + void testLineCentroid(String type) { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + local_path: data/rhode-island.osm.pbf + layers: + - id: testLayer + features: + - source: osm + geometry: %s + """.formatted(type); + this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of())); + testLinestring(config, Map.of( + "natural", "water" + ), feature -> { + assertInstanceOf(Puntal.class, feature.getGeometry()); + }, 1); + } } From aec1207471f6d9ff5e847de0ca4f626e5a4f3037 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 08:44:02 -0400 Subject: [PATCH 4/5] Bump jackson.version from 2.17.1 to 2.17.2 (#948) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f180afb0b8..de454af88c 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 21 21 true - 2.17.1 + 2.17.2 5.10.3 1.19.0 https://sonarcloud.io From ee622ce6fad3f35a9f97bc27666f4f9e229cd72e Mon Sep 17 00:00:00 2001 From: Strubbl <97055+Strubbl@users.noreply.github.com> Date: Mon, 8 Jul 2024 14:44:57 +0200 Subject: [PATCH 5/5] update openmaptiles dependency (#947) --- planetiler-openmaptiles | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetiler-openmaptiles b/planetiler-openmaptiles index 4469a50ad9..2957c27075 160000 --- a/planetiler-openmaptiles +++ b/planetiler-openmaptiles @@ -1 +1 @@ -Subproject commit 4469a50ad9a0a57e4707115225063533241d3dcb +Subproject commit 2957c27075c54c765b99e288df9d202520b96b2c